@lodestar/beacon-node 1.50.0-dev.67e2b54129 → 1.50.0-dev.6d20ee6eb8

package/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.50.0-dev.67e2b54129",
+  "version": "1.50.0-dev.6d20ee6eb8",
   "type": "module",
   "exports": {
     ".": {
@@ -109,7 +109,7 @@
     "@chainsafe/persistent-merkle-tree": "^1.3.1",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
     "@chainsafe/snappy-wasm": "^0.5.2",
-    "@chainsafe/ssz": "^1.8.0",
+    "@chainsafe/ssz": "^1.8.1",
     "@chainsafe/threads": "^1.11.3",
     "@crate-crypto/node-eth-kzg": "0.9.1",
     "@fastify/bearer-auth": "^10.0.1",
@@ -127,17 +127,17 @@
     "@libp2p/prometheus-metrics": "^5.0.28",
     "@libp2p/tcp": "^11.0.27",
     "@libp2p/utils": "^7.4.0",
-    "@lodestar/api": "^1.50.0-dev.67e2b54129",
-    "@lodestar/config": "^1.50.0-dev.67e2b54129",
-    "@lodestar/db": "^1.50.0-dev.67e2b54129",
-    "@lodestar/fork-choice": "^1.50.0-dev.67e2b54129",
-    "@lodestar/logger": "^1.50.0-dev.67e2b54129",
-    "@lodestar/params": "^1.50.0-dev.67e2b54129",
-    "@lodestar/reqresp": "^1.50.0-dev.67e2b54129",
-    "@lodestar/state-transition": "^1.50.0-dev.67e2b54129",
-    "@lodestar/types": "^1.50.0-dev.67e2b54129",
-    "@lodestar/utils": "^1.50.0-dev.67e2b54129",
-    "@lodestar/validator": "^1.50.0-dev.67e2b54129",
+    "@lodestar/api": "^1.50.0-dev.6d20ee6eb8",
+    "@lodestar/config": "^1.50.0-dev.6d20ee6eb8",
+    "@lodestar/db": "^1.50.0-dev.6d20ee6eb8",
+    "@lodestar/fork-choice": "^1.50.0-dev.6d20ee6eb8",
+    "@lodestar/logger": "^1.50.0-dev.6d20ee6eb8",
+    "@lodestar/params": "^1.50.0-dev.6d20ee6eb8",
+    "@lodestar/reqresp": "^1.50.0-dev.6d20ee6eb8",
+    "@lodestar/state-transition": "^1.50.0-dev.6d20ee6eb8",
+    "@lodestar/types": "^1.50.0-dev.6d20ee6eb8",
+    "@lodestar/utils": "^1.50.0-dev.6d20ee6eb8",
+    "@lodestar/validator": "^1.50.0-dev.6d20ee6eb8",
     "@multiformats/multiaddr": "^13.0.1",
     "datastore-core": "^12.0.1",
     "datastore-level": "^13.0.1",
@@ -156,10 +156,11 @@
     "xxhash-wasm": "1.0.2"
   },
   "devDependencies": {
+    "@ethereumjs/rlp": "^4.0.1",
     "@libp2p/interface-internal": "^3.1.12",
     "@libp2p/logger": "^6.2.13",
-    "@lodestar/spec-test-util": "^1.50.0-dev.67e2b54129",
-    "@lodestar/test-utils": "^1.50.0-dev.67e2b54129",
+    "@lodestar/spec-test-util": "^1.50.0-dev.6d20ee6eb8",
+    "@lodestar/test-utils": "^1.50.0-dev.6d20ee6eb8",
     "@types/js-yaml": "^4.0.5",
     "@types/qs": "^6.9.7",
     "@types/tmp": "^0.2.3",
@@ -174,5 +175,5 @@
     "beacon",
     "blockchain"
   ],
-  "gitHead": "d9cf8b5f55968a42338b728e09f610847e33c4e6"
+  "gitHead": "179dc8b642b73ccceafa81da7aa47ae79e30af27"
 }

package/src/api/impl/beacon/state/index.ts

@@ -1,6 +1,11 @@
 import {routes} from "@lodestar/api";
 import {ApplicationMethods} from "@lodestar/api/server";
-import {EPOCHS_PER_HISTORICAL_VECTOR, SLOTS_PER_EPOCH, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params";
+import {
+  EPOCHS_PER_HISTORICAL_VECTOR,
+  MIN_SEED_LOOKAHEAD,
+  SLOTS_PER_EPOCH,
+  SYNC_COMMITTEE_SUBNET_SIZE,
+} from "@lodestar/params";
 import {
   IBeaconStateView,
   computeEpochAtSlot,
@@ -11,7 +16,14 @@ import {
   isStatePostFulu,
   isStatePostGloas,
 } from "@lodestar/state-transition";
-import {ValidatorIndex, getBuilderStatus, getValidatorStatus, ssz} from "@lodestar/types";
+import {
+  ValidatorIndex,
+  getBuilderStatus,
+  getValidatorStatus,
+  mapToGeneralStatus,
+  ssz,
+  sszTypesFor,
+} from "@lodestar/types";
 import {ApiError} from "../../errors.js";
 import {ApiModules} from "../../types.js";
 import {assertUniqueItems} from "../../utils.js";
@@ -100,7 +112,8 @@ export function getBeaconStateApi({
           if (resp.valid) {
             const validatorIndex = resp.validatorIndex;
             const validator = state.getValidator(validatorIndex);
-            if (statuses.length && !statuses.includes(getValidatorStatus(validator, currentEpoch))) {
+            const status = getValidatorStatus(validator, currentEpoch);
+            if (statuses.length && !statuses.includes(status) && !statuses.includes(mapToGeneralStatus(status))) {
               continue;
             }
             const validatorResponse = toValidatorResponse(
@@ -374,6 +387,28 @@ export function getBeaconStateApi({
       };
     },
 
+    async getStatePtc({stateId, slot}) {
+      const {state, executionOptimistic, finalized} = await getState(stateId);
+      if (!isStatePostGloas(state)) {
+        throw new ApiError(400, `Cannot retrieve PTC for pre-gloas state fork=${state.forkName}`);
+      }
+
+      const usedSlot = slot ?? state.slot;
+      const epoch = computeEpochAtSlot(usedSlot);
+      const stateEpoch = computeEpochAtSlot(state.slot);
+      if (epoch < config.GLOAS_FORK_EPOCH) {
+        throw new ApiError(400, `Cannot retrieve PTC for pre-gloas slot=${usedSlot}`);
+      }
+      if (epoch < stateEpoch - 1 || epoch > stateEpoch + MIN_SEED_LOOKAHEAD) {
+        throw new ApiError(400, `Slot ${usedSlot} is outside the PTC window of state epoch ${stateEpoch}`);
+      }
+
+      return {
+        data: {slot: usedSlot, validators: Array.from(state.getPayloadTimelinessCommittee(usedSlot))},
+        meta: {executionOptimistic, finalized},
+      };
+    },
+
     async getPendingDeposits({stateId}, context) {
       const {state, executionOptimistic, finalized} = await getState(stateId);
       const fork = state.forkName;
@@ -385,7 +420,9 @@ export function getBeaconStateApi({
       const pendingDeposits = state.pendingDeposits;
 
       return {
-        data: context?.returnBytes ? ssz.electra.PendingDeposits.serialize(pendingDeposits) : pendingDeposits,
+        data: context?.returnBytes
+          ? sszTypesFor(state.forkName).PendingDeposits.serialize(pendingDeposits)
+          : pendingDeposits,
         meta: {executionOptimistic, finalized, version: fork},
       };
     },
@@ -402,7 +439,7 @@ export function getBeaconStateApi({
 
       return {
         data: context?.returnBytes
-          ? ssz.electra.PendingPartialWithdrawals.serialize(pendingPartialWithdrawals)
+          ? sszTypesFor(state.forkName).PendingPartialWithdrawals.serialize(pendingPartialWithdrawals)
           : pendingPartialWithdrawals,
         meta: {executionOptimistic, finalized, version: fork},
       };
@@ -420,7 +457,7 @@ export function getBeaconStateApi({
 
       return {
         data: context?.returnBytes
-          ? ssz.electra.PendingConsolidations.serialize(pendingConsolidations)
+          ? sszTypesFor(state.forkName).PendingConsolidations.serialize(pendingConsolidations)
           : pendingConsolidations,
         meta: {executionOptimistic, finalized, version: fork},
       };

package/src/api/impl/debug/index.ts

@@ -4,7 +4,7 @@ import {ExecutionStatus, PayloadStatus} from "@lodestar/fork-choice";
 import {ForkPostDeneb, ZERO_HASH_HEX, isForkPostDeneb, isForkPostFulu} from "@lodestar/params";
 import {computeTimeAtSlot} from "@lodestar/state-transition";
 import {BeaconState, DataColumnSidecar, DataColumnSidecars, type SignedBeaconBlock, sszTypesFor} from "@lodestar/types";
-import {toRootHex} from "@lodestar/utils";
+import {fromHex, toRootHex} from "@lodestar/utils";
 import {getBlobKzgCommitments} from "../../../util/dataColumns.js";
 import {isOptimisticBlock} from "../../../util/forkChoice.js";
 import {getStateSlotFromBytes} from "../../../util/multifork.js";
@@ -74,46 +74,55 @@ export function getDebugApi({
 
     async getDebugForkChoiceV2() {
       const {forkChoice} = chain;
+      const nodes = forkChoice.getAllNodes();
+      const head = forkChoice.getHead();
       return {
         data: {
           justifiedCheckpoint: forkChoice.getJustifiedCheckpoint(),
           finalizedCheckpoint: forkChoice.getFinalizedCheckpoint(),
-          forkChoiceNodes: forkChoice.getAllNodes().map((node) => {
-            // Payload-specific fields apply only to a revealed Gloas payload = the FULL variant of a
-            // Gloas block
-            const ptc = node.payloadStatus === PayloadStatus.FULL ? forkChoice.getPTCVoteCounts(node.blockRoot) : null;
+          forkChoiceNodes: nodes.map((node) => {
+            const parent = node.parent === undefined ? undefined : nodes[node.parent];
+            const ptc = forkChoice.getPTCVoteCounts(node.blockRoot);
             return {
-              payloadStatus: toPayloadStatusName(node.payloadStatus),
               slot: node.slot,
               blockRoot: node.blockRoot,
-              parentRoot: node.parentRoot,
+              payloadStatus: toPayloadStatusName(node.payloadStatus),
+              parentRoot:
+                node.parentBlockHash !== null && node.payloadStatus !== PayloadStatus.PENDING
+                  ? node.blockRoot
+                  : node.parentRoot,
+              parentPayloadStatus: parent === undefined ? null : toPayloadStatusName(parent.payloadStatus),
+              justifiedCheckpoint: {epoch: node.justifiedEpoch, root: fromHex(node.justifiedRoot)},
+              finalizedCheckpoint: {epoch: node.finalizedEpoch, root: fromHex(node.finalizedRoot)},
               weight: node.weight,
               validity: toForkChoiceValidity(node.executionStatus),
               executionBlockHash: node.executionPayloadBlockHash ?? ZERO_HASH_HEX,
+              payloadAttesterCount: ptc?.attesterCount ?? 0,
+              payloadAvailabilityYesCount: ptc?.payloadPresentCount ?? 0,
+              payloadDataAvailabilityYesCount: ptc?.dataAvailableCount ?? 0,
               extraData: {
+                attestationScore: node.attestationScore,
                 executionOptimistic: isOptimisticBlock(node),
+                gasLimit:
+                  node.payloadStatus === PayloadStatus.FULL && node.executionStatus !== ExecutionStatus.PreMerge
+                    ? node.executionPayloadGasLimit
+                    : null,
                 timestamp: computeTimeAtSlot(config, node.slot, chain.genesisTime),
+                stateRoot: node.stateRoot,
                 target: node.targetRoot,
-                justifiedEpoch: node.justifiedEpoch,
-                finalizedEpoch: node.finalizedEpoch,
                 unrealizedJustifiedEpoch: node.unrealizedJustifiedEpoch,
+                unrealizedJustifiedRoot: node.unrealizedJustifiedRoot,
                 unrealizedFinalizedEpoch: node.unrealizedFinalizedEpoch,
-                payloadAttesterCount: ptc?.attesterCount ?? null,
-                payloadAvailabilityYesCount: ptc?.payloadPresentCount ?? null,
-                payloadDataAvailabilityYesCount: ptc?.dataAvailableCount ?? null,
-                gasLimit:
-                  node.payloadStatus === PayloadStatus.FULL && "executionPayloadGasLimit" in node
-                    ? node.executionPayloadGasLimit
-                    : null,
+                unrealizedFinalizedRoot: node.unrealizedFinalizedRoot,
               },
             };
           }),
           extraData: {
-            unrealizedJustifiedCheckpoint: forkChoice.getUnrealizedJustifiedCheckpoint(),
-            unrealizedFinalizedCheckpoint: forkChoice.getUnrealizedFinalizedCheckpoint(),
+            head: {blockRoot: head.blockRoot, payloadStatus: toPayloadStatusName(head.payloadStatus)},
             proposerBoostRoot: forkChoice.getProposerBoostRoot(),
             previousProposerBoostRoot: forkChoice.getPreviousProposerBoostRoot(),
-            headRoot: forkChoice.getHeadRoot(),
+            unrealizedJustifiedCheckpoint: forkChoice.getUnrealizedJustifiedCheckpoint(),
+            unrealizedFinalizedCheckpoint: forkChoice.getUnrealizedFinalizedCheckpoint(),
           },
         },
       };

package/src/api/impl/validator/index.ts

@@ -1300,6 +1300,10 @@ export function getValidatorApi(
           blobs: blobsBundle.blobs,
         };
 
+        if (chain.opts.persistProducedPayloadEnvelopes) {
+          chain.persistExecutionPayloadEnvelope(blockContents.executionPayloadEnvelope);
+        }
+
         return {
           data: blockContents,
           meta: {version: fork, consensusBlockValue, executionPayloadValue, executionPayloadIncluded: true},
@@ -2187,6 +2191,10 @@ export function getValidatorApi(
         blockHash: toRootHex(executionPayload.blockHash),
       });
 
+      if (chain.opts.persistProducedPayloadEnvelopes) {
+        chain.persistExecutionPayloadEnvelope(envelope);
+      }
+
       return {
         data: envelope,
         meta: {

package/src/chain/archiveStore/archiveStore.ts

@@ -113,7 +113,9 @@ export class ArchiveStore {
           this.logger,
           this.metrics,
           this.opts.anchorState.finalizedCheckpoint.epoch,
-          this.chain.clock.currentEpoch
+          this.chain.clock.currentEpoch,
+          this.statePruneFromSlot,
+          this.advanceEarliestAvailableSlot
         ),
         () => this.logger.info("Still pruning historical data, please wait..."),
         30_000,
@@ -122,26 +124,6 @@ export class ArchiveStore {
       this.statePruneFromSlot = stateCutoffSlot;
     }
 
-    // Initialize earliestAvailableSlot from the earliest block actually retained (after any pruning
-    // above), rather than the anchor state slot set in the constructor. On an in-place restart the DB
-    // still holds finalized history below the anchor; leaving the value at the anchor makes the node
-    // reject by_range requests (beacon_blocks, execution_payload_envelopes and data_column_sidecars
-    // all gate on this slot) for data it still retains. An empty archive (e.g. a fresh checkpoint
-    // sync with nothing below the anchor) leaves the constructor's anchor slot in place.
-    const earliestArchivedBlockSlot = await this.db.blockArchive.firstKey();
-    if (earliestArchivedBlockSlot != null) {
-      const oldEarliestAvailableSlot = this.chain.earliestAvailableSlot;
-      this.chain.earliestAvailableSlot = earliestArchivedBlockSlot;
-      this.logger.verbose("Initialized earliestAvailableSlot from retained block archive", {
-        oldEarliestAvailableSlot,
-        newEarliestAvailableSlot: earliestArchivedBlockSlot,
-      });
-    } else {
-      this.logger.verbose("Empty block archive on init, keeping anchor earliestAvailableSlot", {
-        earliestAvailableSlot: this.chain.earliestAvailableSlot,
-      });
-    }
-
     if (this.opts.serveHistoricalState) {
       this.historicalStateRegen = await HistoricalStateRegen.init({
         opts: {
@@ -158,6 +140,18 @@ export class ArchiveStore {
     }
   }
 
+  /**
+   * The CLI owns the initial value; this only advances it during pruneHistory.
+   * Persisted before any deletion so a crash mid-prune never leaves the floor below deleted data.
+   */
+  private advanceEarliestAvailableSlot = async (blockCutoffSlot: Slot): Promise<void> => {
+    const earliestAvailableSlot = Math.max(this.chain.earliestAvailableSlot, blockCutoffSlot);
+    if (earliestAvailableSlot !== this.chain.earliestAvailableSlot) {
+      await this.db.earliestAvailableSlot.set(earliestAvailableSlot);
+      this.chain.earliestAvailableSlot = earliestAvailableSlot;
+    }
+  };
+
   async close(): Promise<void> {
     await this.historicalStateRegen?.close();
   }
@@ -259,17 +253,17 @@ export class ArchiveStore {
 
       if (this.opts.pruneHistory) {
         timer = this.metrics?.processFinalizedCheckpoint.durationByTask.startTimer();
-        const {blockCutoffSlot, stateCutoffSlot} = await pruneHistory(
+        const {stateCutoffSlot} = await pruneHistory(
           this.chain.config,
           this.db,
           this.logger,
           this.metrics,
           finalizedEpoch,
           this.chain.clock.currentEpoch,
-          this.statePruneFromSlot
+          this.statePruneFromSlot,
+          this.advanceEarliestAvailableSlot
         );
         this.statePruneFromSlot = stateCutoffSlot;
-        this.chain.earliestAvailableSlot = Math.max(this.chain.earliestAvailableSlot, blockCutoffSlot);
         timer?.({source: ArchiveStoreTask.PruneHistory});
       }
 

package/src/chain/archiveStore/utils/pruneHistory.ts

@@ -12,7 +12,8 @@ export async function pruneHistory(
   metrics: Metrics | null | undefined,
   finalizedEpoch: Epoch,
   currentEpoch: Epoch,
-  statePruneFromSlot: Slot = 0
+  statePruneFromSlot: Slot,
+  beforePrune: (blockCutoffSlot: Slot) => Promise<void>
 ): Promise<{blockCutoffSlot: Slot; stateCutoffSlot: Slot}> {
   const blockCutoffEpoch = Math.min(
     // set by config, with underflow protection
@@ -51,6 +52,8 @@ export async function pruneHistory(
     stateSlots: prettyPrintIndices(states),
   });
 
+  await beforePrune(blockCutoffSlot);
+
   const step1 = metrics?.pruneHistory.pruneKeys.startTimer();
   await Promise.all([
     // ->

package/src/chain/blocks/importBlock.ts

@@ -22,6 +22,7 @@ import {
   RootCache,
   computeEpochAtSlot,
   computeTimeAtSlot,
+  getBeaconCommittees,
   isStatePostAltair,
   isStatePostBellatrix,
 } from "@lodestar/state-transition";
@@ -129,11 +130,17 @@ export async function importBlock(
 
   // Should compute checkpoint balances before forkchoice.onBlock
   this.checkpointBalancesCache.processState(blockRootHex, postState);
-  if (fork >= ForkSeq.gloas) {
+  if (isGloasBeaconBlock(block.message)) {
+    // A gloas block has no payload of its own, it inherits the status of the payload it builds on,
+    // i.e. the parent variant matching the bid's parent block hash. The parent's default (PENDING)
+    // variant does not reflect a SYNCING parent payload, and onBlock would then mark that payload VALID.
     const parentRootHex = toRootHex(block.message.parentRoot);
-    const parentBlock = this.forkChoice.getBlockHexDefaultStatus(parentRootHex);
+    const parentBlockHashHex = toRootHex(block.message.body.signedExecutionPayloadBid.message.parentBlockHash);
+    const parentBlock = this.forkChoice.getBlockHexAndBlockHash(parentRootHex, parentBlockHashHex);
     if (parentBlock === null) {
-      throw Error(`Parent block not found in forkChoice, parentRoot=${parentRootHex}`);
+      throw Error(
+        `Parent block not found in forkChoice, parentRoot=${parentRootHex} parentBlockHash=${parentBlockHashHex}`
+      );
     }
     if (parentBlock.executionStatus === ExecutionStatus.Invalid) {
       throw Error(`Parent block has invalid execution status, parentRoot=${parentRootHex}`);
@@ -657,8 +664,12 @@ export function addAttestationPostElectra(
   } else {
     const attSlot = attestation.data.slot;
     const attEpoch = computeEpochAtSlot(attSlot);
-    const decisionRoot = state.getShufflingDecisionRoot(attEpoch);
-    const committees = this.shufflingCache.getBeaconCommittees(attEpoch, decisionRoot, attSlot, committeeIndices);
+    // calling getShufflingAtEpoch may take some time for NativeBeaconStateView
+    // we should have the shuffling inside ShufflingCache most of the time
+    const shuffling =
+      this.shufflingCache.getSync(attEpoch, state.getShufflingDecisionRoot(attEpoch)) ??
+      state.getShufflingAtEpoch(attEpoch);
+    const committees = getBeaconCommittees(shuffling, attSlot, committeeIndices);
     const aggregationBools = attestation.aggregationBits.toBoolArray();
     let offset = 0;
     for (let i = 0; i < committees.length; i++) {

package/src/chain/blocks/verifyBlock.ts

@@ -4,6 +4,7 @@ import {
   DataAvailabilityStatus,
   IBeaconStateView,
   computeEpochAtSlot,
+  getIndexedAttestation,
   signedBlockToSignedHeader,
 } from "@lodestar/state-transition";
 import {IndexedAttestation, Slot, deneb} from "@lodestar/types";
@@ -81,10 +82,6 @@ export async function verifyBlocksInEpoch(
       throw new BlockError(block0, {code: BlockErrorCode.PRESTATE_MISSING, error: e as Error});
     });
 
-  // in forky condition, make sure to populate ShufflingCache with regened state
-  // otherwise it may fail to get indexed attestations from shuffling cache later
-  this.shufflingCache.processState(preState0);
-
   if (!preState0.isStateValidatorsNodesPopulated()) {
     this.logger.verbose("verifyBlocksInEpoch preState0 SSZ cache stats", {
       slot: preState0.slot,
@@ -117,8 +114,12 @@ export async function verifyBlocksInEpoch(
     for (const [i, block] of blocks.entries()) {
       indexedAttestationsByBlock[i] = block.message.body.attestations.map((attestation) => {
         const attEpoch = computeEpochAtSlot(attestation.data.slot);
-        const decisionRoot = preState0.getShufflingDecisionRoot(attEpoch);
-        return this.shufflingCache.getIndexedAttestation(attEpoch, decisionRoot, fork, attestation);
+        // calling getShufflingAtEpoch may take some time for NativeBeaconStateView
+        // we should have the shuffling inside ShufflingCache most of the time
+        const shuffling =
+          this.shufflingCache.getSync(attEpoch, preState0.getShufflingDecisionRoot(attEpoch)) ??
+          preState0.getShufflingAtEpoch(attEpoch);
+        return getIndexedAttestation(shuffling, fork, attestation);
       });
     }
 

package/src/chain/chain.ts

@@ -128,7 +128,7 @@ import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js";
 import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js";
 import {SeenBlockAttesters} from "./seenCache/seenBlockAttesters.js";
 import {SeenBlockInput} from "./seenCache/seenGossipBlockInput.js";
-import {ShufflingCache} from "./shufflingCache.js";
+import {ShufflingCache, ShufflingPromiseCancelReason} from "./shufflingCache.js";
 import {DbCPStateDatastore, checkpointToDatastoreKey} from "./stateCache/datastore/db.js";
 import {FileCPStateDatastore} from "./stateCache/datastore/file.js";
 import {CPStateDatastore} from "./stateCache/datastore/types.js";
@@ -279,6 +279,7 @@ export class BeaconChain implements IBeaconChain {
       validatorMonitor,
       anchorState,
       isAnchorStateFinalized,
+      earliestAvailableSlot,
       executionEngine,
       executionBuilder,
       builderApiClientOpts,
@@ -298,6 +299,7 @@ export class BeaconChain implements IBeaconChain {
       validatorMonitor: ValidatorMonitor | null;
       anchorState: IBeaconStateView;
       isAnchorStateFinalized: boolean;
+      earliestAvailableSlot: Slot;
       executionEngine: IExecutionEngine;
       executionBuilder?: IExecutionBuilder;
       builderApiClientOpts?: BuilderApiClientOpts;
@@ -326,7 +328,7 @@ export class BeaconChain implements IBeaconChain {
 
     this.blacklistedBlocks = new Map((opts.blacklistedBlocks ?? []).map((hex) => [hex, null]));
     this.attestationPool = new AttestationPool(config, clock, this.opts?.preaggregateSlotDistance, metrics);
-    this.aggregatedAttestationPool = new AggregatedAttestationPool(this.config, metrics);
+    this.aggregatedAttestationPool = new AggregatedAttestationPool(this.config, metrics, logger);
     this.syncCommitteeMessagePool = new SyncCommitteeMessagePool(config, clock, this.opts?.preaggregateSlotDistance);
     this.syncContributionAndProofPool = new SyncContributionAndProofPool(config, clock, metrics, logger);
     this.executionPayloadBidPool = new ExecutionPayloadBidPool();
@@ -363,7 +365,7 @@ export class BeaconChain implements IBeaconChain {
       logger,
     });
 
-    this._earliestAvailableSlot = anchorState.slot;
+    this._earliestAvailableSlot = earliestAvailableSlot;
 
     this.shufflingCache = new ShufflingCache(metrics, logger, this.opts, [
       {
@@ -574,7 +576,11 @@ export class BeaconChain implements IBeaconChain {
   }
 
   seenBlock(blockRoot: RootHex): boolean {
-    return this.seenBlockInputCache.hasBlock(blockRoot) || this.forkChoice.hasBlockHexUnsafe(blockRoot);
+    return (
+      this.seenBlockInputCache.hasBlock(blockRoot) ||
+      this.seenBlockInputCache.isValidatingBlock(blockRoot) ||
+      this.forkChoice.hasBlockHexUnsafe(blockRoot)
+    );
   }
 
   seenPayloadEnvelope(blockRoot: RootHex): boolean {
@@ -614,12 +620,14 @@ export class BeaconChain implements IBeaconChain {
   async loadFromDisk(): Promise<void> {
     await this.regen.init();
     await this.opPool.fromPersisted(this.db, this.getHeadState(), this.bls, this.clock.currentSlot);
+    await this.proposerPreferencesPool.fromPersisted(this.db, this.clock.currentSlot);
   }
 
   /** Persist in-memory data to the DB. Call at least once before stopping the process */
   async persistToDisk(): Promise<void> {
     await this.archiveStore.persistToDisk();
     await this.opPool.toPersisted(this.db);
+    await this.proposerPreferencesPool.toPersisted(this.db);
   }
 
   getHeadState(): IBeaconStateView {
@@ -1536,6 +1544,22 @@ export class BeaconChain implements IBeaconChain {
     }
   }
 
+  persistExecutionPayloadEnvelope(envelope: gloas.ExecutionPayloadEnvelope): void {
+    const blockRoot = toRootHex(envelope.beaconBlockRoot);
+    void this.persistSszObject(
+      "ExecutionPayloadEnvelope",
+      ssz.gloas.ExecutionPayloadEnvelope.serialize(envelope),
+      blockRoot,
+      "produced_execution_payload_envelope"
+    ).catch((e) => {
+      this.logger.error(
+        "Error persisting produced execution payload envelope",
+        {slot: envelope.payload.slotNumber, blockRoot},
+        e as Error
+      );
+    });
+  }
+
   /**
    * Invalid state root error is critical and it causes the node to stale most of the time so we want to always
    * persist preState, postState and block for further investigation.
@@ -1607,21 +1631,48 @@ export class BeaconChain implements IBeaconChain {
     const blockEpoch = computeEpochAtSlot(attHeadBlock.slot);
 
     let state: IBeaconStateView;
-    if (blockEpoch < attEpoch - 1) {
-      // thanks to one epoch look ahead, we don't need to dial up to attEpoch
-      const targetSlot = computeStartSlotAtEpoch(attEpoch - 1);
-      this.metrics?.gossipAttestation.useHeadBlockStateDialedToTargetEpoch.inc({caller: regenCaller});
-      state = await this.regen.getBlockSlotState(attHeadBlock, targetSlot, {dontTransferCache: true}, regenCaller);
-    } else if (blockEpoch > attEpoch) {
-      // should not happen, handled inside attestation verification code
-      throw Error(`Block epoch ${blockEpoch} is after attestation epoch ${attEpoch}`);
-    } else {
-      // should use either current or next shuffling of head state
-      // it's not likely to hit this since these shufflings are cached already
-      // so handle just in case
-      this.metrics?.gossipAttestation.useHeadBlockState.inc({caller: regenCaller});
-      state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller);
+    try {
+      if (blockEpoch < attEpoch - 1) {
+        // thanks to one epoch look ahead, we don't need to dial up to attEpoch
+        const targetSlot = computeStartSlotAtEpoch(attEpoch - 1);
+        this.metrics?.gossipAttestation.useHeadBlockStateDialedToTargetEpoch.inc({caller: regenCaller});
+        state = await this.regen.getBlockSlotState(attHeadBlock, targetSlot, {dontTransferCache: true}, regenCaller);
+      } else if (blockEpoch > attEpoch) {
+        // should not happen, handled inside attestation verification code
+        throw Error(`Block epoch ${blockEpoch} is after attestation epoch ${attEpoch}`);
+      } else {
+        // should use either current or next shuffling of head state
+        // it's not likely to hit this since these shufflings are cached already
+        // so handle just in case
+        this.metrics?.gossipAttestation.useHeadBlockState.inc({caller: regenCaller});
+        state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller);
+      }
+    } catch (e) {
+      this.shufflingCache.cancelPromise(attEpoch, shufflingDependentRoot, ShufflingPromiseCancelReason.regenError);
+      throw e;
+    }
+
+    const stateDecisionRoot = state.getShufflingDecisionRoot(attEpoch);
+    if (stateDecisionRoot !== shufflingDependentRoot) {
+      // the promise would never be resolved by the state's shuffling
+      this.shufflingCache.cancelPromise(
+        attEpoch,
+        shufflingDependentRoot,
+        ShufflingPromiseCancelReason.decisionRootMismatch
+      );
+      this.logger.debug("Shuffling decision root mismatch on attestation regen", {
+        slot: attHeadBlock.slot,
+        root: attHeadBlock.blockRoot,
+        epoch: attEpoch,
+        shufflingDependentRoot,
+        stateDecisionRoot,
+      });
+
+      throw Error(
+        `Shuffling decision root mismatch epoch=${attEpoch} slot=${attHeadBlock.slot} root=${attHeadBlock.blockRoot} expected=${shufflingDependentRoot} actual=${stateDecisionRoot}`
+      );
     }
+
     // resolve the promise to unblock other calls of the same epoch and dependent root
     this.shufflingCache.processState(state);
     return state.getShufflingAtEpoch(attEpoch);
@@ -1731,7 +1782,7 @@ export class BeaconChain implements IBeaconChain {
     // remove date suffixes in file name, and check duplicate to avoid redundant persistence
     await writeIfNotExist(filepath, bytes);
 
-    this.logger.debug("Persisted invalid ssz object", {id: logStr, filepath});
+    this.logger.debug("Persisted SSZ object", {id: logStr, filepath});
   }
 
   private onScrapeMetrics(metrics: Metrics): void {

package/src/chain/interface.ts

@@ -290,6 +290,7 @@ export interface IBeaconChain {
   updateBeaconProposerData(epoch: Epoch, proposers: ProposerPreparationData[]): Promise<void>;
 
   persistBlock(data: BeaconBlock | BlindedBeaconBlock, suffix?: string): void;
+  persistExecutionPayloadEnvelope(envelope: gloas.ExecutionPayloadEnvelope): void;
   persistInvalidStateRoot(
     preState: IBeaconStateView,
     postState: IBeaconStateView,