@lodestar/beacon-node 1.50.0-dev.023ad6c012 → 1.50.0-dev.03c05fa09a

package/src/chain/blocks/importBlock.ts

@@ -22,6 +22,7 @@ import {
   RootCache,
   computeEpochAtSlot,
   computeTimeAtSlot,
+  getBeaconCommittees,
   isStatePostAltair,
   isStatePostBellatrix,
 } from "@lodestar/state-transition";
@@ -44,7 +45,7 @@ import {isQueueErrorAborted} from "../../util/queue/index.js";
 import type {BeaconChain} from "../chain.js";
 import {ChainEvent, ReorgEventData} from "../emitter.js";
 import {ForkchoiceCaller} from "../forkChoice/index.js";
-import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js";
+import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_BPS} from "../reprocess.js";
 import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js";
 import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
 import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
@@ -129,11 +130,17 @@ export async function importBlock(
 
   // Should compute checkpoint balances before forkchoice.onBlock
   this.checkpointBalancesCache.processState(blockRootHex, postState);
-  if (fork >= ForkSeq.gloas) {
+  if (isGloasBeaconBlock(block.message)) {
+    // A gloas block has no payload of its own, it inherits the status of the payload it builds on,
+    // i.e. the parent variant matching the bid's parent block hash. The parent's default (PENDING)
+    // variant does not reflect a SYNCING parent payload, and onBlock would then mark that payload VALID.
     const parentRootHex = toRootHex(block.message.parentRoot);
-    const parentBlock = this.forkChoice.getBlockHexDefaultStatus(parentRootHex);
+    const parentBlockHashHex = toRootHex(block.message.body.signedExecutionPayloadBid.message.parentBlockHash);
+    const parentBlock = this.forkChoice.getBlockHexAndBlockHash(parentRootHex, parentBlockHashHex);
     if (parentBlock === null) {
-      throw Error(`Parent block not found in forkChoice, parentRoot=${parentRootHex}`);
+      throw Error(
+        `Parent block not found in forkChoice, parentRoot=${parentRootHex} parentBlockHash=${parentBlockHashHex}`
+      );
     }
     if (parentBlock.executionStatus === ExecutionStatus.Invalid) {
       throw Error(`Parent block has invalid execution status, parentRoot=${parentRootHex}`);
@@ -328,7 +335,8 @@ export async function importBlock(
       this.metrics.headSlot.set(newHead.slot);
       // Only track "recent" blocks. Otherwise sync can distort this metrics heavily.
       // We want to track recent blocks coming from gossip, unknown block sync, and API.
-      if (delaySec < (SLOTS_PER_EPOCH * this.config.SLOT_DURATION_MS) / 1000) {
+      const headFork = this.config.getForkName(newHead.slot);
+      if (delaySec < (SLOTS_PER_EPOCH * this.config.getSlotDurationMs(headFork)) / 1000) {
         this.metrics.importBlock.elapsedTimeTillBecomeHead.observe(delaySec);
         const cutOffSec = this.config.getAttestationDueMs(this.config.getForkName(blockSlot)) / 1000;
         if (delaySec > cutOffSec) {
@@ -400,9 +408,8 @@ export async function importBlock(
     const proposalSlot = blockSlot + 1;
     try {
       const proposerIndex = postState.getBeaconProposer(proposalSlot);
-      const feeRecipient = this.beaconProposerCache.get(proposerIndex);
 
-      if (feeRecipient) {
+      if (this.beaconProposerCache.has(proposerIndex)) {
         // We would set this to true if
         //  1) This is a gossip block
         //  2) We are proposer of next slot
@@ -580,7 +587,7 @@ export async function importBlock(
     this.validatorMonitor?.registerSyncAggregateInBlock(
       blockEpoch,
       (block as altair.SignedBeaconBlock).message.body.syncAggregate,
-      fullyVerifiedBlock.postState.currentSyncCommitteeIndexed.validatorIndices
+      fullyVerifiedBlock.postState.currentSyncCommitteeValidatorIndices
     );
   }
 
@@ -594,7 +601,10 @@ export async function importBlock(
     }
   }
 
-  const advancedSlot = this.clock.slotWithFutureTolerance(REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC);
+  const reprocessGuardSec =
+    this.config.getSlotComponentDurationMs(this.config.getForkName(blockSlot), REPROCESS_MIN_TIME_TO_NEXT_SLOT_BPS) /
+    1000;
+  const advancedSlot = this.clock.slotWithFutureTolerance(reprocessGuardSec);
 
   // Gossip blocks need to be imported as soon as possible, waiting attestations could be processed
   // in the next event loop. See https://github.com/ChainSafe/lodestar/issues/4789
@@ -657,8 +667,12 @@ export function addAttestationPostElectra(
   } else {
     const attSlot = attestation.data.slot;
     const attEpoch = computeEpochAtSlot(attSlot);
-    const decisionRoot = state.getShufflingDecisionRoot(attEpoch);
-    const committees = this.shufflingCache.getBeaconCommittees(attEpoch, decisionRoot, attSlot, committeeIndices);
+    // calling getShufflingAtEpoch may take some time for NativeBeaconStateView
+    // we should have the shuffling inside ShufflingCache most of the time
+    const shuffling =
+      this.shufflingCache.getSync(attEpoch, state.getShufflingDecisionRoot(attEpoch)) ??
+      state.getShufflingAtEpoch(attEpoch);
+    const committees = getBeaconCommittees(shuffling, attSlot, committeeIndices);
     const aggregationBools = attestation.aggregationBits.toBoolArray();
     let offset = 0;
     for (let i = 0; i < committees.length; i++) {

package/src/chain/blocks/verifyBlock.ts

@@ -4,6 +4,7 @@ import {
   DataAvailabilityStatus,
   IBeaconStateView,
   computeEpochAtSlot,
+  getIndexedAttestation,
   signedBlockToSignedHeader,
 } from "@lodestar/state-transition";
 import {IndexedAttestation, Slot, deneb} from "@lodestar/types";
@@ -81,10 +82,6 @@ export async function verifyBlocksInEpoch(
       throw new BlockError(block0, {code: BlockErrorCode.PRESTATE_MISSING, error: e as Error});
     });
 
-  // in forky condition, make sure to populate ShufflingCache with regened state
-  // otherwise it may fail to get indexed attestations from shuffling cache later
-  this.shufflingCache.processState(preState0);
-
   if (!preState0.isStateValidatorsNodesPopulated()) {
     this.logger.verbose("verifyBlocksInEpoch preState0 SSZ cache stats", {
       slot: preState0.slot,
@@ -117,8 +114,12 @@ export async function verifyBlocksInEpoch(
     for (const [i, block] of blocks.entries()) {
       indexedAttestationsByBlock[i] = block.message.body.attestations.map((attestation) => {
         const attEpoch = computeEpochAtSlot(attestation.data.slot);
-        const decisionRoot = preState0.getShufflingDecisionRoot(attEpoch);
-        return this.shufflingCache.getIndexedAttestation(attEpoch, decisionRoot, fork, attestation);
+        // calling getShufflingAtEpoch may take some time for NativeBeaconStateView
+        // we should have the shuffling inside ShufflingCache most of the time
+        const shuffling =
+          this.shufflingCache.getSync(attEpoch, preState0.getShufflingDecisionRoot(attEpoch)) ??
+          preState0.getShufflingAtEpoch(attEpoch);
+        return getIndexedAttestation(shuffling, fork, attestation);
       });
     }
 
@@ -188,6 +189,7 @@ export async function verifyBlocksInEpoch(
         blockInputs,
         // hack availability for state transition eval as availability is separately determined
         blocks.map(() => DataAvailabilityStatus.Available),
+        this.serializedCache,
         this.logger,
         this.metrics,
         this.validatorMonitor,

package/src/chain/blocks/verifyBlocksStateTransitionOnly.ts

@@ -7,6 +7,7 @@ import {
 import {ErrorAborted, Logger, byteArrayEquals} from "@lodestar/utils";
 import {Metrics} from "../../metrics/index.js";
 import {nextEventLoop} from "../../util/eventLoop.js";
+import {SerializedCache} from "../../util/serializedCache.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {BlockProcessOpts} from "../options.js";
 import {ValidatorMonitor} from "../validatorMonitor.js";
@@ -25,6 +26,7 @@ export async function verifyBlocksStateTransitionOnly(
   preState0: IBeaconStateView,
   blocks: IBlockInput[],
   dataAvailabilityStatuses: DataAvailabilityStatus[],
+  serializedCache: SerializedCache,
   logger: Logger,
   metrics: Metrics | null,
   validatorMonitor: ValidatorMonitor | null,
@@ -37,7 +39,8 @@ export async function verifyBlocksStateTransitionOnly(
 
   for (let i = 0; i < blocks.length; i++) {
     const {validProposerSignature, validSignatures} = opts;
-    const block = blocks[i].getBlock();
+    const blockInput = blocks[i];
+    const block = blockInput.getBlock();
     const preState = i === 0 ? preState0 : postStates[i - 1];
     const dataAvailabilityStatus = dataAvailabilityStatuses[i];
 
@@ -47,7 +50,7 @@ export async function verifyBlocksStateTransitionOnly(
     let postState: IBeaconStateView;
     try {
       postState = preState.stateTransition(
-        block,
+        {block, ssz: serializedCache.get(block)},
         {
           // NOTE: Assume valid for now while sending payload to execution engine in parallel
           // Latter verifyBlocksInEpoch() will make sure that payload is indeed valid
@@ -60,7 +63,7 @@ export async function verifyBlocksStateTransitionOnly(
           verifySignatures: !useBlsBatchVerify && !validSignatures,
           dontTransferCache: false,
         },
-        {metrics, validatorMonitor}
+        {metrics: metrics?.stateTransition, validatorMonitor}
       );
     } catch (e) {
       throw new BlockError(block, {

package/src/chain/chain.ts

@@ -58,7 +58,7 @@ import {
 } from "@lodestar/types";
 import {Logger, fromHex, gweiToWei, isErrorAborted, pruneSetToMax, sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
-import {GENESIS_EPOCH, ZERO_HASH} from "../constants/index.js";
+import {GENESIS_EPOCH, HALF_SLOT_BPS, ZERO_HASH} from "../constants/index.js";
 import {IBeaconDb} from "../db/index.js";
 import {BLOB_SIDECARS_IN_WRAPPER_INDEX} from "../db/repositories/blobSidecars.js";
 import {decodeArchivedEnvelope} from "../db/repositories/index.js";
@@ -128,7 +128,7 @@ import {SeenAggregatedAttestations} from "./seenCache/seenAggregateAndProof.js";
 import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js";
 import {SeenBlockAttesters} from "./seenCache/seenBlockAttesters.js";
 import {SeenBlockInput} from "./seenCache/seenGossipBlockInput.js";
-import {ShufflingCache} from "./shufflingCache.js";
+import {ShufflingCache, ShufflingPromiseCancelReason} from "./shufflingCache.js";
 import {DbCPStateDatastore, checkpointToDatastoreKey} from "./stateCache/datastore/db.js";
 import {FileCPStateDatastore} from "./stateCache/datastore/file.js";
 import {CPStateDatastore} from "./stateCache/datastore/types.js";
@@ -244,6 +244,7 @@ export class BeaconChain implements IBeaconChain {
   protected readonly blockProcessor: BlockProcessor;
   protected readonly payloadEnvelopeProcessor: PayloadEnvelopeProcessor;
   protected readonly db: IBeaconDb;
+  private genesisStateRoot?: RootHex | null;
   // this is only available if nHistoricalStates is enabled
   private readonly cpStateDatastore?: CPStateDatastore;
   private abortController = new AbortController();
@@ -278,6 +279,7 @@ export class BeaconChain implements IBeaconChain {
       validatorMonitor,
       anchorState,
       isAnchorStateFinalized,
+      earliestAvailableSlot,
       executionEngine,
       executionBuilder,
       builderApiClientOpts,
@@ -297,6 +299,7 @@ export class BeaconChain implements IBeaconChain {
       validatorMonitor: ValidatorMonitor | null;
       anchorState: IBeaconStateView;
       isAnchorStateFinalized: boolean;
+      earliestAvailableSlot: Slot;
       executionEngine: IExecutionEngine;
       executionBuilder?: IExecutionBuilder;
       builderApiClientOpts?: BuilderApiClientOpts;
@@ -325,7 +328,7 @@ export class BeaconChain implements IBeaconChain {
 
     this.blacklistedBlocks = new Map((opts.blacklistedBlocks ?? []).map((hex) => [hex, null]));
     this.attestationPool = new AttestationPool(config, clock, this.opts?.preaggregateSlotDistance, metrics);
-    this.aggregatedAttestationPool = new AggregatedAttestationPool(this.config, metrics);
+    this.aggregatedAttestationPool = new AggregatedAttestationPool(this.config, metrics, logger);
     this.syncCommitteeMessagePool = new SyncCommitteeMessagePool(config, clock, this.opts?.preaggregateSlotDistance);
     this.syncContributionAndProofPool = new SyncContributionAndProofPool(config, clock, metrics, logger);
     this.executionPayloadBidPool = new ExecutionPayloadBidPool();
@@ -362,7 +365,7 @@ export class BeaconChain implements IBeaconChain {
       logger,
     });
 
-    this._earliestAvailableSlot = anchorState.slot;
+    this._earliestAvailableSlot = earliestAvailableSlot;
 
     this.shufflingCache = new ShufflingCache(metrics, logger, this.opts, [
       {
@@ -432,6 +435,7 @@ export class BeaconChain implements IBeaconChain {
       blockStateCache,
       checkpointStateCache,
       seenBlockInputCache: this.seenBlockInputCache,
+      serializedCache: this.serializedCache,
       db,
       metrics,
       validatorMonitor,
@@ -572,7 +576,11 @@ export class BeaconChain implements IBeaconChain {
   }
 
   seenBlock(blockRoot: RootHex): boolean {
-    return this.seenBlockInputCache.hasBlock(blockRoot) || this.forkChoice.hasBlockHexUnsafe(blockRoot);
+    return (
+      this.seenBlockInputCache.hasBlock(blockRoot) ||
+      this.seenBlockInputCache.isValidatingBlock(blockRoot) ||
+      this.forkChoice.hasBlockHexUnsafe(blockRoot)
+    );
   }
 
   seenPayloadEnvelope(blockRoot: RootHex): boolean {
@@ -612,12 +620,14 @@ export class BeaconChain implements IBeaconChain {
   async loadFromDisk(): Promise<void> {
     await this.regen.init();
     await this.opPool.fromPersisted(this.db, this.getHeadState(), this.bls, this.clock.currentSlot);
+    await this.proposerPreferencesPool.fromPersisted(this.db, this.clock.currentSlot);
   }
 
   /** Persist in-memory data to the DB. Call at least once before stopping the process */
   async persistToDisk(): Promise<void> {
     await this.archiveStore.persistToDisk();
     await this.opPool.toPersisted(this.db);
+    await this.proposerPreferencesPool.toPersisted(this.db);
   }
 
   getHeadState(): IBeaconStateView {
@@ -702,6 +712,37 @@ export class BeaconChain implements IBeaconChain {
     stateRoot: RootHex,
     opts?: StateGetOpts
   ): Promise<{state: IBeaconStateView | Uint8Array; executionOptimistic: boolean; finalized: boolean} | null> {
+    const finalizedBlock = this.forkChoice.getFinalizedBlock();
+    const finalizedCheckpoint = this.forkChoice.getFinalizedCheckpoint();
+    // Checkpoint state only equals the block post-state if the block is at the epoch start slot
+    if (
+      finalizedBlock.stateRoot === stateRoot &&
+      finalizedBlock.slot === computeStartSlotAtEpoch(finalizedCheckpoint.epoch)
+    ) {
+      const state = this.regen.getCheckpointStateSync({
+        epoch: finalizedCheckpoint.epoch,
+        rootHex: finalizedCheckpoint.rootHex,
+      });
+      if (state) {
+        return {
+          state,
+          executionOptimistic: isOptimisticBlock(finalizedBlock),
+          finalized: finalizedCheckpoint.epoch !== GENESIS_EPOCH,
+        };
+      }
+    }
+
+    if (this.genesisStateRoot === undefined) {
+      const genesisBlock = await this.db.blockArchive.get(GENESIS_SLOT);
+      this.genesisStateRoot = genesisBlock ? toRootHex(genesisBlock.message.stateRoot) : null;
+    }
+    if (this.genesisStateRoot === stateRoot) {
+      const state = await this.db.stateArchive.getBinary(GENESIS_SLOT);
+      if (state) {
+        return {state, executionOptimistic: false, finalized: finalizedCheckpoint.epoch !== GENESIS_EPOCH};
+      }
+    }
+
     if (opts?.allowRegen) {
       const state = await this.regen.getState(stateRoot, RegenCaller.restApi);
       const block = this.forkChoice.getBlockDefaultStatus(
@@ -1503,6 +1544,22 @@ export class BeaconChain implements IBeaconChain {
     }
   }
 
+  persistExecutionPayloadEnvelope(envelope: gloas.ExecutionPayloadEnvelope): void {
+    const blockRoot = toRootHex(envelope.beaconBlockRoot);
+    void this.persistSszObject(
+      "ExecutionPayloadEnvelope",
+      ssz.gloas.ExecutionPayloadEnvelope.serialize(envelope),
+      blockRoot,
+      "produced_execution_payload_envelope"
+    ).catch((e) => {
+      this.logger.error(
+        "Error persisting produced execution payload envelope",
+        {slot: envelope.payload.slotNumber, blockRoot},
+        e as Error
+      );
+    });
+  }
+
   /**
    * Invalid state root error is critical and it causes the node to stale most of the time so we want to always
    * persist preState, postState and block for further investigation.
@@ -1574,21 +1631,48 @@ export class BeaconChain implements IBeaconChain {
     const blockEpoch = computeEpochAtSlot(attHeadBlock.slot);
 
     let state: IBeaconStateView;
-    if (blockEpoch < attEpoch - 1) {
-      // thanks to one epoch look ahead, we don't need to dial up to attEpoch
-      const targetSlot = computeStartSlotAtEpoch(attEpoch - 1);
-      this.metrics?.gossipAttestation.useHeadBlockStateDialedToTargetEpoch.inc({caller: regenCaller});
-      state = await this.regen.getBlockSlotState(attHeadBlock, targetSlot, {dontTransferCache: true}, regenCaller);
-    } else if (blockEpoch > attEpoch) {
-      // should not happen, handled inside attestation verification code
-      throw Error(`Block epoch ${blockEpoch} is after attestation epoch ${attEpoch}`);
-    } else {
-      // should use either current or next shuffling of head state
-      // it's not likely to hit this since these shufflings are cached already
-      // so handle just in case
-      this.metrics?.gossipAttestation.useHeadBlockState.inc({caller: regenCaller});
-      state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller);
+    try {
+      if (blockEpoch < attEpoch - 1) {
+        // thanks to one epoch look ahead, we don't need to dial up to attEpoch
+        const targetSlot = computeStartSlotAtEpoch(attEpoch - 1);
+        this.metrics?.gossipAttestation.useHeadBlockStateDialedToTargetEpoch.inc({caller: regenCaller});
+        state = await this.regen.getBlockSlotState(attHeadBlock, targetSlot, {dontTransferCache: true}, regenCaller);
+      } else if (blockEpoch > attEpoch) {
+        // should not happen, handled inside attestation verification code
+        throw Error(`Block epoch ${blockEpoch} is after attestation epoch ${attEpoch}`);
+      } else {
+        // should use either current or next shuffling of head state
+        // it's not likely to hit this since these shufflings are cached already
+        // so handle just in case
+        this.metrics?.gossipAttestation.useHeadBlockState.inc({caller: regenCaller});
+        state = await this.regen.getState(attHeadBlock.stateRoot, regenCaller);
+      }
+    } catch (e) {
+      this.shufflingCache.cancelPromise(attEpoch, shufflingDependentRoot, ShufflingPromiseCancelReason.regenError);
+      throw e;
+    }
+
+    const stateDecisionRoot = state.getShufflingDecisionRoot(attEpoch);
+    if (stateDecisionRoot !== shufflingDependentRoot) {
+      // the promise would never be resolved by the state's shuffling
+      this.shufflingCache.cancelPromise(
+        attEpoch,
+        shufflingDependentRoot,
+        ShufflingPromiseCancelReason.decisionRootMismatch
+      );
+      this.logger.debug("Shuffling decision root mismatch on attestation regen", {
+        slot: attHeadBlock.slot,
+        root: attHeadBlock.blockRoot,
+        epoch: attEpoch,
+        shufflingDependentRoot,
+        stateDecisionRoot,
+      });
+
+      throw Error(
+        `Shuffling decision root mismatch epoch=${attEpoch} slot=${attHeadBlock.slot} root=${attHeadBlock.blockRoot} expected=${shufflingDependentRoot} actual=${stateDecisionRoot}`
+      );
     }
+
     // resolve the promise to unblock other calls of the same epoch and dependent root
     this.shufflingCache.processState(state);
     return state.getShufflingAtEpoch(attEpoch);
@@ -1698,7 +1782,7 @@ export class BeaconChain implements IBeaconChain {
     // remove date suffixes in file name, and check duplicate to avoid redundant persistence
     await writeIfNotExist(filepath, bytes);
 
-    this.logger.debug("Persisted invalid ssz object", {id: logStr, filepath});
+    this.logger.debug("Persisted SSZ object", {id: logStr, filepath});
   }
 
   private onScrapeMetrics(metrics: Metrics): void {
@@ -1757,7 +1841,8 @@ export class BeaconChain implements IBeaconChain {
     const metrics = this.metrics;
     if (metrics && (slot + 1) % SLOTS_PER_EPOCH === 0) {
       // On the last slot of the epoch
-      sleep(this.config.SLOT_DURATION_MS / 2)
+      const fork = this.config.getForkName(slot);
+      sleep(this.config.getSlotComponentDurationMs(fork, HALF_SLOT_BPS))
         .then(() => this.validatorMonitor?.onceEveryEndOfEpoch(this.getHeadState()))
         .catch((e) => {
           if (!isErrorAborted(e)) this.logger.error("Error on validator monitor onceEveryEndOfEpoch", {slot}, e);
@@ -1852,6 +1937,21 @@ export class BeaconChain implements IBeaconChain {
     }
   }
 
+  async updateAttachedValidators(epoch: Epoch, validatorIndices: ValidatorIndex[]): Promise<void> {
+    let hasNewValidators = false;
+    for (const validatorIndex of validatorIndices) {
+      if (this.beaconProposerCache.track(epoch, validatorIndex)) {
+        hasNewValidators = true;
+      }
+    }
+
+    // Only update validator custody if we discovered new validators
+    if (hasNewValidators) {
+      const finalizedCheckpoint = this.forkChoice.getFinalizedCheckpoint();
+      await this.updateValidatorsCustodyRequirement(finalizedCheckpoint);
+    }
+  }
+
   private async updateValidatorsCustodyRequirement(finalizedCheckpoint: CheckpointWithHex): Promise<void> {
     if (this.custodyConfig.targetCustodyGroupCount === this.config.NUMBER_OF_CUSTODY_GROUPS) {
       // Custody requirements can only be increased, we can disable dynamic custody updates

package/src/chain/emitter.ts

@@ -1,7 +1,7 @@
 import {EventEmitter} from "node:events";
 import {StrictEventEmitter} from "strict-event-emitter-types";
 import {routes} from "@lodestar/api";
-import {CheckpointWithHex} from "@lodestar/fork-choice";
+import {CheckpointWithHex, PtcQuorumEvent} from "@lodestar/fork-choice";
 import {IBeaconStateView} from "@lodestar/state-transition";
 import {DataColumnSidecar, RootHex, Slot, deneb, phase0} from "@lodestar/types";
 import {PeerIdStr} from "../util/peerId.js";
@@ -36,6 +36,11 @@ export enum ChainEvent {
    * This event is guaranteed to be triggered whenever the fork choice finalized checkpoint is updated. This is either in response to a newly processed block or a new clock tick.
    */
   forkChoiceFinalized = "forkChoice:finalized",
+  /**
+   * This event signals that the PTC verdict on a block was decided: a majority against payload presence or blob data
+   * availability, or a majority for both.
+   */
+  forkChoicePtcQuorum = "forkChoice:ptcQuorum",
   /**
    * This event signals that dependent services (e.g. custody sampling) should update to account for the new target group count.
    */
@@ -112,6 +117,7 @@ export type IChainEvents = ApiEvents & {
 
   [ChainEvent.forkChoiceJustified]: (checkpoint: CheckpointWithHex) => void;
   [ChainEvent.forkChoiceFinalized]: (checkpoint: CheckpointWithHex) => void;
+  [ChainEvent.forkChoicePtcQuorum]: (data: PtcQuorumEvent) => void;
 
   [ChainEvent.updateTargetCustodyGroupCount]: (targetGroupCount: number) => void;
 

package/src/chain/forkChoice/index.ts

@@ -124,6 +124,7 @@ export function initializeForkChoiceFromFinalizedState(
         onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp),
         onFastConfirmation: ({block, slot, currentSlot}) =>
           emitter.emit(routes.events.EventType.fastConfirmation, {block, slot, currentSlot}),
+        onPtcQuorum: (data) => emitter.emit(ChainEvent.forkChoicePtcQuorum, data),
       }
     ),
 
@@ -222,6 +223,7 @@ export function initializeForkChoiceFromUnfinalizedState(
       onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp),
       onFastConfirmation: ({block, slot, currentSlot}) =>
         emitter.emit(routes.events.EventType.fastConfirmation, {block, slot, currentSlot}),
+      onPtcQuorum: (data) => emitter.emit(ChainEvent.forkChoicePtcQuorum, data),
     }
   );
 

package/src/chain/interface.ts

@@ -288,8 +288,11 @@ export interface IBeaconChain {
   waitForBlock(slot: Slot, root: RootHex): Promise<boolean>;
 
   updateBeaconProposerData(epoch: Epoch, proposers: ProposerPreparationData[]): Promise<void>;
+  /** Track validators that use this node for duties, e.g. by subscribing to attestation subnets */
+  updateAttachedValidators(epoch: Epoch, validatorIndices: ValidatorIndex[]): Promise<void>;
 
   persistBlock(data: BeaconBlock | BlindedBeaconBlock, suffix?: string): void;
+  persistExecutionPayloadEnvelope(envelope: gloas.ExecutionPayloadEnvelope): void;
   persistInvalidStateRoot(
     preState: IBeaconStateView,
     postState: IBeaconStateView,
@@ -328,6 +331,7 @@ export type SSZObjectType =
   | "syncCommittee"
   | "contributionAndProof";
 
-export type CommonBlockBody = phase0.BeaconBlockBody &
+export type CommonBlockBody = Omit<phase0.BeaconBlockBody, "eth1Data" | "deposits"> &
+  Partial<Pick<phase0.BeaconBlockBody, "eth1Data" | "deposits">> &
   Pick<capella.BeaconBlockBody, "blsToExecutionChanges"> &
   Pick<altair.BeaconBlockBody, "syncAggregate">;

package/src/chain/opPools/aggregatedAttestationPool.ts

@@ -30,7 +30,7 @@ import {
   isStatePostGloas,
 } from "@lodestar/state-transition";
 import {Attestation, Epoch, RootHex, Slot, electra, isElectraAttestation, phase0, ssz} from "@lodestar/types";
-import {MapDef, assert, toRootHex} from "@lodestar/utils";
+import {Logger, MapDef, assert, toRootHex} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
 import {IntersectResult, intersectUint8Arrays} from "../../util/bitArray.js";
 import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
@@ -58,7 +58,7 @@ export type AttestationsConsolidation = {
 
 /**
  * This function returns not seen participation for a given epoch and slot and committee index.
- * Return null if all validators are seen or no info to check.
+ * Return an empty set if all validators are seen, or null if the committee cannot be computed (e.g. missing shuffling).
  */
 type GetNotSeenValidatorsFn = (epoch: Epoch, slot: Slot, committeeIndex: number) => Set<number> | null;
 
@@ -140,7 +140,8 @@ export class AggregatedAttestationPool {
 
   constructor(
     private readonly config: BeaconConfig,
-    private readonly metrics: Metrics | null = null
+    private readonly metrics: Metrics | null = null,
+    private readonly logger: Logger | null = null
   ) {
     metrics?.opPool.aggregatedAttestationPool.attDataPerSlot.addCollect(() => this.onScrapeMetrics(metrics));
   }
@@ -244,6 +245,7 @@ export class AggregatedAttestationPool {
     const consolidations = new Map<AttestationsConsolidation, number>();
     let scannedSlots = 0;
     let stopReason: ScannedSlotsTerminationReason | null = null;
+    const missingShufflingCommitteesByEpoch = new Map<Epoch, number>();
     slot: for (const slot of slots) {
       const attestationGroupByIndexByDataHash = this.attestationGroupByIndexByDataHexBySlot.get(slot);
       // should not happen
@@ -295,7 +297,12 @@ export class AggregatedAttestationPool {
 
         for (const [committeeIndex, attestationGroup] of attestationGroupByIndex.entries()) {
           const notSeenCommitteeMembers = notSeenValidatorsFn(epoch, slot, committeeIndex);
-          if (notSeenCommitteeMembers === null || notSeenCommitteeMembers.size === 0) {
+          if (notSeenCommitteeMembers === null) {
+            // skip this committee instead of failing the whole block production, other epochs may still have shuffling
+            missingShufflingCommitteesByEpoch.set(epoch, (missingShufflingCommitteesByEpoch.get(epoch) ?? 0) + 1);
+            continue;
+          }
+          if (notSeenCommitteeMembers.size === 0) {
             this.metrics?.opPool.aggregatedAttestationPool.packedAttestations.seenCommittees.inc();
             continue;
           }
@@ -386,6 +393,18 @@ export class AggregatedAttestationPool {
       scannedSlots++;
     }
 
+    for (const [epoch, missingCommittees] of missingShufflingCommitteesByEpoch) {
+      this.metrics?.opPool.aggregatedAttestationPool.packedAttestations.missingShufflingCommittees.inc(
+        missingCommittees
+      );
+      this.logger?.debug("Missing shuffling when packing attestations, skipped committees", {
+        slot: stateSlot,
+        epoch,
+        decisionRoot: state.getShufflingDecisionRoot(epoch),
+        missingCommittees,
+      });
+    }
+
     this.metrics?.opPool.aggregatedAttestationPool.packedAttestations.totalConsolidations.set(consolidations.size);
 
     const sortedConsolidationsByScore = Array.from(consolidations.entries())
@@ -770,13 +789,18 @@ export function getNotSeenValidatorsFn(
     }
     const cacheKey = slot + "_" + committeeIndex;
     let notSeenCommitteeMembers = cachedNotSeenValidators.get(cacheKey);
-    if (notSeenCommitteeMembers != null) {
-      // if all validators are seen then return null, we don't need to check for any attestations of same committee again
-      return notSeenCommitteeMembers.size === 0 ? null : notSeenCommitteeMembers;
+    if (notSeenCommitteeMembers !== undefined) {
+      return notSeenCommitteeMembers;
     }
 
     const decisionRoot = state.getShufflingDecisionRoot(computeEpochAtSlot(slot));
-    const committee = shufflingCache.getBeaconCommittee(epoch, decisionRoot, slot, committeeIndex);
+    const committee = shufflingCache.getBeaconCommitteeOrNull(epoch, decisionRoot, slot, committeeIndex);
+    if (committee === null) {
+      // attestations in the pool were validated against a cached shuffling of the previous or current epoch,
+      // so a miss is a ShufflingCache bug. Return null so the caller tracks it via metrics/logs without failing
+      // block production. Don't fall back to the state's shuffling, that would hide the bug
+      return null;
+    }
     notSeenCommitteeMembers = new Set<number>();
     for (const [i, validatorIndex] of committee.entries()) {
       // no need to check flagIsTimelySource as if validator is not seen, it's participation status is 0
@@ -786,8 +810,8 @@ export function getNotSeenValidatorsFn(
       }
     }
     cachedNotSeenValidators.set(cacheKey, notSeenCommitteeMembers);
-    // if all validators are seen then return null, we don't need to check for any attestations of same committee again
-    return notSeenCommitteeMembers.size === 0 ? null : notSeenCommitteeMembers;
+    // an empty set means all validators are seen, we don't need to check for any attestations of same committee again
+    return notSeenCommitteeMembers;
   };
 }
 

package/src/chain/opPools/opPool.ts

@@ -1,5 +1,4 @@
 import {BeaconConfig} from "@lodestar/config";
-import {DbBatch, Id, Repository} from "@lodestar/db";
 import {
   BLS_WITHDRAWAL_PREFIX,
   ForkName,
@@ -42,6 +41,7 @@ import {
   getAttesterSlashingSignatureDomains,
   getProposerSlashingSignatureDomain,
   isValidBlsToExecutionChangeForBlockInclusion,
+  persistDiff,
 } from "./utils.js";
 
 type HexRoot = string;
@@ -500,33 +500,3 @@ function isVoluntaryExitSignatureIncludable(stateFork: ForkSeq, voluntaryExitFor
 function isSlashableAtEpoch(validator: phase0.Validator, epoch: Epoch): boolean {
   return !validator.slashed && validator.activationEpoch <= epoch && epoch < validator.withdrawableEpoch;
 }
-
-/**
- * Persist target items `items` in `dbRepo` doing minimum put and delete writes.
- * Reads all keys in repository to compute the diff between current persisted data and target data.
- */
-async function persistDiff<K extends Id, V>(
-  dbRepo: Repository<K, V>,
-  items: {key: K; value: V}[],
-  serializeKey: (key: K) => number | string,
-  opts: {updateExisting?: boolean} = {}
-): Promise<void> {
-  const persistedKeys = await dbRepo.keys();
-  const batch: DbBatch<K, V> = [];
-
-  const persistedKeysSerialized = new Set(persistedKeys.map(serializeKey));
-  for (const item of items) {
-    if (opts.updateExisting || !persistedKeysSerialized.has(serializeKey(item.key))) {
-      batch.push({type: "put", key: item.key, value: item.value});
-    }
-  }
-
-  const targetKeysSerialized = new Set(items.map((item) => serializeKey(item.key)));
-  for (const persistedKey of persistedKeys) {
-    if (!targetKeysSerialized.has(serializeKey(persistedKey))) {
-      batch.push({type: "del", key: persistedKey});
-    }
-  }
-
-  if (batch.length > 0) await dbRepo.batch(batch);
-}