@lodestar/beacon-node 1.50.0-dev.8ba9d1dd30 → 1.50.0-dev.9c7f3c203f

package/src/chain/emitter.ts

@@ -1,7 +1,7 @@
 import {EventEmitter} from "node:events";
 import {StrictEventEmitter} from "strict-event-emitter-types";
 import {routes} from "@lodestar/api";
-import {CheckpointWithHex} from "@lodestar/fork-choice";
+import {CheckpointWithHex, PtcQuorumEvent} from "@lodestar/fork-choice";
 import {IBeaconStateView} from "@lodestar/state-transition";
 import {DataColumnSidecar, RootHex, Slot, deneb, phase0} from "@lodestar/types";
 import {PeerIdStr} from "../util/peerId.js";
@@ -36,6 +36,11 @@ export enum ChainEvent {
    * This event is guaranteed to be triggered whenever the fork choice finalized checkpoint is updated. This is either in response to a newly processed block or a new clock tick.
    */
   forkChoiceFinalized = "forkChoice:finalized",
+  /**
+   * This event signals that the PTC verdict on a block was decided: a majority against payload presence or blob data
+   * availability, or a majority for both.
+   */
+  forkChoicePtcQuorum = "forkChoice:ptcQuorum",
   /**
    * This event signals that dependent services (e.g. custody sampling) should update to account for the new target group count.
    */
@@ -112,6 +117,7 @@ export type IChainEvents = ApiEvents & {
 
   [ChainEvent.forkChoiceJustified]: (checkpoint: CheckpointWithHex) => void;
   [ChainEvent.forkChoiceFinalized]: (checkpoint: CheckpointWithHex) => void;
+  [ChainEvent.forkChoicePtcQuorum]: (data: PtcQuorumEvent) => void;
 
   [ChainEvent.updateTargetCustodyGroupCount]: (targetGroupCount: number) => void;
 

package/src/chain/errors/envelopeReconstructionError.ts

@@ -4,6 +4,8 @@ import {LodestarError} from "@lodestar/utils";
 export enum EnvelopeReconstructionErrorCode {
   /** The EL failed to answer engine_getPayloadBodiesByHashV2 (transport / EL down). Transient. */
   ENGINE_UNAVAILABLE = "ENVELOPE_RECONSTRUCTION_ERROR_ENGINE_UNAVAILABLE",
+  /** The EL does not have the block or its block access list (pruned, or not yet re-indexed after a restart). May recover. */
+  BODY_UNAVAILABLE = "ENVELOPE_RECONSTRUCTION_ERROR_BODY_UNAVAILABLE",
   /** A body returned by the EL does not hash to the root stored in the header envelope. Local DB/EL inconsistency. */
   BODY_ROOT_MISMATCH = "ENVELOPE_RECONSTRUCTION_ERROR_BODY_ROOT_MISMATCH",
   /** A by-range stream stopped short at a slot that cannot be served; the response so far is still valid */
@@ -12,6 +14,7 @@ export enum EnvelopeReconstructionErrorCode {
 
 export type EnvelopeReconstructionErrorType =
   | {code: EnvelopeReconstructionErrorCode.ENGINE_UNAVAILABLE}
+  | {code: EnvelopeReconstructionErrorCode.BODY_UNAVAILABLE; slot: Slot}
   | {
       code: EnvelopeReconstructionErrorCode.BODY_ROOT_MISMATCH;
       slot: Slot;

package/src/chain/errors/gossipValidation.ts

@@ -9,6 +9,7 @@ export enum GossipErrorCode {
   INVALID_SERIALIZED_BYTES_ERROR_CODE = "GOSSIP_ERROR_INVALID_SERIALIZED_BYTES",
   PAST_SLOT = "GOSSIP_ERROR_PAST_SLOT",
   FUTURE_SLOT = "GOSSIP_ERROR_FUTURE_SLOT",
+  FATAL_PEER = "GOSSIP_ERROR_FATAL_PEER",
   INVALID_SLOT = "GOSSIP_ERROR_INVALID_SLOT",
 }
 

package/src/chain/forkChoice/index.ts

@@ -124,6 +124,7 @@ export function initializeForkChoiceFromFinalizedState(
         onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp),
         onFastConfirmation: ({block, slot, currentSlot}) =>
           emitter.emit(routes.events.EventType.fastConfirmation, {block, slot, currentSlot}),
+        onPtcQuorum: (data) => emitter.emit(ChainEvent.forkChoicePtcQuorum, data),
       }
     ),
 
@@ -222,6 +223,7 @@ export function initializeForkChoiceFromUnfinalizedState(
       onFinalized: (cp) => emitter.emit(ChainEvent.forkChoiceFinalized, cp),
       onFastConfirmation: ({block, slot, currentSlot}) =>
         emitter.emit(routes.events.EventType.fastConfirmation, {block, slot, currentSlot}),
+      onPtcQuorum: (data) => emitter.emit(ChainEvent.forkChoicePtcQuorum, data),
     }
   );
 

package/src/chain/interface.ts

@@ -30,7 +30,7 @@ import {Metrics} from "../metrics/metrics.js";
 import {BufferPool} from "../util/bufferPool.js";
 import {IClock} from "../util/clock.js";
 import {CustodyConfig} from "../util/dataColumns.js";
-import {ReconstructMismatchPolicy} from "../util/execution.js";
+import {ReconstructMissPolicy} from "../util/execution.js";
 import {SerializedCache} from "../util/serializedCache.js";
 import {IArchiveStore} from "./archiveStore/interface.js";
 import {CheckpointBalancesCache} from "./balancesCache.js";
@@ -239,7 +239,7 @@ export interface IBeaconChain {
   getSerializedExecutionPayloadEnvelope(blockSlot: Slot, blockRootHex: string): Promise<Uint8Array | null>;
   getSerializedExecutionPayloadEnvelopes(
     requests: {blockSlot: Slot; blockRootHex: RootHex}[],
-    onMismatch?: ReconstructMismatchPolicy
+    onMiss?: ReconstructMissPolicy
   ): Promise<(Uint8Array | null)[]>;
   getExecutionPayloadEnvelope(
     blockSlot: Slot,
@@ -290,6 +290,7 @@ export interface IBeaconChain {
   updateBeaconProposerData(epoch: Epoch, proposers: ProposerPreparationData[]): Promise<void>;
 
   persistBlock(data: BeaconBlock | BlindedBeaconBlock, suffix?: string): void;
+  persistExecutionPayloadEnvelope(envelope: gloas.ExecutionPayloadEnvelope): void;
   persistInvalidStateRoot(
     preState: IBeaconStateView,
     postState: IBeaconStateView,

package/src/chain/opPools/proposerPreferencesPool.ts

@@ -6,8 +6,8 @@ import {toRootHex} from "@lodestar/utils";
  *
  * The primary consumer is `validateExecutionPayloadBid`, which looks up the matching
  * preferences via `get(bid.slot, dependent_root)` to enforce the IGNORE-existence and
- * REJECT-equality rules from the gloas spec. The beacon API `/pool/proposer_preferences`
- * GET endpoint reads from the same pool via `getAll`.
+ * REJECT-equality rules from the gloas spec. The beacon API `getProposerPreferences`
+ * endpoint reads from the same pool via `getAll`.
  *
  * `validator_index` is intentionally not part of the key: gossip validation enforces
  * `proposers[proposalSlot % SLOTS_PER_EPOCH] === validatorIndex` against the shuffling
@@ -37,15 +37,19 @@ export class ProposerPreferencesPool {
     byRoot.set(rootHex, signed);
   }
 
-  /** API read-out: flatten across branches, optionally filtered by slot. */
-  getAll(slot?: Slot): gloas.SignedProposerPreferences[] {
-    if (slot !== undefined) {
-      const byRoot = this.bySlot.get(slot);
-      return byRoot ? Array.from(byRoot.values()) : [];
-    }
+  /** API read-out: flatten across branches, optionally filtered by slot and dependent root. */
+  getAll(slot?: Slot, dependentRootHex?: RootHex): gloas.SignedProposerPreferences[] {
     const out: gloas.SignedProposerPreferences[] = [];
-    for (const byRoot of this.bySlot.values()) {
-      for (const v of byRoot.values()) out.push(v);
+    for (const [s, byRoot] of this.bySlot) {
+      if (slot !== undefined && s !== slot) {
+        continue;
+      }
+      for (const [root, signed] of byRoot) {
+        if (dependentRootHex !== undefined && root !== dependentRootHex) {
+          continue;
+        }
+        out.push(signed);
+      }
     }
     return out;
   }

package/src/chain/options.ts

@@ -27,8 +27,8 @@ export type IChainOptions = BlockProcessOpts &
     blsVerifyAllMainThread?: boolean;
     blsVerifyAllMultiThread?: boolean;
     blacklistedBlocks?: string[];
-    // TODO GLOAS: add similar option for execution payload envelopes?
     persistProducedBlocks?: boolean;
+    persistProducedPayloadEnvelopes?: boolean;
     /** Archive finalized Gloas payload envelopes as headers (bodies reconstructed from the EL on read) */
     dedupePayloads?: boolean;
     persistInvalidSszObjects?: boolean;
@@ -52,7 +52,7 @@ export type IChainOptions = BlockProcessOpts &
     minSameMessageSignatureSetsToBatch: number;
     archiveDateEpochs?: number;
     nHistoricalStatesFileDataStore?: boolean;
-    nativeStateView?: boolean;
+    nativeStateTransition?: boolean;
     /** Builder circuit breaker fault inspection window in slots */
     faultInspectionWindow?: number;
     /** Canonical EMPTY blocks allowed per `faultInspectionWindow` observed blocks */
@@ -131,7 +131,7 @@ export const defaultChainOptions: IChainOptions = {
   //   - users can prune the persisted checkpoint state files manually to save disc space
   //   - it helps debug easier when network is unfinalized
   nHistoricalStatesFileDataStore: true,
-  nativeStateView: false,
+  nativeStateTransition: false,
   maxBlockStates: DEFAULT_MAX_BLOCK_STATES,
   maxCPStateEpochsInMemory: DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY,
   maxCPStateEpochsOnDisk: DEFAULT_MAX_CP_STATE_ON_DISK,

package/src/chain/produceBlock/computeNewStateRoot.ts

@@ -16,7 +16,7 @@ export function computeNewStateRoot(
   const signedBlock = {message: block, signature: EMPTY_SIGNATURE};
   const {newStateRoot, proposerReward, postState, hashTreeRootTime} = state.computeNewStateRoot(
     {block: signedBlock},
-    {metrics}
+    {metrics: metrics?.stateTransition}
   );
 
   metrics?.stateHashTreeRootTime.observe({source: StateHashTreeRootSource.computeNewStateRoot}, hashTreeRootTime);

package/src/chain/regen/regen.ts

@@ -14,6 +14,7 @@ import {Logger, fromHex, toRootHex} from "@lodestar/utils";
 import {IBeaconDb} from "../../db/index.js";
 import {Metrics} from "../../metrics/index.js";
 import {nextEventLoop} from "../../util/eventLoop.js";
+import {SerializedCache} from "../../util/serializedCache.js";
 import {getCheckpointFromState} from "../blocks/utils/checkpoint.js";
 import {ChainEvent, ChainEventEmitter} from "../emitter.js";
 import {SeenBlockInput} from "../seenCache/seenGossipBlockInput.js";
@@ -28,6 +29,7 @@ export type RegenModules = {
   blockStateCache: BlockStateCache;
   checkpointStateCache: CheckpointStateCache;
   seenBlockInputCache: SeenBlockInput;
+  serializedCache: SerializedCache;
   config: ChainForkConfig;
   emitter: ChainEventEmitter;
   logger: Logger;
@@ -235,7 +237,7 @@ export class StateRegenerator implements IStateRegeneratorInternal {
         // We are only running the state transition to get a specific state's data.
         // stateTransition() does the clone() inside, transfer cache to make the regen faster
         state = state.stateTransition(
-          block,
+          {block, ssz: this.modules.serializedCache.get(block)},
           {
             // Replay previously imported blocks, assume valid and available
             executionPayloadStatus: ExecutionPayloadStatus.valid,
@@ -245,7 +247,7 @@ export class StateRegenerator implements IStateRegeneratorInternal {
             verifySignatures: false,
             dontTransferCache: false,
           },
-          this.modules
+          {metrics: this.modules.metrics?.stateTransition, validatorMonitor: this.modules.validatorMonitor}
         );
 
         const hashTreeRootTimer = this.modules.metrics?.stateHashTreeRootTime.startTimer({
@@ -316,7 +318,10 @@ async function processSlotsByCheckpoint(
 ): Promise<IBeaconStateView> {
   let postState = await processSlotsToNearestCheckpoint(modules, preState, slot, regenCaller, opts);
   if (postState.slot < slot) {
-    postState = postState.processSlots(slot, opts, modules);
+    postState = postState.processSlots(slot, opts, {
+      metrics: modules.metrics?.stateTransition,
+      validatorMonitor: modules.validatorMonitor,
+    });
   }
   return postState;
 }
@@ -361,7 +366,10 @@ export async function processSlotsToNearestCheckpoint(
       caller: regenCaller,
     });
     // processSlots calls .clone() before mutating
-    postState = postState.processSlots(nextEpochSlot, opts, modules);
+    postState = postState.processSlots(nextEpochSlot, opts, {
+      metrics: metrics?.stateTransition,
+      validatorMonitor: modules.validatorMonitor,
+    });
     metrics?.epochTransitionByCaller.inc({caller: regenCaller});
 
     // this is usually added when we prepare for next slot or validate gossip block

package/src/chain/seenCache/seenPayloadEnvelopeInput.ts

@@ -47,8 +47,9 @@ export type SeenPayloadEnvelopeInputModules = {
  * Created whenever we have a block because it needs block bid.
  * Steady state (linear chain, healthy progression): the cache holds ~2 entries — the head
  * (parent for next-slot production) and its parent (proposer-boost-reorg fallback). It can
- * transiently hold more during forks, range-sync bursts, or when `prepareNextSlot` skips
- * ticks; subsequent ticks settle it back.
+ * hold more during forks and range sync, or while an entry's payload is not FULL on the head's
+ * branch or its columns are still being gathered. Such entries are evicted on a later block
+ * import, on finalization or by the size cap.
  */
 export class SeenPayloadEnvelopeInput {
   private readonly config: ChainForkConfig;
@@ -264,18 +265,35 @@ export class SeenPayloadEnvelopeInput {
     }
   }
 
+  /**
+   * Removes a PayloadEnvelopeInput whose envelope failed import with a definitive invalid verdict. The envelope
+   * cannot be replaced in place, sync recreates the entry via `getOrReload()` or `add()` to fetch another one.
+   * No-op if the cached entry is a different object, it may already be the recreated one.
+   */
+  removeInvalid(payloadInput: PayloadEnvelopeInput): void {
+    if (this.payloadInputs.get(payloadInput.blockRootHex) === payloadInput) {
+      this.evictPayloadInput(payloadInput, "invalid");
+    }
+  }
+
   pruneBelowParent(parentBlock: ProtoBlock): void {
-    for (const block of this.forkChoice.getAllAncestorBlocks(parentBlock.blockRoot, parentBlock.payloadStatus)) {
-      // Only evict once the payload is FULL (revealed/imported) — on an EMPTY/PENDING branch we may
-      // still need to download the FULL envelope (see #9475), and evicting would make payload-by-root
-      // sync throw "Missing PayloadEnvelopeInput for known block".
-      if (block.slot < parentBlock.slot && block.payloadStatus === PayloadStatus.FULL) {
-        const input = this.payloadInputs.get(block.blockRoot);
-        // ...and don't evict while columns are still being gathered: writeDataColumnsToDb awaits the
-        // same hasComputedAllData() before persisting. Such entries are pruned by a later call.
-        if (input?.hasComputedAllData()) {
-          this.evictPayloadInput(input, "belowParent");
-        }
+    for (const input of this.payloadInputs.values()) {
+      if (
+        input.slot < parentBlock.slot &&
+        // Don't evict while columns are still being gathered: writeDataColumnsToDb awaits the same
+        // hasComputedAllData() before persisting. Such entries are pruned by a later call.
+        input.hasComputedAllData() &&
+        // Only evict once the payload is FULL (revealed/imported) on the parent's branch. On an EMPTY/PENDING
+        // branch we may still need to download the FULL envelope (see #9475), and without the entry gossip
+        // for the block is ignored until by-root sync reloads it.
+        this.forkChoice.isDescendant(
+          input.blockRootHex,
+          PayloadStatus.FULL,
+          parentBlock.blockRoot,
+          parentBlock.payloadStatus
+        )
+      ) {
+        this.evictPayloadInput(input, "belowParent");
       }
     }
   }

package/src/chain/validatorMonitor.ts

@@ -123,6 +123,11 @@ export type ValidatorMonitorOpts = {
   validatorMonitorLogs?: boolean;
 };
 
+export type NativeValidatorMonitor = {
+  registerLocalValidator(index: ValidatorIndex): void;
+  unregisterLocalValidator(index: ValidatorIndex): void;
+};
+
 export const defaultValidatorMonitorOpts: ValidatorMonitorOpts = {
   validatorMonitorLogs: false,
 };
@@ -296,7 +301,8 @@ export function createValidatorMonitor(
   config: ChainForkConfig,
   genesisTime: number,
   logger: Logger,
-  opts: ValidatorMonitorOpts
+  opts: ValidatorMonitorOpts,
+  nativeValidatorMonitor: NativeValidatorMonitor | null = null
 ): ValidatorMonitor {
   const logLevel = opts.validatorMonitorLogs ? LogLevel.info : LogLevel.debug;
   const log: LogHandler = (message: string, context?: LogData) => {
@@ -327,18 +333,23 @@ export function createValidatorMonitor(
   }));
 
   let lastRegisteredStatusEpoch = -1;
+  let lastAttestationSummaryEpoch = -1;
 
   // Track validator additions/removals per epoch for logging
   const addedValidatorsInEpoch: Set<ValidatorIndex> = new Set();
   const removedValidatorsInEpoch: Set<ValidatorIndex> = new Set();
 
-  const validatorMonitorMetrics = metricsRegister ? createValidatorMonitorMetrics(metricsRegister) : null;
+  const shouldRegisterStateDerivedMetrics = nativeValidatorMonitor === null;
+  const validatorMonitorMetrics = metricsRegister
+    ? createValidatorMonitorMetrics(metricsRegister, shouldRegisterStateDerivedMetrics)
+    : null;
 
   const validatorMonitor: ValidatorMonitor = {
     registerLocalValidator(index) {
       const isNewValidator = !validators.has(index);
       validators.getOrDefault(index).lastRegisteredTimeMs = Date.now();
       if (isNewValidator) {
+        nativeValidatorMonitor?.registerLocalValidator(index);
         addedValidatorsInEpoch.add(index);
       }
     },
@@ -366,7 +377,7 @@ export function createValidatorMonitor(
       // Track total balance instead of per-validator balance to reduce metric cardinality
       let totalBalance = 0;
 
-      for (const [index, monitoredValidator] of validators.entries()) {
+      for (const index of validators.keys()) {
         // We subtract two from the state of the epoch that generated these summaries.
         //
         // - One to account for it being the previous epoch.
@@ -380,68 +391,29 @@ export function createValidatorMonitor(
         );
 
         if (summary.isPrevSourceAttester) {
-          validatorMonitorMetrics?.prevEpochOnChainSourceAttesterHit.inc();
+          validatorMonitorMetrics?.prevEpochOnChainSourceAttesterHit?.inc();
         } else {
-          validatorMonitorMetrics?.prevEpochOnChainSourceAttesterMiss.inc();
+          validatorMonitorMetrics?.prevEpochOnChainSourceAttesterMiss?.inc();
         }
         if (summary.isPrevHeadAttester) {
-          validatorMonitorMetrics?.prevEpochOnChainHeadAttesterHit.inc();
+          validatorMonitorMetrics?.prevEpochOnChainHeadAttesterHit?.inc();
         } else {
-          validatorMonitorMetrics?.prevEpochOnChainHeadAttesterMiss.inc();
+          validatorMonitorMetrics?.prevEpochOnChainHeadAttesterMiss?.inc();
         }
         if (summary.isPrevTargetAttester) {
-          validatorMonitorMetrics?.prevEpochOnChainTargetAttesterHit.inc();
+          validatorMonitorMetrics?.prevEpochOnChainTargetAttesterHit?.inc();
         } else {
-          validatorMonitorMetrics?.prevEpochOnChainTargetAttesterMiss.inc();
-        }
-
-        const prevEpochSummary = monitoredValidator.summaries.get(previousEpoch);
-        const attestationCorrectHead = prevEpochSummary?.attestationCorrectHead;
-        if (attestationCorrectHead !== null && attestationCorrectHead !== undefined) {
-          if (attestationCorrectHead) {
-            validatorMonitorMetrics?.prevOnChainAttesterCorrectHead.inc();
-          } else {
-            validatorMonitorMetrics?.prevOnChainAttesterIncorrectHead.inc();
-          }
-        }
-
-        const attestationMinBlockInclusionDistance = prevEpochSummary?.attestationMinBlockInclusionDistance;
-        const inclusionDistance =
-          attestationMinBlockInclusionDistance != null && attestationMinBlockInclusionDistance > 0
-            ? // altair, attestation is not missed
-              attestationMinBlockInclusionDistance
-            : summary.inclusionDistance
-              ? // phase0, this is from the state transition
-                summary.inclusionDistance
-              : null;
-
-        if (inclusionDistance !== null) {
-          validatorMonitorMetrics?.prevEpochOnChainInclusionDistance.observe(inclusionDistance);
-          validatorMonitorMetrics?.prevEpochOnChainAttesterHit.inc();
-        } else {
-          validatorMonitorMetrics?.prevEpochOnChainAttesterMiss.inc();
+          validatorMonitorMetrics?.prevEpochOnChainTargetAttesterMiss?.inc();
         }
 
         const balance = balances?.[index];
         if (balance !== undefined) {
           totalBalance += balance;
         }
-
-        if (!summary.isPrevSourceAttester || !summary.isPrevTargetAttester || !summary.isPrevHeadAttester) {
-          log("Failed attestation in previous epoch", {
-            validator: index,
-            prevEpoch: currentEpoch - 1,
-            isPrevSourceAttester: summary.isPrevSourceAttester,
-            isPrevHeadAttester: summary.isPrevHeadAttester,
-            isPrevTargetAttester: summary.isPrevTargetAttester,
-            // inclusionDistance is not available in summary since altair
-            inclusionDistance,
-          });
-        }
       }
 
       if (balances !== undefined) {
-        validatorMonitorMetrics?.prevEpochOnChainBalance.set(totalBalance);
+        validatorMonitorMetrics?.prevEpochOnChainBalance?.set(totalBalance);
       }
     },
 
@@ -732,6 +704,7 @@ export function createValidatorMonitor(
       // Prune validators not seen in a while
       for (const [index, validator] of validators.entries()) {
         if (Date.now() - validator.lastRegisteredTimeMs > retainRegisteredValidatorsMs) {
+          nativeValidatorMonitor?.unregisterLocalValidator(index);
           validators.delete(index);
           removedValidatorsInEpoch.add(index);
         }
@@ -767,6 +740,38 @@ export function createValidatorMonitor(
         return;
       }
 
+      // On-chain attestation summary of prevEpoch. Reads only the epoch summaries filled by
+      // registerAttestationInBlock(), so it works with both TS and native state transition.
+      if (prevEpoch > lastAttestationSummaryEpoch) {
+        lastAttestationSummaryEpoch = prevEpoch;
+
+        for (const monitoredValidator of validators.values()) {
+          const prevEpochSummary = monitoredValidator.summaries.get(prevEpoch);
+          const attestationCorrectHead = prevEpochSummary?.attestationCorrectHead;
+          if (attestationCorrectHead !== null && attestationCorrectHead !== undefined) {
+            if (attestationCorrectHead) {
+              validatorMonitorMetrics?.prevOnChainAttesterCorrectHead.inc();
+            } else {
+              validatorMonitorMetrics?.prevOnChainAttesterIncorrectHead.inc();
+            }
+          }
+
+          const attestationMinBlockInclusionDistance = prevEpochSummary?.attestationMinBlockInclusionDistance;
+          const inclusionDistance =
+            attestationMinBlockInclusionDistance != null && attestationMinBlockInclusionDistance > 0
+              ? // altair, attestation is not missed
+                attestationMinBlockInclusionDistance
+              : null;
+
+          if (inclusionDistance !== null) {
+            validatorMonitorMetrics?.prevEpochOnChainInclusionDistance.observe(inclusionDistance);
+            validatorMonitorMetrics?.prevEpochOnChainAttesterHit.inc();
+          } else {
+            validatorMonitorMetrics?.prevEpochOnChainAttesterMiss.inc();
+          }
+        }
+      }
+
       const rootCache = new RootHexCache(headState);
 
       if (isStatePostAltair(headState)) {
@@ -784,6 +789,17 @@ export function createValidatorMonitor(
             epoch: prevEpoch,
             summary,
           });
+
+          if (!flags.timelySource || !flags.timelyTarget || !flags.timelyHead) {
+            log("Failed attestation in previous epoch", {
+              validator: index,
+              prevEpoch,
+              isPrevSourceAttester: flags.timelySource,
+              isPrevHeadAttester: flags.timelyHead,
+              isPrevTargetAttester: flags.timelyTarget,
+              inclusionDistance: validator.summaries.get(prevEpoch)?.attestationMinBlockInclusionDistance ?? null,
+            });
+          }
         }
       }
 
@@ -1198,7 +1214,9 @@ export class RootHexCache {
   }
 }
 
-function createValidatorMonitorMetrics(register: RegistryMetricCreator) {
+function createValidatorMonitorMetrics(register: RegistryMetricCreator, registerStateDerivedMetrics = true) {
+  const stateDerivedRegister = registerStateDerivedMetrics ? register : null;
+
   return {
     validatorsConnected: register.gauge({
       name: "validator_monitor_validators",
@@ -1217,7 +1235,7 @@ function createValidatorMonitorMetrics(register: RegistryMetricCreator) {
     }),
 
     // Validator Monitor Metrics (per-epoch summaries)
-    prevEpochOnChainBalance: register.gauge({
+    prevEpochOnChainBalance: stateDerivedRegister?.gauge({
       name: "validator_monitor_prev_epoch_on_chain_balance",
       help: "Total balance of all monitored validators after an epoch",
     }),
@@ -1229,19 +1247,19 @@ function createValidatorMonitorMetrics(register: RegistryMetricCreator) {
       name: "validator_monitor_prev_epoch_on_chain_attester_miss_total",
       help: "Incremented if validator's submitted attestation is not included in any blocks",
     }),
-    prevEpochOnChainSourceAttesterHit: register.gauge({
+    prevEpochOnChainSourceAttesterHit: stateDerivedRegister?.gauge({
       name: "validator_monitor_prev_epoch_on_chain_source_attester_hit_total",
       help: "Incremented if the validator is flagged as a previous epoch source attester during per epoch processing",
     }),
-    prevEpochOnChainSourceAttesterMiss: register.gauge({
+    prevEpochOnChainSourceAttesterMiss: stateDerivedRegister?.gauge({
       name: "validator_monitor_prev_epoch_on_chain_source_attester_miss_total",
       help: "Incremented if the validator is not flagged as a previous epoch source attester during per epoch processing",
     }),
-    prevEpochOnChainHeadAttesterHit: register.gauge({
+    prevEpochOnChainHeadAttesterHit: stateDerivedRegister?.gauge({
       name: "validator_monitor_prev_epoch_on_chain_head_attester_hit_total",
       help: "Incremented if the validator is flagged as a previous epoch head attester during per epoch processing",
     }),
-    prevEpochOnChainHeadAttesterMiss: register.gauge({
+    prevEpochOnChainHeadAttesterMiss: stateDerivedRegister?.gauge({
       name: "validator_monitor_prev_epoch_on_chain_head_attester_miss_total",
       help: "Incremented if the validator is not flagged as a previous epoch head attester during per epoch processing",
     }),
@@ -1253,11 +1271,11 @@ function createValidatorMonitorMetrics(register: RegistryMetricCreator) {
       name: "validator_monitor_prev_epoch_on_chain_attester_incorrect_head_total",
       help: "Total count of times a validator votes incorrect head",
     }),
-    prevEpochOnChainTargetAttesterHit: register.gauge({
+    prevEpochOnChainTargetAttesterHit: stateDerivedRegister?.gauge({
       name: "validator_monitor_prev_epoch_on_chain_target_attester_hit_total",
       help: "Incremented if the validator is flagged as a previous epoch target attester during per epoch processing",
     }),
-    prevEpochOnChainTargetAttesterMiss: register.gauge({
+    prevEpochOnChainTargetAttesterMiss: stateDerivedRegister?.gauge({
       name: "validator_monitor_prev_epoch_on_chain_target_attester_miss_total",
       help: "Incremented if the validator is not flagged as a previous epoch target attester during per epoch processing",
     }),

package/src/execution/engine/disabled.ts

@@ -21,18 +21,10 @@ export class ExecutionEngineDisabled implements IExecutionEngine {
     throw Error("Execution engine disabled");
   }
 
-  getPayloadBodiesByHash(): Promise<never> {
-    throw Error("Execution engine disabled");
-  }
-
   getPayloadBodiesByHashV2(): Promise<never> {
     throw Error("Execution engine disabled");
   }
 
-  getPayloadBodiesByRange(): Promise<never> {
-    throw Error("Execution engine disabled");
-  }
-
   getBlobs(): Promise<never> {
     throw Error("Execution engine disabled");
   }

package/src/execution/engine/http.ts

@@ -31,13 +31,11 @@ import {
   BLOB_AND_PROOF_V2_RPC_BYTES,
   EngineApiRpcParamTypes,
   EngineApiRpcReturnTypes,
-  ExecutionPayloadBody,
   ExecutionPayloadBodyV2,
   assertReqSizeLimit,
   deserializeBlobAndProofs,
   deserializeBlobAndProofsV2,
   deserializeBlobAndProofsV2IntoBytes,
-  deserializeExecutionPayloadBody,
   deserializeExecutionPayloadBodyV2,
   parseExecutionPayload,
   serializeBeaconBlockRoot,
@@ -46,7 +44,7 @@ import {
   serializePayloadAttributes,
   serializeVersionedHashes,
 } from "./types.js";
-import {bytesToData, getExecutionEngineState, numToQuantity} from "./utils.js";
+import {bytesToData, getExecutionEngineState} from "./utils.js";
 
 export type ExecutionEngineModules = {
   signal: AbortSignal;
@@ -114,7 +112,6 @@ const notifyNewPayloadOpts: ReqOpts = {routeId: "notifyNewPayload"};
 const forkchoiceUpdatedV1Opts: ReqOpts = {routeId: "forkchoiceUpdated"};
 const getPayloadOpts: ReqOpts = {routeId: "getPayload"};
 const getPayloadBodiesByHashOpts: ReqOpts = {routeId: "getPayloadBodiesByHash"};
-const getPayloadBodiesByRangeOpts: ReqOpts = {routeId: "getPayloadBodiesByRange"};
 const getBlobsV1Opts: ReqOpts = {routeId: "getBlobsV1"};
 const getBlobsV2Opts: ReqOpts = {routeId: "getBlobsV2"};
 const getClientVersionOpts: ReqOpts = {routeId: "getClientVersion"};
@@ -468,16 +465,6 @@ export class ExecutionEngineHttp implements IExecutionEngine {
     this.payloadIdCache.prune();
   }
 
-  async getPayloadBodiesByHash(_fork: ForkName, blockHashes: RootHex[]): Promise<(ExecutionPayloadBody | null)[]> {
-    const method = "engine_getPayloadBodiesByHashV1";
-    assertReqSizeLimit(blockHashes.length, 32);
-    const response = await this.rpc.fetchWithRetries<
-      EngineApiRpcReturnTypes[typeof method],
-      EngineApiRpcParamTypes[typeof method]
-    >({method, params: [blockHashes]}, getPayloadBodiesByHashOpts);
-    return response.map(deserializeExecutionPayloadBody);
-  }
-
   async getPayloadBodiesByHashV2(blockHashes: RootHex[]): Promise<(ExecutionPayloadBodyV2 | null)[]> {
     const method = "engine_getPayloadBodiesByHashV2";
     assertReqSizeLimit(blockHashes.length, 32);
@@ -488,22 +475,6 @@ export class ExecutionEngineHttp implements IExecutionEngine {
     return response.map(deserializeExecutionPayloadBodyV2);
   }
 
-  async getPayloadBodiesByRange(
-    _fork: ForkName,
-    startBlockNumber: number,
-    blockCount: number
-  ): Promise<(ExecutionPayloadBody | null)[]> {
-    const method = "engine_getPayloadBodiesByRangeV1";
-    assertReqSizeLimit(blockCount, 32);
-    const start = numToQuantity(startBlockNumber);
-    const count = numToQuantity(blockCount);
-    const response = await this.rpc.fetchWithRetries<
-      EngineApiRpcReturnTypes[typeof method],
-      EngineApiRpcParamTypes[typeof method]
-    >({method, params: [start, count]}, getPayloadBodiesByRangeOpts);
-    return response.map(deserializeExecutionPayloadBody);
-  }
-
   async getBlobs(
     fork: ForkPostFulu,
     versionedHashes: VersionedHashes,

package/src/execution/engine/interface.ts

@@ -13,7 +13,7 @@ import {BlobsBundle, ExecutionPayload, ExecutionRequests, Root, RootHex, Wei, ca
 import {BlobAndProof} from "@lodestar/types/deneb";
 import {BlobAndProofV2} from "@lodestar/types/fulu";
 import {PayloadId, PayloadIdCache, WithdrawalV1} from "./payloadIdCache.js";
-import {ExecutionPayloadBody, ExecutionPayloadBodyV2} from "./types.js";
+import {ExecutionPayloadBodyV2} from "./types.js";
 import {DATA} from "./utils.js";
 
 export {PayloadIdCache, type PayloadId, type WithdrawalV1};
@@ -170,13 +170,9 @@ export interface IExecutionEngine {
     shouldOverrideBuilder?: boolean;
   }>;
 
-  getPayloadBodiesByHash(fork: ForkName, blockHash: DATA[]): Promise<(ExecutionPayloadBody | null)[]>;
-
-  /** Amsterdam: like V1 but also returns the block access list (null if the EL no longer has it) */
+  /** Amsterdam: payload bodies including the block access list (null if the EL no longer has it) */
   getPayloadBodiesByHashV2(blockHash: DATA[]): Promise<(ExecutionPayloadBodyV2 | null)[]>;
 
-  getPayloadBodiesByRange(fork: ForkName, start: number, count: number): Promise<(ExecutionPayloadBody | null)[]>;
-
   getBlobs(
     fork: ForkPostFulu,
     versionedHashes: VersionedHashes,