@lodestar/beacon-node 1.41.0 → 1.42.0-dev.2fd27cd2f6

package/src/chain/blocks/importBlock.ts

@@ -7,20 +7,35 @@ import {
   ForkChoiceErrorCode,
   NotReorgedReason,
   getSafeExecutionBlockHash,
+  isGloasBlock,
 } from "@lodestar/fork-choice";
-import {ForkPostAltair, ForkPostElectra, ForkSeq, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {
-  CachedBeaconStateAltair,
-  EpochCache,
+  ForkPostAltair,
+  ForkPostElectra,
+  ForkPostGloas,
+  ForkSeq,
+  MAX_SEED_LOOKAHEAD,
+  SLOTS_PER_EPOCH,
+} from "@lodestar/params";
+import {
+  IBeaconStateView,
   RootCache,
   computeEpochAtSlot,
   computeStartSlotAtEpoch,
   computeTimeAtSlot,
-  isExecutionStateType,
   isStartSlotOfEpoch,
-  isStateValidatorsNodesPopulated,
 } from "@lodestar/state-transition";
-import {Attestation, BeaconBlock, altair, capella, electra, isGloasBeaconBlock, phase0, ssz} from "@lodestar/types";
+import {
+  Attestation,
+  BeaconBlock,
+  SignedBeaconBlock,
+  altair,
+  capella,
+  electra,
+  isGloasBeaconBlock,
+  phase0,
+  ssz,
+} from "@lodestar/types";
 import {isErrorAborted, toRootHex} from "@lodestar/utils";
 import {ZERO_HASH_HEX} from "../../constants/index.js";
 import {callInNextEventLoop} from "../../util/eventLoop.js";
@@ -30,7 +45,7 @@ import type {BeaconChain} from "../chain.js";
 import {ChainEvent, ReorgEventData} from "../emitter.js";
 import {ForkchoiceCaller} from "../forkChoice/index.js";
 import {REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC} from "../reprocess.js";
-import {toCheckpointHex} from "../stateCache/persistentCheckpointsCache.js";
+import {toCheckpointHexPayload} from "../stateCache/persistentCheckpointsCache.js";
 import {isBlockInputBlobs, isBlockInputColumns} from "./blockInput/blockInput.js";
 import {AttestationImportOpt, FullyVerifiedBlock, ImportBlockOpts} from "./types.js";
 import {getCheckpointFromState} from "./utils/checkpoint.js";
@@ -116,7 +131,28 @@ export async function importBlock(
 
   // This adds the state necessary to process the next block
   // Some block event handlers require state being in state cache so need to do this before emitting EventType.block
-  this.regen.processState(blockRootHex, postState);
+  // Pre-Gloas: blockSummary.payloadStatus is always FULL, payloadPresent = true
+  // Post-Gloas: blockSummary.payloadStatus is always PENDING, so payloadPresent = false (block state only, no payload processing yet)
+  const payloadPresent = !isGloasBlock(blockSummary);
+  // processState manages both block state and payload state variants together for memory/disk management
+  this.regen.processBlockState(blockRootHex, postState);
+
+  // For Gloas blocks, create PayloadEnvelopeInput so it's available for later payload import
+  if (fork >= ForkSeq.gloas) {
+    this.seenPayloadEnvelopeInputCache.add({
+      blockRootHex,
+      block: block as SignedBeaconBlock<ForkPostGloas>,
+      sampledColumns: this.custodyConfig.sampledColumns,
+      custodyColumns: this.custodyConfig.custodyColumns,
+      timeCreatedSec: fullyVerifiedBlock.seenTimestampSec,
+    });
+    this.logger.debug("Created PayloadEnvelopeInput for block", {
+      slot: blockSlot,
+      root: blockRootHex,
+      source: source.source,
+      ...(opts.seenTimestampSec !== undefined ? {recvToImport: Date.now() / 1000 - opts.seenTimestampSec} : {}),
+    });
+  }
 
   this.metrics?.importBlock.bySource.inc({source: source.source});
   this.logger.verbose("Added block to forkchoice and state cache", {slot: blockSlot, root: blockRootHex});
@@ -149,7 +185,7 @@ export async function importBlock(
         const attDataRoot = toRootHex(ssz.phase0.AttestationData.hashTreeRoot(indexedAttestation.data));
         addAttestation.call(
           this,
-          postState.epochCtx,
+          postState,
           target,
           attDataRoot,
           attestation as Attestation<ForkPostElectra>,
@@ -336,7 +372,7 @@ export async function importBlock(
         try {
           this.lightClientServer?.onImportBlockHead(
             block.message as BeaconBlock<ForkPostAltair>,
-            postState as CachedBeaconStateAltair,
+            postState,
             parentBlockSlot
           );
         } catch (e) {
@@ -357,11 +393,11 @@ export async function importBlock(
   // and the block is weak and can potentially be reorged out.
   let shouldOverrideFcu = false;
 
-  if (blockSlot >= currentSlot && isExecutionStateType(postState)) {
+  if (blockSlot >= currentSlot && postState.isExecutionStateType) {
     let notOverrideFcuReason = NotReorgedReason.Unknown;
     const proposalSlot = blockSlot + 1;
     try {
-      const proposerIndex = postState.epochCtx.getBeaconProposer(proposalSlot);
+      const proposerIndex = postState.getBeaconProposer(proposalSlot);
       const feeRecipient = this.beaconProposerCache.get(proposerIndex);
 
       if (feeRecipient) {
@@ -441,7 +477,7 @@ export async function importBlock(
     }
   }
 
-  if (!isStateValidatorsNodesPopulated(postState)) {
+  if (!postState.isStateValidatorsNodesPopulated()) {
     this.logger.verbose("After importBlock caching postState without SSZ cache", {slot: postState.slot});
   }
 
@@ -456,14 +492,14 @@ export async function importBlock(
     // Cache state to preserve epoch transition work
     const checkpointState = postState;
     const cp = getCheckpointFromState(checkpointState);
-    this.regen.addCheckpointState(cp, checkpointState);
+    this.regen.addCheckpointState(cp, checkpointState, payloadPresent);
     // consumers should not mutate state ever
     this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState);
 
     // Note: in-lined code from previos handler of ChainEvent.checkpoint
-    this.logger.verbose("Checkpoint processed", toCheckpointHex(cp));
+    this.logger.verbose("Checkpoint processed", toCheckpointHexPayload(cp, payloadPresent));
 
-    const activeValidatorsCount = checkpointState.epochCtx.currentShuffling.activeIndices.length;
+    const activeValidatorsCount = checkpointState.activeValidatorCount;
     this.metrics?.currentActiveValidators.set(activeValidatorsCount);
     this.metrics?.currentValidators.set({status: "active"}, activeValidatorsCount);
 
@@ -479,7 +515,7 @@ export async function importBlock(
       const justifiedEpoch = justifiedCheckpoint.epoch;
       const preJustifiedEpoch = parentBlockSummary.justifiedEpoch;
       if (justifiedEpoch > preJustifiedEpoch) {
-        this.logger.verbose("Checkpoint justified", toCheckpointHex(justifiedCheckpoint));
+        this.logger.verbose("Checkpoint justified", toCheckpointHexPayload(justifiedCheckpoint, payloadPresent));
         this.metrics?.previousJustifiedEpoch.set(checkpointState.previousJustifiedCheckpoint.epoch);
         this.metrics?.currentJustifiedEpoch.set(justifiedCheckpoint.epoch);
       }
@@ -493,7 +529,7 @@ export async function importBlock(
           state: toRootHex(checkpointState.hashTreeRoot()),
           executionOptimistic: false,
         });
-        this.logger.verbose("Checkpoint finalized", toCheckpointHex(finalizedCheckpoint));
+        this.logger.verbose("Checkpoint finalized", toCheckpointHexPayload(finalizedCheckpoint, payloadPresent));
         this.metrics?.finalizedEpoch.set(finalizedCheckpoint.epoch);
       }
     }
@@ -548,7 +584,7 @@ export async function importBlock(
     this.validatorMonitor?.registerSyncAggregateInBlock(
       blockEpoch,
       (block as altair.SignedBeaconBlock).message.body.syncAggregate,
-      fullyVerifiedBlock.postState.epochCtx.currentSyncCommitteeIndexed.validatorIndices
+      fullyVerifiedBlock.postState.currentSyncCommitteeIndexed.validatorIndices
     );
   }
 
@@ -590,7 +626,7 @@ export async function importBlock(
 export function addAttestationPreElectra(
   this: BeaconChain,
   // added to have the same signature as addAttestationPostElectra
-  _: EpochCache,
+  _: IBeaconStateView,
   target: phase0.Checkpoint,
   attDataRoot: string,
   attestation: Attestation,
@@ -607,7 +643,7 @@ export function addAttestationPreElectra(
 
 export function addAttestationPostElectra(
   this: BeaconChain,
-  epochCtx: EpochCache,
+  state: IBeaconStateView,
   target: phase0.Checkpoint,
   attDataRoot: string,
   attestation: Attestation<ForkPostElectra>,
@@ -625,7 +661,7 @@ export function addAttestationPostElectra(
   } else {
     const attSlot = attestation.data.slot;
     const attEpoch = computeEpochAtSlot(attSlot);
-    const decisionRoot = epochCtx.getShufflingDecisionRoot(attEpoch);
+    const decisionRoot = state.getShufflingDecisionRoot(attEpoch);
     const committees = this.shufflingCache.getBeaconCommittees(attEpoch, decisionRoot, attSlot, committeeIndices);
     const aggregationBools = attestation.aggregationBits.toBoolArray();
     let offset = 0;

package/src/chain/blocks/importExecutionPayload.ts

@@ -0,0 +1,236 @@
+import {routes} from "@lodestar/api";
+import {ForkName} from "@lodestar/params";
+import {getExecutionPayloadEnvelopeSignatureSet} from "@lodestar/state-transition";
+import {byteArrayEquals, fromHex, toRootHex} from "@lodestar/utils";
+import {ExecutionPayloadStatus} from "../../execution/index.js";
+import {isQueueErrorAborted} from "../../util/queue/index.js";
+import {BeaconChain} from "../chain.js";
+import {RegenCaller} from "../regen/interface.js";
+import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js";
+import {ImportPayloadOpts} from "./types.js";
+
+const EVENTSTREAM_EMIT_RECENT_EXECUTION_PAYLOAD_SLOTS = 64;
+
+export enum PayloadErrorCode {
+  EXECUTION_ENGINE_INVALID = "PAYLOAD_ERROR_EXECUTION_ENGINE_INVALID",
+  EXECUTION_ENGINE_ERROR = "PAYLOAD_ERROR_EXECUTION_ENGINE_ERROR",
+  BLOCK_NOT_IN_FORK_CHOICE = "PAYLOAD_ERROR_BLOCK_NOT_IN_FORK_CHOICE",
+  STATE_TRANSITION_ERROR = "PAYLOAD_ERROR_STATE_TRANSITION_ERROR",
+  INVALID_SIGNATURE = "PAYLOAD_ERROR_INVALID_SIGNATURE",
+}
+
+export type PayloadErrorType =
+  | {
+      code: PayloadErrorCode.EXECUTION_ENGINE_INVALID;
+      execStatus: ExecutionPayloadStatus;
+      errorMessage: string;
+    }
+  | {
+      code: PayloadErrorCode.EXECUTION_ENGINE_ERROR;
+      execStatus: ExecutionPayloadStatus;
+      errorMessage: string;
+    }
+  | {
+      code: PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE;
+      blockRootHex: string;
+    }
+  | {
+      code: PayloadErrorCode.STATE_TRANSITION_ERROR;
+      message: string;
+    }
+  | {
+      code: PayloadErrorCode.INVALID_SIGNATURE;
+    };
+
+export class PayloadError extends Error {
+  type: PayloadErrorType;
+
+  constructor(type: PayloadErrorType, message?: string) {
+    super(message ?? type.code);
+    this.type = type;
+  }
+}
+
+/**
+ * Import an execution payload envelope after all data is available.
+ *
+ * This function:
+ * 1. Gets the ProtoBlock from fork choice
+ * 2. Applies write-queue backpressure (waitForSpace) early, before verification
+ * 3. Regenerates the block state
+ * 4. Runs EL verification (notifyNewPayload) in parallel with signature verification and processExecutionPayloadEnvelope
+ * 5. Persists verified payload envelope to hot DB
+ * 6. Updates fork choice
+ * 7. Caches the post-execution payload state
+ * 8. Records metrics for column sources
+ *
+ */
+export async function importExecutionPayload(
+  this: BeaconChain,
+  payloadInput: PayloadEnvelopeInput,
+  opts: ImportPayloadOpts = {}
+): Promise<void> {
+  const envelope = payloadInput.getPayloadEnvelope();
+  const blockRootHex = payloadInput.blockRootHex;
+
+  // 1. Get ProtoBlock for parent root lookup
+  const protoBlock = this.forkChoice.getBlockHexDefaultStatus(blockRootHex);
+  if (!protoBlock) {
+    throw new PayloadError({
+      code: PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE,
+      blockRootHex,
+    });
+  }
+
+  // 2. Apply backpressure from the write queue early, before doing verification work.
+  // The actual DB write is deferred until after verification succeeds.
+  await this.unfinalizedPayloadEnvelopeWrites.waitForSpace();
+
+  // 3. Get pre-state for processExecutionPayloadEnvelope
+  // We need the block state (post-block, pre-payload) to process the envelope
+  const blockState = await this.regen.getBlockSlotState(
+    protoBlock,
+    protoBlock.slot,
+    {dontTransferCache: true},
+    RegenCaller.processBlock
+  );
+
+  // 4. Run verification steps in parallel
+  // Note: No data availability check needed here - importExecutionPayload is only
+  // called when payloadInput.isComplete() is true, so all data is already available.
+  const [execResult, signatureValid, postPayloadResult] = await Promise.all([
+    this.executionEngine.notifyNewPayload(
+      ForkName.gloas,
+      envelope.message.payload,
+      payloadInput.getVersionedHashes(),
+      fromHex(protoBlock.parentRoot),
+      envelope.message.executionRequests
+    ),
+
+    opts.validSignature === true
+      ? Promise.resolve(true)
+      : (async () => {
+          const signatureSet = getExecutionPayloadEnvelopeSignatureSet(
+            this.config,
+            this.pubkeyCache,
+            blockState,
+            envelope,
+            payloadInput.proposerIndex
+          );
+          return this.bls.verifySignatureSets([signatureSet]);
+        })(),
+
+    // Signature verified separately above.
+    // State root check is done separately below with better error typing (matching block pipeline pattern).
+    (async () => {
+      try {
+        return {
+          postPayloadState: blockState.processExecutionPayloadEnvelope(envelope, {
+            verifySignature: false,
+            verifyStateRoot: false,
+          }),
+        };
+      } catch (e) {
+        throw new PayloadError(
+          {
+            code: PayloadErrorCode.STATE_TRANSITION_ERROR,
+            message: (e as Error).message,
+          },
+          `State transition error: ${(e as Error).message}`
+        );
+      }
+    })(),
+  ]);
+
+  // 4b. Check signature verification result
+  if (!signatureValid) {
+    throw new PayloadError({code: PayloadErrorCode.INVALID_SIGNATURE});
+  }
+
+  // 5. Handle EL response
+  switch (execResult.status) {
+    case ExecutionPayloadStatus.VALID:
+      break;
+
+    case ExecutionPayloadStatus.INVALID:
+      throw new PayloadError({
+        code: PayloadErrorCode.EXECUTION_ENGINE_INVALID,
+        execStatus: execResult.status,
+        errorMessage: execResult.validationError ?? "",
+      });
+
+    case ExecutionPayloadStatus.ACCEPTED:
+    case ExecutionPayloadStatus.SYNCING:
+      // TODO GLOAS: Handle optimistic import for payload - for now treat as error
+      throw new PayloadError({
+        code: PayloadErrorCode.EXECUTION_ENGINE_ERROR,
+        execStatus: execResult.status,
+        errorMessage: execResult.validationError ?? "EL syncing, payload not yet validated",
+      });
+
+    case ExecutionPayloadStatus.INVALID_BLOCK_HASH:
+    case ExecutionPayloadStatus.ELERROR:
+    case ExecutionPayloadStatus.UNAVAILABLE:
+      throw new PayloadError({
+        code: PayloadErrorCode.EXECUTION_ENGINE_ERROR,
+        execStatus: execResult.status,
+        errorMessage: execResult.validationError ?? "",
+      });
+  }
+
+  // 5b. Verify envelope state root matches post-state
+  const postPayloadState = postPayloadResult.postPayloadState;
+  const postPayloadStateRoot = postPayloadState.hashTreeRoot();
+  if (!byteArrayEquals(envelope.message.stateRoot, postPayloadStateRoot)) {
+    throw new PayloadError({
+      code: PayloadErrorCode.STATE_TRANSITION_ERROR,
+      message: `Envelope state root mismatch expected=${toRootHex(envelope.message.stateRoot)} actual=${toRootHex(postPayloadStateRoot)}`,
+    });
+  }
+
+  // 5c. Persist payload envelope to hot DB (performed asynchronously to avoid blocking)
+  this.unfinalizedPayloadEnvelopeWrites.push(payloadInput).catch((e) => {
+    if (!isQueueErrorAborted(e)) {
+      this.logger.error(
+        "Error pushing payload envelope to unfinalized write queue",
+        {slot: payloadInput.slot, root: blockRootHex},
+        e as Error
+      );
+    }
+  });
+
+  // 6. Update fork choice
+  this.forkChoice.onExecutionPayload(
+    blockRootHex,
+    payloadInput.getBlockHashHex(),
+    envelope.message.payload.blockNumber,
+    toRootHex(postPayloadStateRoot)
+  );
+
+  // 7. Cache payload state
+  // TODO GLOAS: Enable when PR #8868 merged (adds processPayloadState)
+  // this.regen.processPayloadState(postPayloadState);
+  // if epoch boundary also call
+  // this.regen.addCheckpointState(cp, checkpointState, true);
+
+  // 8. Record metrics for payload envelope and column sources
+  this.metrics?.importPayload.bySource.inc({source: payloadInput.getPayloadEnvelopeSource().source});
+  for (const {source} of payloadInput.getSampledColumnsWithSource()) {
+    this.metrics?.importPayload.columnsBySource.inc({source});
+  }
+
+  this.logger.verbose("Execution payload imported", {
+    slot: payloadInput.slot,
+    root: blockRootHex,
+    blockHash: payloadInput.getBlockHashHex(),
+  });
+
+  // 9. Emit event after payload is fully verified and imported to fork choice, only for recent enough payloads
+  const currentSlot = this.clock.currentSlot;
+  if (currentSlot - payloadInput.slot < EVENTSTREAM_EMIT_RECENT_EXECUTION_PAYLOAD_SLOTS) {
+    this.emitter.emit(routes.events.EventType.executionPayloadAvailable, {
+      slot: payloadInput.slot,
+      blockRoot: blockRootHex,
+    });
+  }
+}

package/src/chain/blocks/index.ts

@@ -126,7 +126,7 @@ export async function processBlocks(
         const {state} = err.type;
         const forkTypes = this.config.getForkTypes(blockSlot);
         this.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `${blockSlot}_invalid_signature`);
-        this.persistInvalidSszView(state, `${state.slot}_invalid_signature`);
+        this.persistInvalidSszBytes("BeaconState", state.serialize(), `${state.slot}_invalid_signature`);
       } else if (err.type.code === BlockErrorCode.INVALID_STATE_ROOT) {
         const {signedBlock} = err;
         const blockSlot = signedBlock.message.slot;

package/src/chain/blocks/payloadEnvelopeInput/index.ts

@@ -0,0 +1,2 @@
+export * from "./payloadEnvelopeInput.js";
+export * from "./types.js";