@lodestar/beacon-node 1.48.0-dev.f22c5ce63e → 1.48.0-rc.0

package/lib/network/processor/gossipHandlers.js

@@ -3,7 +3,7 @@ import { PayloadStatus } from "@lodestar/fork-choice";
 import { ForkSeq, NUMBER_OF_COLUMNS, isForkPostElectra, isForkPostGloas, } from "@lodestar/params";
 import { computeTimeAtSlot } from "@lodestar/state-transition";
 import { isGloasDataColumnSidecar, ssz, sszTypesFor, } from "@lodestar/types";
-import { LogLevel, prettyBytes, toHex, toRootHex } from "@lodestar/utils";
+import { LogLevel, toHex, toRootHex } from "@lodestar/utils";
 import { BlockInputSource, isBlockInputColumns, } from "../../chain/blocks/blockInput/index.js";
 import { PayloadError, PayloadErrorCode } from "../../chain/blocks/importExecutionPayload.js";
 import { PayloadEnvelopeInputSource } from "../../chain/blocks/payloadEnvelopeInput/index.js";
@@ -22,6 +22,7 @@ import { validateGossipProposerPreferences } from "../../chain/validation/propos
 import { OpSource } from "../../chain/validatorMonitor.js";
 import { kzgCommitmentToVersionedHash } from "../../util/blobs.js";
 import { getBlobKzgCommitments, getDataColumnSidecarSlot } from "../../util/dataColumns.js";
+import { callInNextEventLoop } from "../../util/eventLoop.js";
 import { GossipType, } from "../gossip/interface.js";
 import { sszDeserialize } from "../gossip/topic.js";
 import { PeerAction } from "../peers/index.js";
@@ -54,7 +55,6 @@ function getSequentialHandlers(modules, options) {
         const slot = signedBlock.message.slot;
         const forkTypes = config.getForkTypes(slot);
         const blockRootHex = toRootHex(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message));
-        const blockShortHex = prettyBytes(blockRootHex);
         const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
         const recvToValLatency = Date.now() / 1000 - seenTimestampSec;
         // always set block to seen cache for all forks so that we don't need to download it
@@ -62,6 +62,7 @@ function getSequentialHandlers(modules, options) {
         // tracked in https://github.com/ChainSafe/lodestar/issues/7957
         const logCtx = {
             slot,
+            root: blockRootHex,
             currentSlot: chain.clock.currentSlot,
             peerId: peerIdStr,
             delaySec,
@@ -113,7 +114,7 @@ function getSequentialHandlers(modules, options) {
         }
         catch (e) {
             if (e instanceof BlockGossipError) {
-                logger.debug("Gossip block has error", { slot, root: blockShortHex, code: e.type.code });
+                logger.debug("Gossip block has error", { slot, root: blockRootHex, code: e.type.code });
                 if ((e.type.code === BlockErrorCode.PARENT_BLOCK_UNKNOWN ||
                     e.type.code === BlockErrorCode.PARENT_PAYLOAD_UNKNOWN) &&
                     blockInput) {
@@ -131,7 +132,7 @@ function getSequentialHandlers(modules, options) {
                 if (e.action === GossipAction.IGNORE) {
                     throw e;
                 }
-                chain.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `gossip_reject_slot_${slot}`);
+                chain.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `gossip_reject_slot_${slot}`, blockRootHex);
             }
             // REJECT or unexpected (non-BlockGossipError) error: drop the optimistically-added entries from
             // both caches, keeping them consistent.
@@ -155,7 +156,6 @@ function getSequentialHandlers(modules, options) {
         const slot = blobBlockHeader.slot;
         const fork = config.getForkName(slot);
         const blockRootHex = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(blobBlockHeader));
-        const blockShortHex = prettyBytes(blockRootHex);
         const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
         const recvToValLatency = Date.now() / 1000 - seenTimestampSec;
         try {
@@ -204,7 +204,7 @@ function getSequentialHandlers(modules, options) {
             if (e instanceof BlobSidecarGossipError) {
                 // Don't trigger this yet if full block and blobs haven't arrived yet
                 if (e.type.code === BlobSidecarErrorCode.PARENT_UNKNOWN) {
-                    logger.debug("Gossip blob has error", { slot, root: blockShortHex, code: e.type.code });
+                    logger.debug("Gossip blob has error", { slot, root: blockRootHex, code: e.type.code });
                     // no need to trigger `unknownBlockParent` event here, as we already did it in `validateBeaconBlock()`
                     //
                     // TODO(fulu): is this note above correct? Could have random blob that we see that could trigger
@@ -509,7 +509,7 @@ function getSequentialHandlers(modules, options) {
                 logLevel = LogLevel.error;
             }
             metrics?.gossipBlock.processBlockErrors.inc({ error: e instanceof BlockError ? e.type.code : "NOT_BLOCK_ERROR" });
-            logger[logLevel]("Error processing block", { slot, peer: peerIdStr, blockRoot: prettyBytes(blockInput.blockRootHex) }, e);
+            logger[logLevel]("Error processing block", { slot, root: blockInput.blockRootHex, peer: peerIdStr }, e);
             // TODO(fulu): Revisit when we prune block inputs
             chain.seenBlockInputCache.prune(blockInput.blockRootHex);
         });
@@ -520,8 +520,16 @@ function getSequentialHandlers(modules, options) {
             const signedBlock = sszDeserialize(topic, serializedData);
             try {
                 const blockInput = await validateBeaconBlock(signedBlock, topic.boundary.fork, peerIdStr, seenTimestampSec);
-                chain.serializedCache.set(signedBlock, serializedData);
-                handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec);
+                // Handler - deferred to next event loop so the validation result propagates first
+                callInNextEventLoop(() => {
+                    try {
+                        chain.serializedCache.set(signedBlock, serializedData);
+                        handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec);
+                    }
+                    catch (e) {
+                        logger.debug("Error handling gossip block", { slot: signedBlock.message.slot, root: blockInput.blockRootHex }, e);
+                    }
+                });
             }
             catch (e) {
                 // Spec: IGNORE the block, ie not to re-publish to peers
@@ -534,6 +542,8 @@ function getSequentialHandlers(modules, options) {
                     // blockInput was optimistically seeded in validateBeaconBlock and retained on IGNORE
                     const blockInput = chain.seenBlockInputCache.get(e.type.root);
                     if (blockInput) {
+                        // We're returning IGNORE (thrown below), so this block is not forwarded to peers. Unlike the
+                        // happy path there is no rush to forward, so we don't need to wrap in callInNextEventLoop.
                         chain.serializedCache.set(signedBlock, serializedData);
                         // this is technically not a valid gossip block but gossip validation is a cheap subset of checks
                         // this runs the full state transition, so importing an equivocating-but-valid block here is safe.
@@ -553,26 +563,34 @@ function getSequentialHandlers(modules, options) {
                 throw new GossipActionError(GossipAction.REJECT, { code: "PRE_DENEB_BLOCK" });
             }
             const blockInput = await validateBeaconBlob(blobSidecar, topic.subnet, peerIdStr, seenTimestampSec);
-            chain.serializedCache.set(blobSidecar, serializedData);
-            if (!blockInput.hasBlockAndAllData()) {
-                const cutoffTimeMs = getCutoffTimeMs(chain, blobSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
-                chain.logger.debug("Received gossip blob, waiting for full data availability", {
-                    msToWait: cutoffTimeMs,
-                    blobIndex: index,
-                    ...blockInput.getLogMeta(),
-                });
-                blockInput.waitForAllData(cutoffTimeMs).catch((_e) => {
-                    chain.logger.debug("Waited for data after receiving gossip blob. Cut-off reached so attempting to fetch remainder of BlockInput", {
-                        blobIndex: index,
-                        ...blockInput.getLogMeta(),
-                    });
-                    chain.emitter.emit(ChainEvent.incompleteBlockInput, {
-                        blockInput,
-                        peer: peerIdStr,
-                        source: BlockInputSource.gossip,
-                    });
-                });
-            }
+            // Handler - deferred to next event loop so the validation result propagates first
+            callInNextEventLoop(() => {
+                try {
+                    chain.serializedCache.set(blobSidecar, serializedData);
+                    if (!blockInput.hasBlockAndAllData()) {
+                        const cutoffTimeMs = getCutoffTimeMs(chain, blobSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
+                        chain.logger.debug("Received gossip blob, waiting for full data availability", {
+                            msToWait: cutoffTimeMs,
+                            blobIndex: index,
+                            ...blockInput.getLogMeta(),
+                        });
+                        blockInput.waitForAllData(cutoffTimeMs).catch((_e) => {
+                            chain.logger.debug("Waited for data after receiving gossip blob. Cut-off reached so attempting to fetch remainder of BlockInput", {
+                                blobIndex: index,
+                                ...blockInput.getLogMeta(),
+                            });
+                            chain.emitter.emit(ChainEvent.incompleteBlockInput, {
+                                blockInput,
+                                peer: peerIdStr,
+                                source: BlockInputSource.gossip,
+                            });
+                        });
+                    }
+                }
+                catch (e) {
+                    logger.debug("Error handling gossip blob", { slot: blobSlot, root: blockInput.blockRootHex, index }, e);
+                }
+            });
         },
         [GossipType.data_column_sidecar]: async ({ gossipData, topic, peerIdStr, seenTimestampSec, }) => {
             const { fork } = topic.boundary;
@@ -592,50 +610,58 @@ function getSequentialHandlers(modules, options) {
                 }
                 // After gloas, data columns are tracked in PayloadEnvelopeInput
                 const payloadInput = await validatePayloadDataColumn(dataColumnSidecar, topic.subnet, peerIdStr, seenTimestampSec);
-                chain.serializedCache.set(dataColumnSidecar, serializedData);
-                const payloadInputMeta = payloadInput.getLogMeta();
-                const { receivedColumns } = payloadInputMeta;
-                // it's not helpful to track every single column received
-                // instead of that, track 1st, 8th, 16th 32th, 64th, and 128th column
-                switch (receivedColumns) {
-                    case 1:
-                    case config.SAMPLES_PER_SLOT:
-                    case 2 * config.SAMPLES_PER_SLOT:
-                    case NUMBER_OF_COLUMNS / 4:
-                    case NUMBER_OF_COLUMNS / 2:
-                    case NUMBER_OF_COLUMNS:
-                        metrics?.dataColumns.elapsedTimeTillReceived.observe({ receivedOrder: receivedColumns }, delaySec);
-                        break;
-                }
-                if (!payloadInput.hasComputedAllData()) {
-                    // if we've received at least half of the columns, trigger reconstruction of the rest
-                    if (receivedColumns >= NUMBER_OF_COLUMNS / 2) {
-                        chain.columnReconstructionTracker.triggerColumnReconstruction(payloadInput);
+                // Handler - deferred to next event loop so the validation result propagates first
+                callInNextEventLoop(() => {
+                    try {
+                        chain.serializedCache.set(dataColumnSidecar, serializedData);
+                        const payloadInputMeta = payloadInput.getLogMeta();
+                        const { receivedColumns } = payloadInputMeta;
+                        // it's not helpful to track every single column received
+                        // instead of that, track 1st, 8th, 16th 32th, 64th, and 128th column
+                        switch (receivedColumns) {
+                            case 1:
+                            case config.SAMPLES_PER_SLOT:
+                            case 2 * config.SAMPLES_PER_SLOT:
+                            case NUMBER_OF_COLUMNS / 4:
+                            case NUMBER_OF_COLUMNS / 2:
+                            case NUMBER_OF_COLUMNS:
+                                metrics?.dataColumns.elapsedTimeTillReceived.observe({ receivedOrder: receivedColumns }, delaySec);
+                                break;
+                        }
+                        if (!payloadInput.hasComputedAllData()) {
+                            // if we've received at least half of the columns, trigger reconstruction of the rest
+                            if (receivedColumns >= NUMBER_OF_COLUMNS / 2) {
+                                chain.columnReconstructionTracker.triggerColumnReconstruction(payloadInput);
+                            }
+                            chain.logger.debug("Received gossip data column, payload envelope input not yet complete", {
+                                dataColumnIndex: index,
+                                ...payloadInputMeta,
+                            });
+                        }
+                        // NOTE: we do NOT call chain.processExecutionPayload here. That is triggered only by
+                        // envelope arrival (gossip or API). An in-flight importExecutionPayload is awaiting
+                        // payloadInput.waitForAllData(); addColumn above will resolve it once hasAllData flips.
+                        if (!payloadInput.isComplete()) {
+                            const cutoffTimeMs = getCutoffTimeMs(chain, dataColumnSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
+                            // do not await here to not delay gossip validation
+                            payloadInput.waitForEnvelopeAndAllData(cutoffTimeMs).catch((_e) => {
+                                chain.logger.debug("Waited for envelope and data after receiving gossip column. Cut-off reached so emitting incompletePayloadEnvelope", {
+                                    dataColumnIndex: index,
+                                    ...payloadInputMeta,
+                                });
+                                // TODO GLOAS: UnknownBlockSync to handle this event
+                                chain.emitter.emit(ChainEvent.incompletePayloadEnvelope, {
+                                    payloadInput,
+                                    peer: peerIdStr,
+                                    source: BlockInputSource.gossip,
+                                });
+                            });
+                        }
                     }
-                    chain.logger.debug("Received gossip data column, payload envelope input not yet complete", {
-                        dataColumnIndex: index,
-                        ...payloadInputMeta,
-                    });
-                }
-                // NOTE: we do NOT call chain.processExecutionPayload here. That is triggered only by
-                // envelope arrival (gossip or API). An in-flight importExecutionPayload is awaiting
-                // payloadInput.waitForAllData(); addColumn above will resolve it once hasAllData flips.
-                if (!payloadInput.isComplete()) {
-                    const cutoffTimeMs = getCutoffTimeMs(chain, dataColumnSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
-                    // do not await here to not delay gossip validation
-                    payloadInput.waitForEnvelopeAndAllData(cutoffTimeMs).catch((_e) => {
-                        chain.logger.debug("Waited for envelope and data after receiving gossip column. Cut-off reached so emitting incompletePayloadEnvelope", {
-                            dataColumnIndex: index,
-                            ...payloadInputMeta,
-                        });
-                        // TODO GLOAS: UnknownBlockSync to handle this event
-                        chain.emitter.emit(ChainEvent.incompletePayloadEnvelope, {
-                            payloadInput,
-                            peer: peerIdStr,
-                            source: BlockInputSource.gossip,
-                        });
-                    });
-                }
+                    catch (e) {
+                        logger.debug("Error handling gossip data column", { slot: dataColumnSlot, root: payloadInput.blockRootHex, index }, e);
+                    }
+                });
             }
             else {
                 if (config.getForkSeq(dataColumnSlot) < ForkSeq.fulu) {
@@ -651,49 +677,57 @@ function getSequentialHandlers(modules, options) {
                 }
                 // Before gloas, data columns are tracked in BlockInput
                 const blockInput = await validateBeaconDataColumn(dataColumnSidecar, serializedData, topic.subnet, peerIdStr, seenTimestampSec);
-                chain.serializedCache.set(dataColumnSidecar, serializedData);
-                const blockInputMeta = blockInput.getLogMeta();
-                const { receivedColumns } = blockInputMeta;
-                // it's not helpful to track every single column received
-                // instead of that, track 1st, 8th, 16th 32th, 64th, and 128th column
-                switch (receivedColumns) {
-                    case 1:
-                    case config.SAMPLES_PER_SLOT:
-                    case 2 * config.SAMPLES_PER_SLOT:
-                    case NUMBER_OF_COLUMNS / 4:
-                    case NUMBER_OF_COLUMNS / 2:
-                    case NUMBER_OF_COLUMNS:
-                        metrics?.dataColumns.elapsedTimeTillReceived.observe({ receivedOrder: receivedColumns }, delaySec);
-                        break;
-                }
-                if (!blockInput.hasComputedAllData()) {
-                    // immediately attempt fetch of data columns from execution engine
-                    chain.getBlobsTracker.triggerGetBlobs(blockInput);
-                    // if we've received at least half of the columns, trigger reconstruction of the rest
-                    if (blockInput.columnCount >= NUMBER_OF_COLUMNS / 2) {
-                        chain.columnReconstructionTracker.triggerColumnReconstruction(blockInput);
+                // Handler - deferred to next event loop so the validation result propagates first
+                callInNextEventLoop(() => {
+                    try {
+                        chain.serializedCache.set(dataColumnSidecar, serializedData);
+                        const blockInputMeta = blockInput.getLogMeta();
+                        const { receivedColumns } = blockInputMeta;
+                        // it's not helpful to track every single column received
+                        // instead of that, track 1st, 8th, 16th 32th, 64th, and 128th column
+                        switch (receivedColumns) {
+                            case 1:
+                            case config.SAMPLES_PER_SLOT:
+                            case 2 * config.SAMPLES_PER_SLOT:
+                            case NUMBER_OF_COLUMNS / 4:
+                            case NUMBER_OF_COLUMNS / 2:
+                            case NUMBER_OF_COLUMNS:
+                                metrics?.dataColumns.elapsedTimeTillReceived.observe({ receivedOrder: receivedColumns }, delaySec);
+                                break;
+                        }
+                        if (!blockInput.hasComputedAllData()) {
+                            // immediately attempt fetch of data columns from execution engine
+                            chain.getBlobsTracker.triggerGetBlobs(blockInput);
+                            // if we've received at least half of the columns, trigger reconstruction of the rest
+                            if (blockInput.columnCount >= NUMBER_OF_COLUMNS / 2) {
+                                chain.columnReconstructionTracker.triggerColumnReconstruction(blockInput);
+                            }
+                        }
+                        if (!blockInput.hasBlockAndAllData()) {
+                            const cutoffTimeMs = getCutoffTimeMs(chain, dataColumnSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
+                            chain.logger.debug("Received gossip data column, waiting for full data availability", {
+                                msToWait: cutoffTimeMs,
+                                dataColumnIndex: index,
+                                ...blockInputMeta,
+                            });
+                            // do not await here to not delay gossip validation
+                            blockInput.waitForBlockAndAllData(cutoffTimeMs).catch((_e) => {
+                                chain.logger.debug("Waited for data after receiving gossip column. Cut-off reached so attempting to fetch remainder of BlockInput", {
+                                    dataColumnIndex: index,
+                                    ...blockInputMeta,
+                                });
+                                chain.emitter.emit(ChainEvent.incompleteBlockInput, {
+                                    blockInput,
+                                    peer: peerIdStr,
+                                    source: BlockInputSource.gossip,
+                                });
+                            });
+                        }
                     }
-                }
-                if (!blockInput.hasBlockAndAllData()) {
-                    const cutoffTimeMs = getCutoffTimeMs(chain, dataColumnSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
-                    chain.logger.debug("Received gossip data column, waiting for full data availability", {
-                        msToWait: cutoffTimeMs,
-                        dataColumnIndex: index,
-                        ...blockInputMeta,
-                    });
-                    // do not await here to not delay gossip validation
-                    blockInput.waitForBlockAndAllData(cutoffTimeMs).catch((_e) => {
-                        chain.logger.debug("Waited for data after receiving gossip column. Cut-off reached so attempting to fetch remainder of BlockInput", {
-                            dataColumnIndex: index,
-                            ...blockInputMeta,
-                        });
-                        chain.emitter.emit(ChainEvent.incompleteBlockInput, {
-                            blockInput,
-                            peer: peerIdStr,
-                            source: BlockInputSource.gossip,
-                        });
-                    });
-                }
+                    catch (e) {
+                        logger.debug("Error handling gossip data column", { slot: dataColumnSlot, root: blockInput.blockRootHex, index }, e);
+                    }
+                });
             }
         },
         [GossipType.beacon_aggregate_and_proof]: async ({ gossipData, topic, seenTimestampSec, }) => {
@@ -710,62 +744,76 @@ function getSequentialHandlers(modules, options) {
                 }
                 throw e;
             }
-            // Handler
+            // Handler - deferred to next event loop so the validation result propagates first
             const { indexedAttestation, committeeValidatorIndices, attDataRootHex } = validationResult;
-            chain.validatorMonitor?.registerGossipAggregatedAttestation(seenTimestampSec, signedAggregateAndProof, indexedAttestation);
-            const aggregatedAttestation = signedAggregateAndProof.message.aggregate;
-            const insertOutcome = chain.aggregatedAttestationPool.add(aggregatedAttestation, attDataRootHex, indexedAttestation.attestingIndices.length, committeeValidatorIndices);
-            metrics?.opPool.aggregatedAttestationPool.gossipInsertOutcome.inc({ insertOutcome });
-            if (!options.dontSendGossipAttestationsToForkchoice) {
+            callInNextEventLoop(() => {
                 try {
-                    chain.forkChoice.onAttestation(indexedAttestation, attDataRootHex);
+                    chain.validatorMonitor?.registerGossipAggregatedAttestation(seenTimestampSec, signedAggregateAndProof, indexedAttestation);
+                    const aggregatedAttestation = signedAggregateAndProof.message.aggregate;
+                    const insertOutcome = chain.aggregatedAttestationPool.add(aggregatedAttestation, attDataRootHex, indexedAttestation.attestingIndices.length, committeeValidatorIndices);
+                    metrics?.opPool.aggregatedAttestationPool.gossipInsertOutcome.inc({ insertOutcome });
+                    if (!options.dontSendGossipAttestationsToForkchoice) {
+                        chain.forkChoice.onAttestation(indexedAttestation, attDataRootHex);
+                    }
+                    chain.emitter.emit(routes.events.EventType.attestation, signedAggregateAndProof.message.aggregate);
                 }
                 catch (e) {
-                    logger.debug("Error adding gossip aggregated attestation to forkchoice", { slot: aggregatedAttestation.data.slot }, e);
+                    logger.debug("Error handling gossip aggregate and proof", {
+                        slot: signedAggregateAndProof.message.aggregate.data.slot,
+                        root: toRootHex(signedAggregateAndProof.message.aggregate.data.beaconBlockRoot),
+                    }, e);
                 }
-            }
-            chain.emitter.emit(routes.events.EventType.attestation, signedAggregateAndProof.message.aggregate);
+            });
         },
         [GossipType.attester_slashing]: async ({ gossipData, topic, }) => {
             const { serializedData } = gossipData;
             const { fork } = topic.boundary;
             const attesterSlashing = sszDeserialize(topic, serializedData);
             await validateGossipAttesterSlashing(chain, attesterSlashing);
-            // Handler
-            try {
-                chain.opPool.insertAttesterSlashing(fork, attesterSlashing);
-                chain.forkChoice.onAttesterSlashing(attesterSlashing);
-            }
-            catch (e) {
-                logger.error("Error adding attesterSlashing to pool", {}, e);
-            }
-            chain.emitter.emit(routes.events.EventType.attesterSlashing, attesterSlashing);
+            // Handler - deferred to next event loop so the validation result propagates first
+            callInNextEventLoop(() => {
+                try {
+                    chain.opPool.insertAttesterSlashing(fork, attesterSlashing);
+                    chain.forkChoice.onAttesterSlashing(attesterSlashing);
+                    chain.emitter.emit(routes.events.EventType.attesterSlashing, attesterSlashing);
+                }
+                catch (e) {
+                    logger.debug("Error handling gossip attester slashing", { slot: attesterSlashing.attestation1.data.slot }, e);
+                }
+            });
         },
         [GossipType.proposer_slashing]: async ({ gossipData, topic, }) => {
             const { serializedData } = gossipData;
             const proposerSlashing = sszDeserialize(topic, serializedData);
             await validateGossipProposerSlashing(chain, proposerSlashing);
-            // Handler
-            try {
-                chain.opPool.insertProposerSlashing(proposerSlashing);
-            }
-            catch (e) {
-                logger.error("Error adding attesterSlashing to pool", {}, e);
-            }
-            chain.emitter.emit(routes.events.EventType.proposerSlashing, proposerSlashing);
+            // Handler - deferred to next event loop so the validation result propagates first
+            callInNextEventLoop(() => {
+                try {
+                    chain.opPool.insertProposerSlashing(proposerSlashing);
+                    chain.emitter.emit(routes.events.EventType.proposerSlashing, proposerSlashing);
+                }
+                catch (e) {
+                    logger.debug("Error handling gossip proposer slashing", {
+                        slot: proposerSlashing.signedHeader1.message.slot,
+                        proposerIndex: proposerSlashing.signedHeader1.message.proposerIndex,
+                    }, e);
+                }
+            });
         },
         [GossipType.voluntary_exit]: async ({ gossipData, topic }) => {
             const { serializedData } = gossipData;
             const voluntaryExit = sszDeserialize(topic, serializedData);
             await validateGossipVoluntaryExit(chain, voluntaryExit);
-            // Handler
-            try {
-                chain.opPool.insertVoluntaryExit(voluntaryExit);
-            }
-            catch (e) {
-                logger.error("Error adding voluntaryExit to pool", {}, e);
-            }
-            chain.emitter.emit(routes.events.EventType.voluntaryExit, voluntaryExit);
+            // Handler - deferred to next event loop so the validation result propagates first
+            callInNextEventLoop(() => {
+                try {
+                    chain.opPool.insertVoluntaryExit(voluntaryExit);
+                    chain.emitter.emit(routes.events.EventType.voluntaryExit, voluntaryExit);
+                }
+                catch (e) {
+                    logger.debug("Error handling gossip voluntary exit", { epoch: voluntaryExit.message.epoch, validatorIndex: voluntaryExit.message.validatorIndex }, e);
+                }
+            });
         },
         [GossipType.sync_committee_contribution_and_proof]: async ({ gossipData, topic, }) => {
             const { serializedData } = gossipData;
@@ -776,16 +824,21 @@ function getSequentialHandlers(modules, options) {
                 }
                 throw e;
             });
-            // Handler
-            chain.validatorMonitor?.registerGossipSyncContributionAndProof(contributionAndProof.message, syncCommitteeParticipantIndices);
-            try {
-                const insertOutcome = chain.syncContributionAndProofPool.add(contributionAndProof.message, syncCommitteeParticipantIndices.length);
-                metrics?.opPool.syncContributionAndProofPool.gossipInsertOutcome.inc({ insertOutcome });
-            }
-            catch (e) {
-                logger.error("Error adding to contributionAndProof pool", {}, e);
-            }
-            chain.emitter.emit(routes.events.EventType.contributionAndProof, contributionAndProof);
+            // Handler - deferred to next event loop so the validation result propagates first
+            callInNextEventLoop(() => {
+                try {
+                    chain.validatorMonitor?.registerGossipSyncContributionAndProof(contributionAndProof.message, syncCommitteeParticipantIndices);
+                    const insertOutcome = chain.syncContributionAndProofPool.add(contributionAndProof.message, syncCommitteeParticipantIndices.length);
+                    metrics?.opPool.syncContributionAndProofPool.gossipInsertOutcome.inc({ insertOutcome });
+                    chain.emitter.emit(routes.events.EventType.contributionAndProof, contributionAndProof);
+                }
+                catch (e) {
+                    logger.debug("Error handling gossip contribution and proof", {
+                        slot: contributionAndProof.message.contribution.slot,
+                        subcommitteeIndex: contributionAndProof.message.contribution.subcommitteeIndex,
+                    }, e);
+                }
+            });
         },
         [GossipType.sync_committee]: async ({ gossipData, topic }) => {
             const { serializedData } = gossipData;
@@ -801,16 +854,19 @@ function getSequentialHandlers(modules, options) {
                 }
                 throw e;
             }
-            // Handler — add for ALL positions this validator holds in the subcommittee
-            try {
-                for (const indexInSubcommittee of indicesInSubcommittee) {
-                    const insertOutcome = chain.syncCommitteeMessagePool.add(subnet, syncCommittee, indexInSubcommittee);
-                    metrics?.opPool.syncCommitteeMessagePoolInsertOutcome.inc({ insertOutcome });
+            // Handler - deferred to next event loop so the validation result propagates first
+            // add for ALL positions this validator holds in the subcommittee
+            callInNextEventLoop(() => {
+                try {
+                    for (const indexInSubcommittee of indicesInSubcommittee) {
+                        const insertOutcome = chain.syncCommitteeMessagePool.add(subnet, syncCommittee, indexInSubcommittee);
+                        metrics?.opPool.syncCommitteeMessagePoolInsertOutcome.inc({ insertOutcome });
+                    }
                 }
-            }
-            catch (e) {
-                logger.debug("Error adding to syncCommittee pool", { subnet }, e);
-            }
+                catch (e) {
+                    logger.debug("Error handling gossip sync committee", { slot: syncCommittee.slot, subnet, validatorIndex: syncCommittee.validatorIndex }, e);
+                }
+            });
         },
         [GossipType.light_client_finality_update]: async ({ gossipData, topic, }) => {
             const { serializedData } = gossipData;
@@ -827,14 +883,16 @@ function getSequentialHandlers(modules, options) {
             const { serializedData } = gossipData;
             const blsToExecutionChange = sszDeserialize(topic, serializedData);
             await validateGossipBlsToExecutionChange(chain, blsToExecutionChange);
-            // Handler
-            try {
-                chain.opPool.insertBlsToExecutionChange(blsToExecutionChange);
-            }
-            catch (e) {
-                logger.error("Error adding blsToExecutionChange to pool", {}, e);
-            }
-            chain.emitter.emit(routes.events.EventType.blsToExecutionChange, blsToExecutionChange);
+            // Handler - deferred to next event loop so the validation result propagates first
+            callInNextEventLoop(() => {
+                try {
+                    chain.opPool.insertBlsToExecutionChange(blsToExecutionChange);
+                    chain.emitter.emit(routes.events.EventType.blsToExecutionChange, blsToExecutionChange);
+                }
+                catch (e) {
+                    logger.debug("Error handling gossip bls to execution change", { validatorIndex: blsToExecutionChange.message.validatorIndex }, e);
+                }
+            });
         },
         [GossipType.execution_payload]: async ({ gossipData, topic, peerIdStr, seenTimestampSec, }) => {
             const { serializedData } = gossipData;
@@ -878,99 +936,137 @@ function getSequentialHandlers(modules, options) {
                     blockRoot: blockRootHex,
                 });
             }
-            chain.serializedCache.set(signedEnvelope, serializedData);
-            payloadInput.addPayloadEnvelope({
-                envelope: signedEnvelope,
-                source: PayloadEnvelopeInputSource.gossip,
-                seenTimestampSec,
-                peerIdStr,
-            });
-            chain.emitter.emit(routes.events.EventType.executionPayloadGossip, {
-                slot,
-                builderIndex: envelope.builderIndex,
-                blockHash: toRootHex(envelope.payload.blockHash),
-                blockRoot: blockRootHex,
-            });
-            chain.processExecutionPayload(payloadInput, { validSignature: true }).catch((e) => {
-                // Adjust verbosity based on error type
-                let logLevel;
-                if (e instanceof PayloadError) {
-                    switch (e.type.code) {
-                        // BLOCK_NOT_IN_FORK_CHOICE should not happen, validateGossipExecutionPayloadEnvelope above
-                        // already verified the block is in fork choice
-                        case PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE:
-                        case PayloadErrorCode.MISS_BLOCK_STATE:
-                        case PayloadErrorCode.EXECUTION_ENGINE_ERROR:
-                            // Errors might indicate an issue with our node or the connected EL client
+            // Handler - deferred to next event loop so the validation result propagates first
+            callInNextEventLoop(() => {
+                try {
+                    chain.serializedCache.set(signedEnvelope, serializedData);
+                    payloadInput.addPayloadEnvelope({
+                        envelope: signedEnvelope,
+                        source: PayloadEnvelopeInputSource.gossip,
+                        seenTimestampSec,
+                        peerIdStr,
+                    });
+                    chain.emitter.emit(routes.events.EventType.executionPayloadGossip, {
+                        slot,
+                        builderIndex: envelope.builderIndex,
+                        blockHash: toRootHex(envelope.payload.blockHash),
+                        blockRoot: blockRootHex,
+                    });
+                    chain.processExecutionPayload(payloadInput, { validSignature: true }).catch((e) => {
+                        // Adjust verbosity based on error type
+                        let logLevel;
+                        if (e instanceof PayloadError) {
+                            switch (e.type.code) {
+                                // BLOCK_NOT_IN_FORK_CHOICE should not happen, validateGossipExecutionPayloadEnvelope above
+                                // already verified the block is in fork choice
+                                case PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE:
+                                case PayloadErrorCode.MISS_BLOCK_STATE:
+                                case PayloadErrorCode.EXECUTION_ENGINE_ERROR:
+                                    // Errors might indicate an issue with our node or the connected EL client
+                                    logLevel = LogLevel.error;
+                                    break;
+                                // INVALID_SIGNATURE should not happen, signature is verified during gossip validation
+                                case PayloadErrorCode.INVALID_SIGNATURE:
+                                case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
+                                case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
+                                    core.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadGossipPayload");
+                                    // Misbehaving peer, but could highlight an issue in another client
+                                    logLevel = LogLevel.warn;
+                                    break;
+                            }
+                        }
+                        else {
+                            // Any unexpected error
                             logLevel = LogLevel.error;
-                            break;
-                        // INVALID_SIGNATURE should not happen, signature is verified during gossip validation
-                        case PayloadErrorCode.INVALID_SIGNATURE:
-                        case PayloadErrorCode.ENVELOPE_VERIFICATION_ERROR:
-                        case PayloadErrorCode.EXECUTION_ENGINE_INVALID:
-                            core.reportPeer(peerIdStr, PeerAction.LowToleranceError, "BadGossipPayload");
-                            // Misbehaving peer, but could highlight an issue in another client
-                            logLevel = LogLevel.warn;
-                            break;
-                    }
+                        }
+                        metrics?.gossipExecutionPayloadEnvelope.processPayloadErrors.inc({
+                            error: e instanceof PayloadError ? e.type.code : "NOT_PAYLOAD_ERROR",
+                        });
+                        chain.logger[logLevel]("Error processing execution payload from gossip", { slot, root: blockRootHex, peer: peerIdStr }, e);
+                    });
                 }
-                else {
-                    // Any unexpected error
-                    logLevel = LogLevel.error;
+                catch (e) {
+                    logger.debug("Error handling gossip execution payload", { slot, root: blockRootHex }, e);
                 }
-                metrics?.gossipExecutionPayloadEnvelope.processPayloadErrors.inc({
-                    error: e instanceof PayloadError ? e.type.code : "NOT_PAYLOAD_ERROR",
-                });
-                chain.logger[logLevel]("Error processing execution payload from gossip", { slot, peer: peerIdStr, root: blockRootHex }, e);
             });
         },
         [GossipType.payload_attestation_message]: async ({ gossipData, topic, seenTimestampSec, }) => {
             const { serializedData } = gossipData;
             const payloadAttestationMessage = sszDeserialize(topic, serializedData);
             const validationResult = await validateGossipPayloadAttestationMessage(chain, payloadAttestationMessage);
-            const delaySec = chain.clock.secFromSlot(payloadAttestationMessage.data.slot, seenTimestampSec);
-            metrics?.gossipPayloadAttestationMessage.elapsedTimeTillReceived.observe({ source: OpSource.gossip }, delaySec);
-            try {
-                const insertOutcome = chain.payloadAttestationPool.add(payloadAttestationMessage, validationResult.attDataRootHex, validationResult.validatorCommitteeIndices);
-                metrics?.opPool.payloadAttestationPool.gossipInsertOutcome.inc({ insertOutcome });
-            }
-            catch (e) {
-                logger.error("Error adding to payloadAttestation pool", {}, e);
-            }
-            chain.forkChoice.notifyPtcMessages(toRootHex(payloadAttestationMessage.data.beaconBlockRoot), payloadAttestationMessage.data.slot, validationResult.validatorCommitteeIndices, payloadAttestationMessage.data.payloadPresent, payloadAttestationMessage.data.blobDataAvailable);
-            chain.emitter.emit(routes.events.EventType.payloadAttestationMessage, {
-                version: config.getForkName(payloadAttestationMessage.data.slot),
-                data: payloadAttestationMessage,
+            // Handler - deferred to next event loop so the validation result propagates first
+            callInNextEventLoop(() => {
+                try {
+                    const delaySec = chain.clock.secFromSlot(payloadAttestationMessage.data.slot, seenTimestampSec);
+                    metrics?.gossipPayloadAttestationMessage.elapsedTimeTillReceived.observe({ source: OpSource.gossip }, delaySec);
+                    const nextSlotProposer = chain.getHeadState().getBeaconProposer(payloadAttestationMessage.data.slot + 1);
+                    const isNextSlotProposer = chain.beaconProposerCache.get(nextSlotProposer) !== undefined;
+                    if (isNextSlotProposer) {
+                        try {
+                            const insertOutcome = chain.payloadAttestationPool.add(payloadAttestationMessage, validationResult.attDataRootHex, validationResult.validatorCommitteeIndices);
+                            metrics?.opPool.payloadAttestationPool.gossipInsertOutcome.inc({ insertOutcome });
+                        }
+                        catch (e) {
+                            logger.debug("Error adding to payloadAttestation pool", {}, e);
+                        }
+                    }
+                    chain.forkChoice.notifyPtcMessages(toRootHex(payloadAttestationMessage.data.beaconBlockRoot), payloadAttestationMessage.data.slot, validationResult.validatorCommitteeIndices, payloadAttestationMessage.data.payloadPresent, payloadAttestationMessage.data.blobDataAvailable);
+                    chain.emitter.emit(routes.events.EventType.payloadAttestationMessage, {
+                        version: config.getForkName(payloadAttestationMessage.data.slot),
+                        data: payloadAttestationMessage,
+                    });
+                }
+                catch (e) {
+                    logger.debug("Error handling gossip payload attestation message", {
+                        slot: payloadAttestationMessage.data.slot,
+                        root: toRootHex(payloadAttestationMessage.data.beaconBlockRoot),
+                    }, e);
+                }
             });
         },
         [GossipType.execution_payload_bid]: async ({ gossipData, topic, seenTimestampSec, }) => {
             const { serializedData } = gossipData;
             const executionPayloadBid = sszDeserialize(topic, serializedData);
             const { proposerIndex } = await validateGossipExecutionPayloadBid(chain, executionPayloadBid);
-            // this could be negative, because it's most likely the bid of next slot comes at this clock slot
-            const elapsedSec = chain.clock.secFromSlot(executionPayloadBid.message.slot, seenTimestampSec);
-            metrics?.gossipExecutionPayloadBid.elapsedTimeTillReceived.observe({ source: OpSource.gossip }, elapsedSec);
-            // Handle valid payload bid by storing in a bid pool
-            try {
-                const insertOutcome = chain.executionPayloadBidPool.add(executionPayloadBid, Math.floor(elapsedSec * 1000));
-                metrics?.opPool.executionPayloadBidPool.gossipInsertOutcome.inc({ insertOutcome });
-            }
-            catch (e) {
-                logger.error("Error adding to executionPayloadBid pool", {}, e);
-            }
-            chain.validatorMonitor?.registerExecutionPayloadBid(OpSource.gossip, proposerIndex, executionPayloadBid.message);
-            chain.emitter.emit(routes.events.EventType.executionPayloadBid, {
-                version: config.getForkName(executionPayloadBid.message.slot),
-                data: executionPayloadBid,
+            // Handler - deferred to next event loop so the validation result propagates first
+            callInNextEventLoop(() => {
+                try {
+                    // this could be negative, because it's most likely the bid of next slot comes at this clock slot
+                    const elapsedSec = chain.clock.secFromSlot(executionPayloadBid.message.slot, seenTimestampSec);
+                    metrics?.gossipExecutionPayloadBid.elapsedTimeTillReceived.observe({ source: OpSource.gossip }, elapsedSec);
+                    // Handle valid payload bid by storing in a bid pool
+                    const insertOutcome = chain.executionPayloadBidPool.add(executionPayloadBid, Math.floor(elapsedSec * 1000));
+                    metrics?.opPool.executionPayloadBidPool.gossipInsertOutcome.inc({ insertOutcome });
+                    chain.validatorMonitor?.registerExecutionPayloadBid(OpSource.gossip, proposerIndex, executionPayloadBid.message);
+                    chain.emitter.emit(routes.events.EventType.executionPayloadBid, {
+                        version: config.getForkName(executionPayloadBid.message.slot),
+                        data: executionPayloadBid,
+                    });
+                }
+                catch (e) {
+                    logger.debug("Error handling gossip execution payload bid", {
+                        slot: executionPayloadBid.message.slot,
+                        root: toRootHex(executionPayloadBid.message.parentBlockRoot),
+                        proposerIndex,
+                    }, e);
+                }
             });
         },
         [GossipType.proposer_preferences]: async ({ gossipData, topic, }) => {
             const { serializedData } = gossipData;
             const signedProposerPreferences = sszDeserialize(topic, serializedData);
             await validateGossipProposerPreferences(chain, signedProposerPreferences);
-            chain.emitter.emit(routes.events.EventType.proposerPreferences, {
-                version: config.getForkName(signedProposerPreferences.message.proposalSlot),
-                data: signedProposerPreferences,
+            // Handler - deferred to next event loop so the validation result propagates first
+            callInNextEventLoop(() => {
+                try {
+                    chain.emitter.emit(routes.events.EventType.proposerPreferences, {
+                        version: config.getForkName(signedProposerPreferences.message.proposalSlot),
+                        data: signedProposerPreferences,
+                    });
+                }
+                catch (e) {
+                    logger.debug("Error handling gossip proposer preferences", { proposalSlot: signedProposerPreferences.message.proposalSlot }, e);
+                }
             });
         },
     };
@@ -997,43 +1093,9 @@ function getBatchHandlers(modules, options) {
                 subnet: param.topic.subnet,
             }));
             const { results: validationResults, batchableBls } = await validateGossipAttestationsSameAttData(fork, chain, validationParams);
-            for (const [i, validationResult] of validationResults.entries()) {
-                if (validationResult.err) {
-                    results.push(validationResult.err);
-                    continue;
-                }
-                // null means no error
-                results.push(null);
-                // Handler
-                const { indexedAttestation, attDataRootHex, attestation, committeeIndex, validatorCommitteeIndex, committeeSize, } = validationResult.result;
-                chain.validatorMonitor?.registerGossipUnaggregatedAttestation(gossipHandlerParams[i].seenTimestampSec, indexedAttestation);
-                const { subnet } = validationResult.result;
-                try {
-                    // Node may be subscribe to extra subnets (long-lived random subnets). For those, validate the messages
-                    // but don't add to attestation pool, to save CPU and RAM
-                    if (aggregatorTracker.shouldAggregate(subnet, indexedAttestation.data.slot)) {
-                        const insertOutcome = chain.attestationPool.add(committeeIndex, attestation, attDataRootHex, validatorCommitteeIndex, committeeSize);
-                        metrics?.opPool.attestationPool.gossipInsertOutcome.inc({ insertOutcome });
-                    }
-                }
-                catch (e) {
-                    logger.error("Error adding unaggregated attestation to pool", { subnet }, e);
-                }
-                if (!options.dontSendGossipAttestationsToForkchoice) {
-                    try {
-                        chain.forkChoice.onAttestation(indexedAttestation, attDataRootHex);
-                    }
-                    catch (e) {
-                        logger.debug("Error adding gossip unaggregated attestation to forkchoice", { subnet }, e);
-                    }
-                }
-                if (isForkPostElectra(fork)) {
-                    chain.emitter.emit(routes.events.EventType.singleAttestation, attestation);
-                }
-                else {
-                    chain.emitter.emit(routes.events.EventType.attestation, attestation);
-                    chain.emitter.emit(routes.events.EventType.singleAttestation, toElectraSingleAttestation(attestation, indexedAttestation.attestingIndices[0]));
-                }
+            // verdicts computed synchronously and returned immediately
+            for (const validationResult of validationResults) {
+                results.push(validationResult.err ? validationResult.err : null);
             }
             if (batchableBls) {
                 metrics?.gossipAttestation.attestationBatchHistogram.observe(attestationCount);
@@ -1041,6 +1103,47 @@ function getBatchHandlers(modules, options) {
             else {
                 metrics?.gossipAttestation.attestationNonBatchCount.inc(attestationCount);
             }
+            // Handler - deferred to next event loop so the validation result propagates first.
+            callInNextEventLoop(() => {
+                for (const [i, validationResult] of validationResults.entries()) {
+                    if (validationResult.err)
+                        continue;
+                    const { indexedAttestation, attDataRootHex, attestation, committeeIndex, validatorCommitteeIndex, committeeSize, subnet, } = validationResult.result;
+                    try {
+                        chain.validatorMonitor?.registerGossipUnaggregatedAttestation(gossipHandlerParams[i].seenTimestampSec, indexedAttestation);
+                        try {
+                            // Node may be subscribe to extra subnets (long-lived random subnets). For those, validate the messages
+                            // but don't add to attestation pool, to save CPU and RAM
+                            if (aggregatorTracker.shouldAggregate(subnet, indexedAttestation.data.slot)) {
+                                const insertOutcome = chain.attestationPool.add(committeeIndex, attestation, attDataRootHex, validatorCommitteeIndex, committeeSize);
+                                metrics?.opPool.attestationPool.gossipInsertOutcome.inc({ insertOutcome });
+                            }
+                        }
+                        catch (e) {
+                            logger.debug("Error adding gossip unaggregated attestation to pool", { subnet }, e);
+                        }
+                        // Separate boundary: a pool insertion error above must not skip the fork-choice vote
+                        if (!options.dontSendGossipAttestationsToForkchoice) {
+                            try {
+                                chain.forkChoice.onAttestation(indexedAttestation, attDataRootHex);
+                            }
+                            catch (e) {
+                                logger.debug("Error adding gossip unaggregated attestation to forkchoice", { subnet }, e);
+                            }
+                        }
+                        if (isForkPostElectra(fork)) {
+                            chain.emitter.emit(routes.events.EventType.singleAttestation, attestation);
+                        }
+                        else {
+                            chain.emitter.emit(routes.events.EventType.attestation, attestation);
+                            chain.emitter.emit(routes.events.EventType.singleAttestation, toElectraSingleAttestation(attestation, indexedAttestation.attestingIndices[0]));
+                        }
+                    }
+                    catch (e) {
+                        logger.debug("Error handling gossip attestation", { slot: indexedAttestation.data.slot, subnet }, e);
+                    }
+                }
+            });
             return results;
         },
     };