@lodestar/beacon-node 1.48.0-dev.0dbea0de1b → 1.48.0-dev.27e5bfcdb6

package/src/chain/archiveStore/utils/archiveBlocks.ts

@@ -8,6 +8,7 @@ import {Epoch, Slot} from "@lodestar/types";
 import {Logger, fromAsync, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/index.js";
 import {BlockArchiveBatchPutBinaryItem} from "../../../db/repositories/index.js";
+import {Metrics} from "../../../metrics/metrics.js";
 import {ensureDir, writeIfNotExist} from "../../../util/file.js";
 import {BlockRootHex} from "../../../util/sszBytes.js";
 import {LightClientServer} from "../../lightClient/index.js";
@@ -20,6 +21,16 @@ const BLOB_SIDECAR_BATCH_SIZE = 32;
 
 type BlockRootSlot = {slot: Slot; root: Uint8Array};
 
+/**
+ * Why a finalized-canonical block was imported after the attestation cutoff.
+ */
+export enum LateCanonicalBlockReason {
+  // received on time, but finished importing after the attestation cutoff
+  SlowImport = "slow_import",
+  // block reached this node late on the network thread
+  LateReceive = "late_receive",
+}
+
 /**
  * Persist orphaned block to disk
  */
@@ -54,6 +65,8 @@ export async function archiveBlocks(
   logger: Logger,
   finalizedCheckpoint: CheckpointWithHex,
   currentEpoch: Epoch,
+  metrics: Metrics | null,
+  isNodeSynced: boolean,
   archiveDataEpochs?: number,
   persistOrphanedBlocks?: boolean,
   persistOrphanedBlocksDir?: string
@@ -76,6 +89,29 @@ export async function archiveBlocks(
 
   const logCtx = {currentEpoch, finalizedEpoch: finalizedCheckpoint.epoch, finalizedRoot: finalizedCheckpoint.rootHex};
 
+  // `finalizedCanonicalBlocks` is newest -> oldest; its LAST element is the previous-finalized
+  // boundary block, already audited on the prior run, so exclude it to avoid double counting.
+  if (isNodeSynced && finalizedCanonicalBlocks.length > 1) {
+    const lateBlocks = finalizedCanonicalBlocks.slice(0, -1).filter((block) => !block.importedTimely);
+    if (lateBlocks.length > 0) {
+      let slowImport = 0;
+      for (const block of lateBlocks) {
+        // received late => late_receive; otherwise we had it on time but were slow to import => slow_import
+        const reason = block.timeliness ? LateCanonicalBlockReason.SlowImport : LateCanonicalBlockReason.LateReceive;
+        if (block.timeliness) slowImport++;
+        metrics?.importBlock.lateCanonicalBlock.inc({reason});
+      }
+      const lateSlots = lateBlocks.map((block) => block.slot).sort((a, b) => a - b);
+      logger.verbose("Late imported canonical blocks", {
+        ...logCtx,
+        count: lateBlocks.length,
+        slowImport,
+        lateReceive: lateBlocks.length - slowImport,
+        slotRange: prettyPrintIndices(lateSlots),
+      });
+    }
+  }
+
   if (finalizedCanonicalBlockRoots.length > 0) {
     const migratedSlots = await migrateBlocksFromHotToColdDb(db, logger, finalizedCanonicalBlockRoots);
     logger.verbose("Migrated blocks from hot DB to cold DB", {
@@ -298,7 +334,7 @@ async function migrateBlocksFromHotToColdDb(db: IBeaconDb, logger: Logger, block
     ]);
     for (const entry of canonicalBlockEntries) migratedSlots.push(entry.slot);
   }
-  // Ancestor walk is newest → oldest; sort ascending so `prettyPrintIndices` renders cleanly.
+  // Ancestor walk is newest to oldest; sort ascending so `prettyPrintIndices` renders cleanly.
   return migratedSlots.sort((a, b) => a - b);
 }
 
@@ -437,7 +473,7 @@ async function migrateDataColumnSidecarsFromHotToColdDb(
     await Promise.all(promises);
   }
 
-  // Ancestor walk is newest → oldest; sort ascending so `prettyPrintIndices` renders cleanly.
+  // Ancestor walk is newest to oldest; sort ascending so `prettyPrintIndices` renders cleanly.
   return migratedSlots.sort((a, b) => a - b);
 }
 
@@ -485,7 +521,7 @@ async function migrateExecutionPayloadEnvelopesFromHotToColdDb(
   ]);
 
   // Slots are ascending in hot-db key order — sort to guarantee `prettyPrintIndices` output is clean
-  // regardless of ancestor-walk order (newest → oldest).
+  // regardless of ancestor-walk order (newest to oldest).
   return envelopeEntries.map((entry) => entry.key).sort((a, b) => a - b);
 }
 

package/src/chain/blocks/importBlock.ts

@@ -85,7 +85,7 @@ export async function importBlock(
   const currentEpoch = computeEpochAtSlot(currentSlot);
   const blockEpoch = computeEpochAtSlot(blockSlot);
   const prevFinalizedEpoch = this.forkChoice.getFinalizedCheckpoint().epoch;
-  const blockDelaySec =
+  const receiveDelaySec =
     fullyVerifiedBlock.seenTimestampSec - computeTimeAtSlot(this.config, blockSlot, postState.genesisTime);
   const recvToValLatency = Date.now() / 1000 - (opts.seenTimestampSec ?? Date.now() / 1000);
   const fork = this.config.getForkSeq(blockSlot);
@@ -122,10 +122,12 @@ export async function importBlock(
     executionStatus = parentBlock.executionStatus;
   }
 
+  const importDelaySec = this.clock.secFromSlot(blockSlot);
   const blockSummary = this.forkChoice.onBlock(
     block.message,
     postState,
-    blockDelaySec,
+    receiveDelaySec,
+    importDelaySec,
     currentSlot,
     executionStatus,
     dataAvailabilityStatus

package/src/chain/forkChoice/index.ts

@@ -135,6 +135,7 @@ export function initializeForkChoiceFromFinalizedState(
         blockRoot: toRootHex(checkpoint.root),
         timeliness: true, // Optimistically assume is timely
         ptcTimeliness: true, // Spec: block_timeliness for anchor = [True, True]
+        importedTimely: true, // Optimistically assume is timely
         proposerIndex: blockHeader.proposerIndex,
 
         justifiedEpoch: justifiedCheckpoint.epoch,
@@ -233,6 +234,7 @@ export function initializeForkChoiceFromUnfinalizedState(
     targetRoot: headRoot,
     timeliness: true, // Optimistically assume is timely
     ptcTimeliness: true, // Spec: block_timeliness for anchor = [True, True]
+    importedTimely: true, // Optimistically assume is timely
     proposerIndex: blockHeader.proposerIndex,
 
     justifiedEpoch: justifiedCheckpoint.epoch,

package/src/chain/validation/payloadAttestationMessage.ts

@@ -7,7 +7,6 @@ import {RootHex, gloas, ssz} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
 import {GossipAction, PayloadAttestationError, PayloadAttestationErrorCode} from "../errors/index.js";
 import {IBeaconChain} from "../index.js";
-import {RegenCaller} from "../regen/index.js";
 
 export type PayloadAttestationValidationResult = {
   attDataRootHex: RootHex;
@@ -82,15 +81,15 @@ async function validatePayloadAttestationMessage(
   // TODO GLOAS: implement this. Technically if we cannot get proto block from fork choice,
   // it is possible that the block didn't pass the validation
 
-  // Use the referenced block's branch state for the PTC committee check
-  const state = await chain.regen
-    .getBlockSlotState(block, data.slot, {dontTransferCache: true}, RegenCaller.validateGossipPayloadAttestationMessage)
-    .catch(() => {
-      throw new PayloadAttestationError(GossipAction.IGNORE, {
-        code: PayloadAttestationErrorCode.UNKNOWN_BLOCK_ROOT,
-        blockRoot: toRootHex(data.beaconBlockRoot),
-      });
+  // block.slot === data.slot is enforced above, so use the block's post-state directly to avoid
+  // getting through regen queue
+  const state = chain.regen.getStateSync(block.stateRoot);
+  if (state == null) {
+    throw new PayloadAttestationError(GossipAction.IGNORE, {
+      code: PayloadAttestationErrorCode.UNKNOWN_BLOCK_ROOT,
+      blockRoot: toRootHex(data.beaconBlockRoot),
     });
+  }
 
   if (!isStatePostGloas(state)) {
     throw new Error(`Expected gloas+ state for payload attestation validation, got fork=${state.forkName}`);

package/src/metrics/metrics/lodestar.ts

@@ -2,6 +2,7 @@
 import {NotReorgedReason} from "@lodestar/fork-choice";
 import {ArchiveStoreTask} from "../../chain/archiveStore/archiveStore.js";
 import {FrequencyStateArchiveStep} from "../../chain/archiveStore/strategies/frequencyStateArchiveStrategy.js";
+import type {LateCanonicalBlockReason} from "../../chain/archiveStore/utils/archiveBlocks.js";
 import {BlockInputSource} from "../../chain/blocks/blockInput/index.js";
 import {PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
 import {
@@ -1047,6 +1048,11 @@ export function createLodestarMetrics(
         name: "lodestar_import_block_set_head_after_cutoff_total",
         help: "Total times an imported block is set as head after ATTESTATION_DUE_BPS of the slot",
       }),
+      lateCanonicalBlock: register.counter<{reason: LateCanonicalBlockReason}>({
+        name: "lodestar_import_block_late_canonical_total",
+        help: "Total finalized-canonical blocks this node imported after the attestation cutoff; reason distinguishes this node's processing lag (slow_import) from late reception (late_receive)",
+        labelNames: ["reason"],
+      }),
       bySource: register.gauge<{source: BlockInputSource}>({
         name: "lodestar_import_block_by_source_total",
         help: "Total number of imported blocks by source",

package/src/network/processor/gossipHandlers.ts

@@ -1268,16 +1268,22 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       const delaySec = chain.clock.secFromSlot(payloadAttestationMessage.data.slot, seenTimestampSec);
       metrics?.gossipPayloadAttestationMessage.elapsedTimeTillReceived.observe({source: OpSource.gossip}, delaySec);
 
-      try {
-        const insertOutcome = chain.payloadAttestationPool.add(
-          payloadAttestationMessage,
-          validationResult.attDataRootHex,
-          validationResult.validatorCommitteeIndices
-        );
-        metrics?.opPool.payloadAttestationPool.gossipInsertOutcome.inc({insertOutcome});
-      } catch (e) {
-        logger.error("Error adding to payloadAttestation pool", {}, e as Error);
+      const nextSlotProposer = chain.getHeadState().getBeaconProposer(payloadAttestationMessage.data.slot + 1);
+      const isNextSlotProposer = chain.beaconProposerCache.get(nextSlotProposer) !== undefined;
+
+      if (isNextSlotProposer) {
+        try {
+          const insertOutcome = chain.payloadAttestationPool.add(
+            payloadAttestationMessage,
+            validationResult.attDataRootHex,
+            validationResult.validatorCommitteeIndices
+          );
+          metrics?.opPool.payloadAttestationPool.gossipInsertOutcome.inc({insertOutcome});
+        } catch (e) {
+          logger.debug("Error adding to payloadAttestation pool", {}, e as Error);
+        }
       }
+
       chain.forkChoice.notifyPtcMessages(
         toRootHex(payloadAttestationMessage.data.beaconBlockRoot),
         payloadAttestationMessage.data.slot,