@lodestar/beacon-node 1.43.0-dev.38479366cc → 1.43.0-dev.3bcc6d0ad5

package/src/sync/range/batch.ts

@@ -1,7 +1,7 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkName, isForkPostDeneb, isForkPostFulu, isForkPostGloas} from "@lodestar/params";
 import {Epoch, RootHex, Slot, phase0} from "@lodestar/types";
-import {LodestarError} from "@lodestar/utils";
+import {LodestarError, prettyPrintIndices} from "@lodestar/utils";
 import {isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
 import {isDaOutOfRange} from "../../chain/blocks/blockInput/utils.js";
@@ -79,10 +79,36 @@ export type BatchState =
     };
 
 export type BatchMetadata = {
+  // Batch-level slot window (always present)
   startEpoch: Epoch;
+  startSlot: Slot;
+  count: number;
   status: BatchStatus;
+
+  // Per-type outstanding request shapes; only present when that sub-request exists.
+  // Format: "startSlot=<n>,count=<n>" (plus ",cols=<indices>" for columns).
+  blocksReq?: string;
+  blobsReq?: string;
+  columnsReq?: string;
+  envelopesReq?: string;
+
+  // Retry counters
+  downloadAttempts: number;
+  processingAttempts: number;
+
+  // Cumulative peer attribution for failed attempts (only present when non-empty)
+  failedDownloadPeers?: string;
+  failedProcessingPeers?: string;
 };
 
+function formatRangeReq(req: {startSlot: Slot; count: number}): string {
+  return `startSlot=${req.startSlot},count=${req.count}`;
+}
+
+function formatColumnsReq(req: {startSlot: Slot; count: number; columns: number[]}): string {
+  return `startSlot=${req.startSlot},count=${req.count},cols=${prettyPrintIndices(req.columns)}`;
+}
+
 /**
  * Batches are downloaded at the first block of the epoch.
  *
@@ -286,7 +312,26 @@ export class Batch {
   }
 
   getMetadata(): BatchMetadata {
-    return {startEpoch: this.startEpoch, status: this.state.status};
+    const {blocksRequest, blobsRequest, columnsRequest, envelopesRequest} = this.requests;
+    const failedProcessingPeerList = this.failedProcessingAttempts.flatMap((a) => a.peers);
+    return {
+      startEpoch: this.startEpoch,
+      startSlot: this.startSlot,
+      count: this.count,
+      status: this.state.status,
+      ...(blocksRequest && {blocksReq: formatRangeReq(blocksRequest)}),
+      ...(blobsRequest && {blobsReq: formatRangeReq(blobsRequest)}),
+      ...(columnsRequest && {columnsReq: formatColumnsReq(columnsRequest)}),
+      ...(envelopesRequest && {envelopesReq: formatRangeReq(envelopesRequest)}),
+      downloadAttempts: this.failedDownloadAttempts.length,
+      processingAttempts: this.failedProcessingAttempts.length,
+      ...(this.failedDownloadAttempts.length > 0 && {
+        failedDownloadPeers: this.failedDownloadAttempts.join(","),
+      }),
+      ...(failedProcessingPeerList.length > 0 && {
+        failedProcessingPeers: failedProcessingPeerList.join(","),
+      }),
+    };
   }
 
   getBlocks(): IBlockInput[] {
@@ -383,7 +428,11 @@ export class Batch {
   /**
    * AwaitingProcessing -> Processing
    */
-  startProcessing(): {blocks: IBlockInput[]; payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null} {
+  startProcessing(): {
+    blocks: IBlockInput[];
+    payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null;
+    peers: PeerIdStr[];
+  } {
     if (this.state.status !== BatchStatus.AwaitingProcessing) {
       throw new BatchError(this.wrongStatusErrorType(BatchStatus.AwaitingProcessing));
     }
@@ -396,7 +445,7 @@ export class Batch {
     const peers = this.goodPeers;
     this.goodPeers = [];
     this.state = {status: BatchStatus.Processing, blocks, payloadEnvelopes, attempt: {peers, hash}};
-    return {blocks, payloadEnvelopes};
+    return {blocks, payloadEnvelopes, peers};
   }
 
   /**
@@ -479,7 +528,7 @@ export class Batch {
 
   /** Helper to construct typed BatchError. Stack traces are correct as the error is thrown above */
   private errorType(type: BatchErrorType): BatchErrorType & BatchErrorMetadata {
-    return {...type, ...this.getMetadata()};
+    return {...type, startEpoch: this.startEpoch, status: this.state.status};
   }
 
   private wrongStatusErrorType(expectedStatus: BatchStatus): BatchErrorType & BatchErrorMetadata {

package/src/sync/range/chain.ts

@@ -1,6 +1,6 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {Epoch, Root, Slot} from "@lodestar/types";
-import {ErrorAborted, LodestarError, Logger, toRootHex} from "@lodestar/utils";
+import {ErrorAborted, LodestarError, Logger, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../../chain/blocks/blockInput/blockInput.js";
 import {BlockInputErrorCode} from "../../chain/blocks/blockInput/errors.js";
 import {IBlockInput} from "../../chain/blocks/blockInput/types.js";
@@ -222,12 +222,14 @@ export class SyncChain {
    */
   stopSyncing(): void {
     this.status = SyncChainStatus.Stopped;
+    this.logger.debug("SyncChain stopSyncing", {id: this.logId});
   }
 
   /**
    * Permanently remove this chain. Throws the main AsyncIterable
    */
   remove(): void {
+    this.logger.debug("SyncChain remove", {id: this.logId});
     this.batchProcessor.end(new ErrorAborted("SyncChain"));
   }
 
@@ -534,7 +536,7 @@ export class SyncChain {
             this.metrics?.syncRange.downloadByRange.warn.inc({client: peer.client, code: warning.type.code});
             this.logger.debug(
               "Batch downloaded with warning",
-              {id: this.logId, epoch: batch.startEpoch, ...logMeta, peer: prettyPrintPeerIdStr(peer.peerId)},
+              {id: this.logId, ...batch.getMetadata(), ...logMeta, peer: prettyPrintPeerIdStr(peer.peerId)},
               warning
             );
           }
@@ -560,10 +562,17 @@ export class SyncChain {
           // the flow will continue to call triggerBatchDownloader() below
         }
 
+        const blockSlots = downloadSuccessOutput.blocks.map((b) => b.slot);
+        const envelopeSlots = downloadSuccessOutput.payloadEnvelopes
+          ? Array.from(downloadSuccessOutput.payloadEnvelopes.keys())
+          : null;
+
         this.logger.debug(logMessage, {
           id: this.logId,
-          epoch: batch.startEpoch,
+          ...batch.getMetadata(),
           ...logMeta,
+          blockSlots: prettyPrintIndices(blockSlots),
+          ...(envelopeSlots ? {envelopeSlots: prettyPrintIndices(envelopeSlots)} : {}),
           peer: prettyPrintPeerIdStr(peer.peerId),
         });
       }
@@ -586,13 +595,24 @@ export class SyncChain {
    * Sends `batch` to the processor. Note: batch may be empty
    */
   private async processBatch(batch: Batch): Promise<void> {
-    const {blocks, payloadEnvelopes} = batch.startProcessing();
+    const {blocks, payloadEnvelopes, peers} = batch.startProcessing();
+
+    const logCtx = {
+      id: this.logId,
+      ...batch.getMetadata(),
+      blockCount: blocks.length,
+      blockSlots: prettyPrintIndices(blocks.map((b) => b.slot)),
+      ...(payloadEnvelopes ? {envelopeSlots: prettyPrintIndices(Array.from(payloadEnvelopes.keys()))} : {}),
+      peers: peers.map(prettyPrintPeerIdStr).join(","),
+    };
+    this.logger.verbose("Processing batch", logCtx);
 
     // wrapError ensures to never call both batch success() and batch error()
     const res = await wrapError(this.processChainSegment(blocks, payloadEnvelopes, this.syncType));
 
     if (!res.err) {
       batch.processingSuccess();
+      this.logger.verbose("Processed batch", {...logCtx, ...batch.getMetadata()});
 
       // If the processed batch is not empty, validate previous AwaitingValidation blocks.
       if (blocks.length > 0) {
@@ -602,7 +622,7 @@ export class SyncChain {
       // Potentially process next AwaitingProcessing batch
       this.triggerBatchProcessor();
     } else {
-      this.logger.verbose("Batch process error", {id: this.logId, ...batch.getMetadata()}, res.err);
+      this.logger.verbose("Batch process error", logCtx, res.err);
       batch.processingError(res.err); // Throws after MAX_BATCH_PROCESSING_ATTEMPTS
 
       // At least one block was successfully verified and imported, so we can be sure all

package/src/sync/unknownBlock.ts

@@ -145,7 +145,6 @@ export class BlockInputSync {
       this.chain.emitter.on(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
       this.chain.emitter.on(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
       this.chain.emitter.on(ChainEvent.blockUnknownParent, this.onUnknownParent);
-      this.chain.emitter.on(ChainEvent.envelopeUnknownBlock, this.onEnvelopeUnknownBlock);
       this.chain.emitter.on(routes.events.EventType.block, this.onBlockImported);
       this.chain.emitter.on(routes.events.EventType.executionPayload, this.onPayloadImported);
       this.network.events.on(NetworkEvent.peerConnected, this.onPeerConnected);
@@ -161,7 +160,6 @@ export class BlockInputSync {
     this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
     this.chain.emitter.off(ChainEvent.incompletePayloadEnvelope, this.onIncompletePayloadEnvelope);
     this.chain.emitter.off(ChainEvent.blockUnknownParent, this.onUnknownParent);
-    this.chain.emitter.off(ChainEvent.envelopeUnknownBlock, this.onEnvelopeUnknownBlock);
     this.chain.emitter.off(routes.events.EventType.block, this.onBlockImported);
     this.chain.emitter.off(routes.events.EventType.executionPayload, this.onPayloadImported);
     this.network.events.off(NetworkEvent.peerConnected, this.onPeerConnected);
@@ -263,19 +261,6 @@ export class BlockInputSync {
     }
   };
 
-  private onEnvelopeUnknownBlock = (data: ChainEventData[ChainEvent.envelopeUnknownBlock]): void => {
-    try {
-      const blockRootHex = toRootHex(data.envelope.message.beaconBlockRoot);
-      this.addByRootHex(blockRootHex, data.peer);
-      this.addByPayloadEnvelope(data.envelope, data.peer);
-      this.triggerUnknownBlockSearch();
-      this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_DATA});
-      this.metrics?.blockInputSync.source.inc({source: data.source});
-    } catch (e) {
-      this.logger.debug("Error handling envelopeUnknownBlock event", {}, e as Error);
-    }
-  };
-
   private onBlockImported = (): void => {
     if (this.pendingPayloads.size > 0) {
       this.triggerUnknownBlockSearch();
@@ -381,41 +366,6 @@ export class BlockInputSync {
     return added;
   };
 
-  private addByPayloadEnvelope = (envelope: gloas.SignedExecutionPayloadEnvelope, peerIdStr?: PeerIdStr): void => {
-    const rootHex = toRootHex(envelope.message.beaconBlockRoot);
-    const existingPendingPayload = this.pendingPayloads.get(rootHex);
-    let pendingPayload = this.pendingPayloads.get(rootHex);
-    if (!pendingPayload || !isPendingPayloadEnvelope(pendingPayload)) {
-      pendingPayload = {
-        status: PendingPayloadInputStatus.waitingForBlock,
-        envelope,
-        peerIdStrings: new Set(existingPendingPayload?.peerIdStrings ?? []),
-        timeAddedSec: existingPendingPayload?.timeAddedSec ?? Date.now() / 1000,
-      };
-      this.pendingPayloads.set(rootHex, pendingPayload);
-
-      this.logger.verbose("Added payload envelope to BlockInputSync.pendingPayloads", {
-        slot: envelope.message.payload.slotNumber,
-        root: rootHex,
-      });
-    } else {
-      this.logger.debug("Overwriting pending payload envelope for root already waiting for block", {
-        slot: envelope.message.payload.slotNumber,
-        root: rootHex,
-      });
-      pendingPayload.envelope = envelope;
-    }
-
-    if (peerIdStr) {
-      pendingPayload.peerIdStrings.add(peerIdStr);
-    }
-
-    const prunedItemCount = pruneSetToMax(this.pendingPayloads, this.maxPendingBlocks);
-    if (prunedItemCount > 0) {
-      this.logger.verbose(`Pruned ${prunedItemCount} items from BlockInputSync.pendingPayloads`);
-    }
-  };
-
   private addByPayloadInput = (
     payloadInput: PayloadEnvelopeInput,
     peerIdStr?: PeerIdStr,

package/src/sync/utils/downloadByRange.ts

@@ -185,9 +185,24 @@ export function cacheByRangeResponses({
     }
   }
 
-  // Build payloadEnvelopes map for gloas: start from existing (partial download) state.
-  // The entries are wrappers around (block + envelope + sampled columns) and also seeded into
-  // seenPayloadEnvelopeInputCache so importBlock can find them without creating a duplicate.
+  // Seed seenPayloadEnvelopeInputCache for every gloas block in the batch, regardless of whether
+  // the peer returned its envelope. Without this, a block returned without its envelope would be
+  // imported with no cache entry, and later payload-by-root sync would throw
+  // "Missing PayloadEnvelopeInput for known block" (see issue #9306).
+  for (const blockInput of updatedBatchBlocks.values()) {
+    if (!blockInput.hasBlock() || !isForkPostGloas(blockInput.forkName)) continue;
+    seenPayloadEnvelopeInputCache.add({
+      blockRootHex: blockInput.blockRootHex,
+      block: blockInput.getBlock() as SignedBeaconBlock<ForkPostGloas>,
+      forkName: blockInput.forkName,
+      sampledColumns: custodyConfig.sampledColumns,
+      custodyColumns: custodyConfig.custodyColumns,
+      timeCreatedSec: seenTimestampSec,
+    });
+  }
+
+  // Attach envelopes to entries whose envelope was returned by the peer. The returned
+  // payloadEnvelopes map only contains entries with envelopes ready for importExecutionPayload.
   let payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null = null;
   if (downloadedPayloadEnvelopes !== null) {
     payloadEnvelopes = new Map(existingPayloadEnvelopes ?? []);
@@ -198,20 +213,11 @@ export function cacheByRangeResponses({
         // No block to pair this envelope with; drop silently
         continue;
       }
-      const {blockRootHex} = blockInput;
 
-      // Reuse any existing PayloadEnvelopeInput (e.g. gossip arrived first) to avoid
-      // duplicate cache entries. If missing, create a fresh one from the block's bid.
-      let payloadInput = seenPayloadEnvelopeInputCache.get(blockRootHex);
+      const payloadInput = seenPayloadEnvelopeInputCache.get(blockInput.blockRootHex);
       if (payloadInput === undefined) {
-        payloadInput = seenPayloadEnvelopeInputCache.add({
-          blockRootHex,
-          block: blockInput.getBlock() as SignedBeaconBlock<ForkPostGloas>,
-          forkName: blockInput.forkName,
-          sampledColumns: custodyConfig.sampledColumns,
-          custodyColumns: custodyConfig.custodyColumns,
-          timeCreatedSec: seenTimestampSec,
-        });
+        // Unreachable given the loop above seeded an entry for every gloas block in the batch.
+        continue;
       }
 
       if (!payloadInput.hasPayloadEnvelope()) {

package/src/sync/utils/downloadByRoot.ts

@@ -3,6 +3,7 @@ import {ChainForkConfig} from "@lodestar/config";
 import {
   ForkPostDeneb,
   ForkPostFulu,
+  ForkPostGloas,
   ForkPreFulu,
   isForkPostDeneb,
   isForkPostFulu,
@@ -114,6 +115,17 @@ export async function downloadByRoot({
     });
   }
 
+  if (isForkPostGloas(blockInput.forkName)) {
+    chain.seenPayloadEnvelopeInputCache.add({
+      blockRootHex: rootHex,
+      block: blockInput.getBlock() as SignedBeaconBlock<ForkPostGloas>,
+      forkName: blockInput.forkName,
+      sampledColumns: chain.custodyConfig.sampledColumns,
+      custodyColumns: chain.custodyConfig.custodyColumns,
+      timeCreatedSec: Date.now() / 1000,
+    });
+  }
+
   const hasAllDataPreDownload = blockInput.hasBlockAndAllData();
 
   if (isBlockInputBlobs(blockInput) && !hasAllDataPreDownload) {