@lodestar/beacon-node 1.49.0-dev.fa4ae94ccc → 1.49.0-rc.0

package/src/sync/unknownBlock.ts

@@ -227,8 +227,11 @@ export class BlockInputSync {
   private onUnknownBlockRoot = (data: ChainEventData[ChainEvent.unknownBlockRoot]): void => {
     try {
       const isNewRoot = this.addByRootHex(data.rootHex, data.peer);
-      if (isNewRoot) {
+      // A retained entry (an earlier download failed, #10018) is not new but still needs a retry pass.
+      if (this.pendingBlocks.get(data.rootHex)?.status === PendingBlockInputStatus.pending) {
         this.triggerUnknownBlockSearch();
+      }
+      if (isNewRoot) {
         this.metrics?.blockInputSync.requests.inc({type: PendingBlockType.UNKNOWN_BLOCK_ROOT});
         this.metrics?.blockInputSync.source.inc({source: data.source});
       }
@@ -393,12 +396,21 @@ export class BlockInputSync {
       }
     }
 
-    let deletedBlocks = 0;
+    let belowFinalizedBlocks = 0;
+    let agedOutBlocks = 0;
     for (const [rootHex, block] of this.pendingBlocks) {
       const slot = getBlockInputSyncCacheItemSlot(block);
-      if (typeof slot === "number" && slot < finalizedSlot) {
+      if (typeof slot !== "number") {
+        // we may not be able to download a root, hence there is no slot for it
+        if (nowSec - block.timeAddedSec > maxSecInCache) {
+          this.pendingBlocks.delete(rootHex);
+          agedOutBlocks++;
+        }
+        continue;
+      }
+      if (slot < finalizedSlot) {
         this.pendingBlocks.delete(rootHex);
-        deletedBlocks++;
+        belowFinalizedBlocks++;
       }
     }
 
@@ -411,16 +423,20 @@ export class BlockInputSync {
     if (agedOutPayloads > 0) {
       this.metrics?.blockInputSync.removedPayloads.inc({reason: DroppedItemReason.agedOut}, agedOutPayloads);
     }
-    if (deletedBlocks > 0) {
-      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.belowFinalized}, deletedBlocks);
+    if (belowFinalizedBlocks > 0) {
+      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.belowFinalized}, belowFinalizedBlocks);
+    }
+    if (agedOutBlocks > 0) {
+      this.metrics?.blockInputSync.removedBlocks.inc({reason: DroppedItemReason.agedOut}, agedOutBlocks);
     }
 
-    this.logger.debug("BlockInputSync.pruneFinalized dropped pre-finalized pending items", {
+    this.logger.debug("BlockInputSync.pruneFinalized dropped stale pending items", {
       finalizedSlot,
       finalizedRoot: checkpoint.rootHex,
       belowFinalizedPayloads,
       agedOutPayloads,
-      deletedBlocks,
+      belowFinalizedBlocks,
+      agedOutBlocks,
       unresolvedSlotPayloads,
     });
   };
@@ -973,7 +989,12 @@ export class BlockInputSync {
     if (!res.err) {
       this.metrics?.blockInputSync.downloadedBlocksSuccess.inc();
       const pending = res.result;
-      this.pendingBlocks.set(pending.blockInput.blockRootHex, pending);
+      // pruning may have deleted this entry while we were awaiting the network, do not resurrect it via the set below
+      if (!this.pendingBlocks.has(rootHex)) {
+        this.logger.verbose("Dropping downloaded block, entry pruned during fetch", logCtx);
+        return;
+      }
+      this.pendingBlocks.set(rootHex, pending);
       const blockSlot = pending.blockInput.slot;
       const finalizedSlot = this.chain.forkChoice.getFinalizedBlock().slot;
       const delaySec = Date.now() / 1000 - computeTimeAtSlot(this.config, blockSlot, this.chain.genesisTime);
@@ -1035,8 +1056,11 @@ export class BlockInputSync {
       }
 
       this.metrics?.blockInputSync.downloadedBlocksError.inc();
-      this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
-      this.removeAndDownScoreAllDescendants(block, DroppedItemReason.unavailable);
+      this.logger.debug("Failed to download unknown block root, retained for retry", logCtx, res.err);
+      const pendingBlock = this.pendingBlocks.get(rootHex);
+      if (pendingBlock && pendingBlock.status === PendingBlockInputStatus.fetching) {
+        pendingBlock.status = PendingBlockInputStatus.pending;
+      }
     }
   }
 
@@ -1303,7 +1327,7 @@ export class BlockInputSync {
     }
 
     this.metrics?.blockInputSync.downloadedPayloadsResult.inc({result: DownloadResult.Failed});
-    this.logger.debug("Ignoring unknown payload root after failed download", logCtx, res.err);
+    this.logger.debug("Failed to download unknown payload root, retained for retry", logCtx, res.err);
     if (!isPendingPayloadEnvelope(payload)) {
       payload.status = PendingPayloadInputStatus.pending;
     }
@@ -1763,7 +1787,11 @@ export class BlockInputSync {
         this.peerBalancer.onRequestCompleted(peerId);
       }
 
-      this.pendingBlocks.set(getBlockInputSyncCacheItemRootHex(cacheItem), cacheItem);
+      const cacheItemRootHex = getBlockInputSyncCacheItemRootHex(cacheItem);
+      // pruning may have dropped it mid-fetch
+      if (this.pendingBlocks.has(cacheItemRootHex)) {
+        this.pendingBlocks.set(cacheItemRootHex, cacheItem);
+      }
 
       if (cacheItem.status === PendingBlockInputStatus.downloaded) {
         // download was successful, no need to go with another peer, return

package/src/util/execution.ts

@@ -2,12 +2,18 @@ import {routes} from "@lodestar/api";
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkPostFulu, ForkPreFulu} from "@lodestar/params";
 import {signedBlockToSignedHeader} from "@lodestar/state-transition";
-import {DataColumnSidecar, SignedBeaconBlock, deneb, isGloasDataColumnSidecar} from "@lodestar/types";
-import {fromHex, toHex} from "@lodestar/utils";
+import {DataColumnSidecar, SignedBeaconBlock, Slot, deneb, gloas, isGloasDataColumnSidecar, ssz} from "@lodestar/types";
+import {Logger, fromHex, toHex, toRootHex} from "@lodestar/utils";
 import {isBlockInputBlobs, isBlockInputColumns} from "../chain/blocks/blockInput/blockInput.js";
 import {BlockInputSource, IBlockInput} from "../chain/blocks/blockInput/types.js";
 import {PayloadEnvelopeInput, PayloadEnvelopeInputSource} from "../chain/blocks/payloadEnvelopeInput/index.js";
 import {ChainEvent, ChainEventEmitter} from "../chain/emitter.js";
+import {
+  EnvelopeReconstructionError,
+  EnvelopeReconstructionErrorCode,
+} from "../chain/errors/envelopeReconstructionError.js";
+import {IBeaconDb} from "../db/index.js";
+import {ArchivedEnvelope, decodeArchivedEnvelope} from "../db/repositories/index.js";
 import {IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/index.js";
 import {computePreFuluKzgCommitmentsInclusionProof} from "./blobs.js";
@@ -17,6 +23,7 @@ import {
   getDataColumnSidecarsFromColumnSidecar,
   getGloasDataColumnSidecars,
 } from "./dataColumns.js";
+import {signedHeaderEnvelopeToFull} from "./headerEnvelope.js";
 
 export enum DataColumnEngineResult {
   PreFulu = "pre_fulu",
@@ -251,3 +258,169 @@ export async function getDataColumnSidecarsFromExecution(
   );
   return DataColumnEngineResult.SuccessResolved;
 }
+
+/** engine_getPayloadBodiesByHashV2: ELs MUST support at least 32 hashes per request. */
+const MAX_BODIES_REQUEST = 32;
+
+type SlotEnvelopeBytes = {slot: Slot; envelopeBytes: Uint8Array};
+
+type RangeEntry = ArchivedEnvelope & {slot: Slot};
+
+/** What a body root mismatch means on a given serving path */
+export type ReconstructMismatchPolicy = "throw" | "omit";
+
+export type RebuildMiss =
+  /** EL does not have the block, or has pruned its block access list */
+  | {slot: Slot; reason: "unavailable"}
+  /** An EL body does not hash to its stored root (local inconsistency) */
+  | {slot: Slot; reason: "mismatch"; error: EnvelopeReconstructionError};
+
+/**
+ * Stream finalized envelopes over [startSlot, endSlot) as serialized bytes, rebuilding header
+ * entries from EL bodies 32 per round-trip.
+ *
+ * Ends at the first entry that cannot be served by throwing RANGE_UNSERVABLE with that slot: the
+ * by-range spec inherits BeaconBlocksByRange v2 semantics (consecutive, MAY be short), and a hole
+ * looks like a lying peer to one that already holds the blocks. Entries are attempted regardless of
+ * age; the EL's block access list retention is the floor, not MIN_EPOCHS_FOR_BLOCK_REQUESTS.
+ *
+ * Throws ENGINE_UNAVAILABLE if the EL call fails. Either may surface after envelopes were yielded.
+ */
+export async function* reconstructExecutionPayloadEnvelopesByRange(
+  db: IBeaconDb,
+  executionEngine: IExecutionEngine,
+  logger: Logger,
+  metrics: Metrics | null,
+  startSlot: Slot,
+  endSlot: Slot
+): AsyncIterable<SlotEnvelopeBytes> {
+  const archive = db.executionPayloadEnvelopeArchive;
+  let batch: RangeEntry[] = [];
+
+  for await (const {key, value: bytes} of archive.binaryEntriesStream({gte: startSlot, lt: endSlot})) {
+    batch.push({slot: archive.decodeKey(key), ...decodeArchivedEnvelope(bytes)});
+    if (batch.length === MAX_BODIES_REQUEST) {
+      yield* reconstructBatch(executionEngine, logger, metrics, batch);
+      batch = [];
+    }
+  }
+  if (batch.length > 0) {
+    yield* reconstructBatch(executionEngine, logger, metrics, batch);
+  }
+}
+
+/**
+ * Rebuild a range batch, yielding what precedes the first unservable entry before throwing
+ * RANGE_UNSERVABLE for it
+ */
+async function* reconstructBatch(
+  executionEngine: IExecutionEngine,
+  logger: Logger,
+  metrics: Metrics | null,
+  batch: RangeEntry[]
+): AsyncIterable<SlotEnvelopeBytes> {
+  const headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[] = [];
+  for (const entry of batch) {
+    if (entry.headerEnvelope !== undefined) headerEnvelopes.push(entry.headerEnvelope);
+  }
+  const rebuilt = await reconstructEnvelopesBatch(executionEngine, metrics, headerEnvelopes);
+
+  let rebuiltIdx = 0;
+  for (const entry of batch) {
+    if (entry.envelopeBytes !== undefined) {
+      yield {slot: entry.slot, envelopeBytes: entry.envelopeBytes};
+      continue;
+    }
+    const result = rebuilt[rebuiltIdx++];
+    if (isRebuildMiss(result)) {
+      // Peer-triggered, so debug: a persistent local mismatch would otherwise log on every request
+      if (result.reason === "mismatch") {
+        logger.debug("Archived envelope failed body root check against EL bodies", {slot: entry.slot}, result.error);
+      } else {
+        logger.debug("EL cannot serve bodies for archived envelope, ending range", {slot: entry.slot});
+      }
+      throw new EnvelopeReconstructionError(
+        {code: EnvelopeReconstructionErrorCode.RANGE_UNSERVABLE, slot: entry.slot},
+        `archived envelope range unservable from slot=${entry.slot}`
+      );
+    }
+    yield {slot: entry.slot, envelopeBytes: ssz.gloas.SignedExecutionPayloadEnvelope.serialize(result)};
+  }
+}
+
+/**
+ * Rebuild header envelopes from EL bodies, 32 per round-trip. Aligned with the input, with a
+ * {@link RebuildMiss} where the envelope could not be rebuilt; the caller decides what a miss means
+ * on its path. Throws ENGINE_UNAVAILABLE only.
+ */
+export async function reconstructExecutionPayloadEnvelopes(
+  executionEngine: IExecutionEngine,
+  metrics: Metrics | null,
+  headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[]
+): Promise<(gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[]> {
+  const out: (gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[] = [];
+  for (let i = 0; i < headerEnvelopes.length; i += MAX_BODIES_REQUEST) {
+    out.push(
+      ...(await reconstructEnvelopesBatch(executionEngine, metrics, headerEnvelopes.slice(i, i + MAX_BODIES_REQUEST)))
+    );
+  }
+  return out;
+}
+
+export function isRebuildMiss(result: gloas.SignedExecutionPayloadEnvelope | RebuildMiss): result is RebuildMiss {
+  return "reason" in result;
+}
+
+/**
+ * One EL round-trip. Aligned with the input; never throws per envelope, only ENGINE_UNAVAILABLE.
+ * Every serving path (by-range, by-root, REST) comes through here, so the outcome metrics are
+ * incremented once, in this function.
+ */
+async function reconstructEnvelopesBatch(
+  executionEngine: IExecutionEngine,
+  metrics: Metrics | null,
+  headerEnvelopes: gloas.SignedExecutionPayloadHeaderEnvelope[]
+): Promise<(gloas.SignedExecutionPayloadEnvelope | RebuildMiss)[]> {
+  if (headerEnvelopes.length === 0) return [];
+  const hashes = headerEnvelopes.map((headerEnvelope) => toRootHex(headerEnvelope.message.payloadHeader.blockHash));
+
+  let bodies: Awaited<ReturnType<IExecutionEngine["getPayloadBodiesByHashV2"]>>;
+  try {
+    bodies = await executionEngine.getPayloadBodiesByHashV2(hashes);
+  } catch (e) {
+    metrics?.payloadEnvelopeReconstruction.engineErrors.inc();
+    throw new EnvelopeReconstructionError(
+      {code: EnvelopeReconstructionErrorCode.ENGINE_UNAVAILABLE},
+      `engine_getPayloadBodiesByHashV2 failed: ${(e as Error).message}`
+    );
+  }
+
+  return headerEnvelopes.map((headerEnvelope, i) => {
+    const slot = headerEnvelope.message.payloadHeader.slotNumber;
+    const body = bodies[i];
+    // A zero-length block access list cannot be valid, RLP encodes an empty list as 0xc0
+    if (body == null || body.withdrawals == null || body.blockAccessList == null || body.blockAccessList.length === 0) {
+      metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "unavailable"});
+      return {slot, reason: "unavailable"};
+    }
+    try {
+      const envelope = signedHeaderEnvelopeToFull(headerEnvelope, {
+        transactions: body.transactions,
+        withdrawals: body.withdrawals,
+        blockAccessList: body.blockAccessList,
+      });
+      metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "ok"});
+      return envelope;
+    } catch (e) {
+      if (
+        e instanceof EnvelopeReconstructionError &&
+        e.type.code === EnvelopeReconstructionErrorCode.BODY_ROOT_MISMATCH
+      ) {
+        metrics?.payloadEnvelopeReconstruction.envelopes.inc({result: "mismatch"});
+        metrics?.payloadEnvelopeReconstruction.mismatchByField.inc({field: e.type.field});
+        return {slot, reason: "mismatch", error: e};
+      }
+      throw e;
+    }
+  });
+}

package/src/util/headerEnvelope.ts

@@ -0,0 +1,76 @@
+import {ssz} from "@lodestar/types";
+import {
+  BlockAccessList,
+  ExecutionPayloadEnvelope,
+  ExecutionPayloadHeaderEnvelope,
+  SignedExecutionPayloadEnvelope,
+  SignedExecutionPayloadHeaderEnvelope,
+  Transactions,
+  Withdrawals,
+} from "@lodestar/types/gloas";
+import {toRootHex} from "@lodestar/utils";
+import {
+  EnvelopeReconstructionError,
+  EnvelopeReconstructionErrorCode,
+} from "../chain/errors/envelopeReconstructionError.js";
+
+/** The parts of an execution payload the EL serves back via engine_getPayloadBodiesByHashV2 */
+export type ExecutionPayloadBodies = {
+  transactions: Transactions;
+  withdrawals: Withdrawals;
+  blockAccessList: BlockAccessList;
+};
+
+export type ExecutionPayloadBodyField = keyof ExecutionPayloadBodies;
+
+const {transactions, withdrawals, blockAccessList} = ssz.gloas.ExecutionPayload.fields;
+
+/** Replace the three body lists by their roots; hashes to the same root as the full envelope */
+export function toHeaderEnvelope(envelope: ExecutionPayloadEnvelope): ExecutionPayloadHeaderEnvelope {
+  const {payload, ...envelopeFields} = envelope;
+  const {transactions: txs, withdrawals: wds, blockAccessList: bal, ...scalars} = payload;
+  return {
+    ...envelopeFields,
+    payloadHeader: {
+      ...scalars,
+      transactionsRoot: transactions.hashTreeRoot(txs),
+      withdrawalsRoot: withdrawals.hashTreeRoot(wds),
+      blockAccessListRoot: blockAccessList.hashTreeRoot(bal),
+    },
+  };
+}
+
+export function toSignedHeaderEnvelope(envelope: SignedExecutionPayloadEnvelope): SignedExecutionPayloadHeaderEnvelope {
+  return {message: toHeaderEnvelope(envelope.message), signature: envelope.signature};
+}
+
+/**
+ * Rebuild the full signed envelope from header envelope + EL bodies, each body verified against its stored
+ * root. Throws {@link EnvelopeReconstructionError} BODY_ROOT_MISMATCH naming the first body that differs.
+ */
+export function signedHeaderEnvelopeToFull(
+  headerEnvelope: SignedExecutionPayloadHeaderEnvelope,
+  bodies: ExecutionPayloadBodies
+): SignedExecutionPayloadEnvelope {
+  const {payloadHeader, ...envelopeFields} = headerEnvelope.message;
+  const {transactionsRoot, withdrawalsRoot, blockAccessListRoot, ...scalars} = payloadHeader;
+  const slot = scalars.slotNumber;
+
+  assertBodyRoot("transactions", slot, transactionsRoot, transactions.hashTreeRoot(bodies.transactions));
+  assertBodyRoot("withdrawals", slot, withdrawalsRoot, withdrawals.hashTreeRoot(bodies.withdrawals));
+  assertBodyRoot("blockAccessList", slot, blockAccessListRoot, blockAccessList.hashTreeRoot(bodies.blockAccessList));
+
+  return {
+    message: {...envelopeFields, payload: {...scalars, ...bodies}},
+    signature: headerEnvelope.signature,
+  };
+}
+
+function assertBodyRoot(field: ExecutionPayloadBodyField, slot: number, stored: Uint8Array, rebuilt: Uint8Array): void {
+  if (!ssz.Root.equals(stored, rebuilt)) {
+    throw new EnvelopeReconstructionError(
+      {code: EnvelopeReconstructionErrorCode.BODY_ROOT_MISMATCH, slot, field},
+      `reconstructed ${field} root mismatch slot=${slot} stored=${toRootHex(stored)} rebuilt=${toRootHex(rebuilt)}`
+    );
+  }
+}