@lodestar/beacon-node 1.51.0-dev.ec493d509d → 1.51.0-dev.efd5633468

package/src/api/impl/validator/index.ts

@@ -94,7 +94,7 @@ import {
 import {validateGossipProposerPreferences} from "../../../chain/validation/proposerPreferences.js";
 import {validateSyncCommitteeGossipContributionAndProof} from "../../../chain/validation/syncCommitteeContributionAndProof.js";
 import {ZERO_HASH} from "../../../constants/index.js";
-import {BUILDER_BID_DEADLINE_MS, BuilderApiBid, decodeBuilderUrl} from "../../../execution/builder/apiClient.js";
+import {BuilderApiBid, decodeBuilderUrl} from "../../../execution/builder/apiClient.js";
 import {BuilderStatus, NoBidReceived} from "../../../execution/builder/http.js";
 import {getBuilderBidTotalGwei, validateBuilderApiExecutionPayloadBid} from "../../../execution/builder/validateBid.js";
 import {validateGossipFnRetryUnknownRoot} from "../../../network/processor/gossipHandlers.js";
@@ -123,10 +123,14 @@ import {
  * A cutoff of 1667 BPS (2 seconds for 12s slots) gives enough time and if there are unexpected delays
  * it ensures we publish in time as proposals after the attestation deadline will likely be orphaned
  * due to proposer boost reorg.
- *
- * TODO GLOAS: re-evaluate cutoff timing due to attestation deadline changes in gloas
  */
 const BLOCK_PRODUCTION_RACE_CUTOFF_BPS = 1667;
+/**
+ * Post-gloas cutoff, kept at 1667 BPS (2 seconds for 12s slots) for now.
+ *
+ * TODO GLOAS: revisit, the block propagates faster as it only commits to a bid and the attestation deadline changed
+ */
+export const BLOCK_PRODUCTION_RACE_CUTOFF_GLOAS_BPS = 1667;
 /** Rejection message of the bid block branch when there is no viable bid to commit to */
 const NO_BID_AVAILABLE = "No builder bid available";
 
@@ -1005,33 +1009,50 @@ export function getValidatorApi(
         (!circuitBreakerActive &&
           chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex) !== null);
 
-      // Select the p2p bid once builders had time to bid up, matching the deadline advertised
-      // on builder API bid requests, unless the circuit breaker is active
-      const p2pBidPromise: Promise<PooledExecutionPayloadBid | null> = circuitBreakerActive
-        ? Promise.resolve(null)
-        : sleep(Math.max(0, BUILDER_BID_DEADLINE_MS - chain.clock.msFromSlot(slot))).then(() => {
-            const p2pBid = chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
-            // Discard p2p bids below the proposer's configured floor on the total payment.
-            // A p2p bid's total is just its value since gossip validation enforces executionPayment=0.
-            if (p2pBid !== null && BigInt(p2pBid.signedBid.message.value) < builderConfig.minBid) {
-              logger.info("Best p2p bid below configured minimum", {
-                slot,
-                bidValue: prettyGweiToEth(p2pBid.signedBid.message.value),
-                minBid: prettyGweiToEth(builderConfig.minBid),
-              });
-              return null;
-            }
-            return p2pBid;
-          });
+      // Calculate cutoff time based on start of the slot, ensures a slow local payload build does
+      // not delay the proposal when a builder bid block is available (and vice versa)
+      const cutoffMs = Math.max(
+        0,
+        config.getSlotComponentDurationMs(fork, BLOCK_PRODUCTION_RACE_CUTOFF_GLOAS_BPS) - chain.clock.msFromSlot(slot)
+      );
+
+      // Give p2p bids the same time to bid up as granted to builder API bid requests via the advertised
+      // deadline, the pool is read once the deadline passed and the bid requests settled. A late request
+      // does not wait past the race cutoff, a pooled bid is still the fastest fallback if the local block fails
+      const bidDeadlinePromise = circuitBreakerActive
+        ? Promise.resolve()
+        : sleep(Math.min(chain.builderApiClient.bidDeadline, cutoffMs));
 
       // Candidates are ranked by their boosted counted total payment, the p2p bid is governed
       // by the top-level factors and each builder API bid by its own entry. Ties prefer the
       // earliest received builder API bid, so the signed block can be routed back to its
       // builder directly.
       const bestBidPromise: Promise<BidCandidate | null> = (async () => {
-        const [builderApiBids, p2pBid] = await Promise.all([builderApiBidsPromise, p2pBidPromise]);
+        // Past the race cutoff a pooled p2p bid is used right away rather than waiting for pending
+        // builder API requests, the local block is selected at the cutoff anyway if it is ready
+        let [builderApiBids] = await Promise.all([
+          Promise.race([builderApiBidsPromise, sleep(cutoffMs).then(() => null)]),
+          bidDeadlinePromise,
+        ]);
         let parentExecutionRequestsPromise: Promise<gloas.ExecutionRequests> | null = null;
 
+        let p2pBid: PooledExecutionPayloadBid | null = circuitBreakerActive
+          ? null
+          : chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
+        // Discard p2p bids below the proposer's configured floor on the total payment.
+        // A p2p bid's total is just its value since gossip validation enforces executionPayment=0.
+        if (p2pBid !== null && BigInt(p2pBid.signedBid.message.value) < builderConfig.minBid) {
+          logger.info("Best p2p bid below configured minimum", {
+            slot,
+            bidValue: prettyGweiToEth(p2pBid.signedBid.message.value),
+            minBid: prettyGweiToEth(builderConfig.minBid),
+          });
+          p2pBid = null;
+        }
+        if (builderApiBids === null) {
+          builderApiBids = p2pBid !== null ? [] : await builderApiBidsPromise;
+        }
+
         const candidates = (
           await Promise.all(
             builderApiBids.map(async ({url, entry, signedBid, receivedMs}): Promise<BidCandidate | null> => {
@@ -1118,13 +1139,6 @@ export function getValidatorApi(
         return fn().finally(() => t?.({source}));
       };
 
-      // Calculate cutoff time based on start of the slot, ensures a slow local payload build does
-      // not delay the proposal when a builder bid block is available (and vice versa)
-      const cutoffMs = Math.max(
-        0,
-        config.getSlotComponentDurationMs(fork, BLOCK_PRODUCTION_RACE_CUTOFF_BPS) - chain.clock.msFromSlot(slot)
-      );
-
       // use abort controller to stop waiting for the bid block if the engine block will be selected
       const controller = new AbortController();
 
@@ -1756,7 +1770,10 @@ export function getValidatorApi(
         }
       }
 
-      const dependentRoot = fromHex(state.getShufflingDecisionRoot(epoch));
+      // The fork epoch PTC is computed at the fork from effective balances updated at the end of the
+      // last pre-fork epoch, so its dependent root is one epoch later than for other epochs
+      const dependentRootEpoch = epoch === config.GLOAS_FORK_EPOCH ? epoch + 1 : epoch;
+      const dependentRoot = fromHex(state.getShufflingDecisionRoot(dependentRootEpoch));
 
       return {
         data: duties,

package/src/chain/GetBlobsTracker.ts

@@ -1,22 +1,22 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
-import {Logger} from "@lodestar/utils";
+import {Logger, pruneSetToMax} from "@lodestar/utils";
 import {BLOB_AND_PROOF_V2_RPC_BYTES} from "../execution/engine/types.js";
 import {IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/metrics.js";
-import {callInNextEventLoop} from "../util/eventLoop.js";
-import {
-  DataColumnEngineResult,
-  getBlobSidecarsFromExecution,
-  getDataColumnSidecarsFromExecution,
-} from "../util/execution.js";
-import {IBlockInput, isBlockInputBlobs} from "./blocks/blockInput/index.js";
+import {DataColumnEngineResult, getDataColumnSidecarsFromExecution} from "../util/execution.js";
+import {IBlockInput, isBlockInputColumns} from "./blocks/blockInput/index.js";
 import {PayloadEnvelopeInput} from "./blocks/payloadEnvelopeInput/index.js";
 import {ChainEventEmitter} from "./emitter.js";
 
+/** A `null` answer only changes if the EL receives the transactions later, so retries are few and spaced out */
+export const MAX_GET_BLOBS_ATTEMPTS = 3;
+export const GET_BLOBS_RETRY_INTERVAL_MS = 1000;
+const MAX_TRACKED_BLOCK_ROOTS = 64;
+
 export type GetBlobsTrackerInit = {
   logger: Logger;
-  executionEngine: IExecutionEngine;
+  executionEngine: Pick<IExecutionEngine, "getBlobs">;
   emitter: ChainEventEmitter;
   metrics: Metrics | null;
   config: ChainForkConfig;
@@ -27,11 +27,13 @@ export type GetBlobsTrackerInit = {
  */
 export class GetBlobsTracker {
   logger: Logger;
-  executionEngine: IExecutionEngine;
+  executionEngine: Pick<IExecutionEngine, "getBlobs">;
   emitter: ChainEventEmitter;
   metrics: Metrics | null;
   config: ChainForkConfig;
   activeReconstructions = new Set<string>();
+  failedAttempts = new Map<string, {count: number; lastAttemptMs: number; retryTimer?: NodeJS.Timeout}>();
+  pendingTriggers = new Set<string>();
   // Preallocate buffers for getBlobsV2 RPC calls
   // See https://github.com/ChainSafe/lodestar/pull/8282 for context
   blobsAndProofsBuffers: {buffers: Uint8Array[]; inUse: boolean}[] = [];
@@ -45,26 +47,31 @@ export class GetBlobsTracker {
   }
 
   triggerGetBlobs(input: IBlockInput | PayloadEnvelopeInput): void {
+    if (!(input instanceof PayloadEnvelopeInput) && !isBlockInputColumns(input)) {
+      return;
+    }
+
     if (this.activeReconstructions.has(input.blockRootHex)) {
+      this.pendingTriggers.add(input.blockRootHex);
       return;
     }
 
-    if (!(input instanceof PayloadEnvelopeInput) && isBlockInputBlobs(input)) {
-      // there is not preallocation for blob sidecars like there is for columns sidecars so no need to
-      // store the index for the preallocated buffers
-      this.activeReconstructions.add(input.blockRootHex);
-      callInNextEventLoop(() => {
-        const logCtx = {slot: input.slot, root: input.blockRootHex};
-        this.logger.verbose("Trigger getBlobsV1 for block", logCtx);
-        getBlobSidecarsFromExecution(this.config, this.executionEngine, this.metrics, this.emitter, input).finally(
-          () => {
-            this.logger.verbose("Completed getBlobsV1 for block", logCtx);
-            this.activeReconstructions.delete(input.blockRootHex);
+    const failed = this.failedAttempts.get(input.blockRootHex);
+    if (failed) {
+      if (failed.count >= MAX_GET_BLOBS_ATTEMPTS) {
+        return;
+      }
+      const waitMs = GET_BLOBS_RETRY_INTERVAL_MS - (Date.now() - failed.lastAttemptMs);
+      if (waitMs > 0) {
+        // Columns tend to arrive in one burst, so keep a single trailing retry instead of relying on a later trigger
+        failed.retryTimer ??= setTimeout(() => {
+          failed.retryTimer = undefined;
+          if (this.failedAttempts.get(input.blockRootHex) === failed && !input.hasAllData()) {
+            this.triggerGetBlobs(input);
           }
-        );
-      });
-
-      return;
+        }, waitMs).unref();
+        return;
+      }
     }
 
     let freeIndex = this.blobsAndProofsBuffers.findIndex(({inUse}) => !inUse);
@@ -83,34 +90,57 @@ export class GetBlobsTracker {
     }
     timer?.();
 
-    // We don't care about the outcome of this call,
-    // just that it has been triggered for this block root.
     this.activeReconstructions.add(input.blockRootHex);
     this.blobsAndProofsBuffers[freeIndex].inUse = true;
-    callInNextEventLoop(() => {
-      const logCtx = {slot: input.slot, root: input.blockRootHex};
-      this.logger.verbose("Trigger getBlobsV2 for block", logCtx);
-      getDataColumnSidecarsFromExecution(
-        this.config,
-        this.executionEngine,
-        this.emitter,
-        input,
-        this.metrics,
-        this.blobsAndProofsBuffers[freeIndex].buffers
-      )
-        .then((result) => {
-          this.logger.debug("getBlobsV2 result for block", {...logCtx, result});
-          this.metrics?.dataColumns.dataColumnEngineResult.inc({result});
-        })
-        .catch((error) => {
-          this.logger.debug("Error during getBlobsV2 for block", logCtx, error as Error);
-          this.metrics?.dataColumns.dataColumnEngineResult.inc({result: DataColumnEngineResult.Failed});
-        })
-        .finally(() => {
-          this.logger.verbose("Completed getBlobsV2 for block", logCtx);
-          this.activeReconstructions.delete(input.blockRootHex);
-          this.blobsAndProofsBuffers[freeIndex].inUse = false;
-        });
-    });
+    const logCtx = {slot: input.slot, root: input.blockRootHex};
+    this.logger.verbose("Trigger getBlobsV2 for block", logCtx);
+    let requestFailed = false;
+    // The request is sent right away, before block processing issues newPayload: ELs that drop a
+    // transaction's blobs once the payload is validated can only answer a request that gets there first
+    getDataColumnSidecarsFromExecution(
+      this.config,
+      this.executionEngine,
+      this.emitter,
+      input,
+      this.metrics,
+      this.blobsAndProofsBuffers[freeIndex].buffers
+    )
+      .then((result) => {
+        this.logger.debug("getBlobsV2 result for block", {...logCtx, result});
+        this.metrics?.dataColumns.dataColumnEngineResult.inc({result});
+        if (result === DataColumnEngineResult.NullResponse) {
+          requestFailed = true;
+          this.recordFailedAttempt(input.blockRootHex);
+        }
+      })
+      .catch((error) => {
+        this.logger.debug("Error during getBlobsV2 for block", logCtx, error as Error);
+        this.metrics?.dataColumns.dataColumnEngineResult.inc({result: DataColumnEngineResult.Failed});
+        requestFailed = true;
+        this.recordFailedAttempt(input.blockRootHex);
+      })
+      .finally(() => {
+        this.logger.verbose("Completed getBlobsV2 for block", logCtx);
+        this.activeReconstructions.delete(input.blockRootHex);
+        this.blobsAndProofsBuffers[freeIndex].inUse = false;
+        this.retryPendingTrigger(input, requestFailed);
+      });
+  }
+
+  private retryPendingTrigger(input: IBlockInput | PayloadEnvelopeInput, requestFailed: boolean): void {
+    // Triggers dropped while the request was in flight re-enter once it failed, the attempt that was just
+    // recorded makes this arm the trailing retry instead of sending another request right away.
+    // Post-gloas only the bid triggers a request, so every failed attempt re-enters to arm its own retry
+    const hadPendingTrigger = this.pendingTriggers.delete(input.blockRootHex);
+    if (requestFailed && (hadPendingTrigger || input instanceof PayloadEnvelopeInput)) {
+      this.triggerGetBlobs(input);
+    }
+  }
+
+  private recordFailedAttempt(blockRootHex: string): void {
+    const failed = this.failedAttempts.get(blockRootHex);
+    clearTimeout(failed?.retryTimer);
+    this.failedAttempts.set(blockRootHex, {count: (failed?.count ?? 0) + 1, lastAttemptMs: Date.now()});
+    pruneSetToMax(this.failedAttempts, MAX_TRACKED_BLOCK_ROOTS);
   }
 }

package/src/chain/archiveStore/archiveStore.ts

@@ -16,7 +16,7 @@ import {HistoricalStateRegen} from "./historicalState/historicalStateRegen.js";
 import {ArchiveMode, ArchiveStoreOpts, StateArchiveStrategy} from "./interface.js";
 import {FrequencyStateArchiveStrategy} from "./strategies/frequencyStateArchiveStrategy.js";
 import {archiveBlocks} from "./utils/archiveBlocks.js";
-import {pruneHistory} from "./utils/pruneHistory.js";
+import {MAX_PRUNE_BLOCK_SLOTS_PER_RUN, pruneHistory} from "./utils/pruneHistory.js";
 
 type ArchiveStoreModules = {
   chain: IBeaconChain;
@@ -104,7 +104,6 @@ export class ArchiveStore {
 
   async init(): Promise<void> {
     if (this.opts.pruneHistory) {
-      // prune ALL stale data before starting
       this.logger.info("Pruning historical data");
       const {stateCutoffSlot} = await callFnWhenAwait(
         pruneHistory(
@@ -115,7 +114,8 @@ export class ArchiveStore {
           this.opts.anchorState.finalizedCheckpoint.epoch,
           this.chain.clock.currentEpoch,
           this.statePruneFromSlot,
-          this.advanceEarliestAvailableSlot
+          this.advanceEarliestAvailableSlot,
+          MAX_PRUNE_BLOCK_SLOTS_PER_RUN
         ),
         () => this.logger.info("Still pruning historical data, please wait..."),
         30_000,
@@ -197,6 +197,7 @@ export class ArchiveStore {
     //
     // `jobQueue.push()` can throw synchronously (queue aborted / full) or reject asynchronously;
     // awaiting it inside the deferred callback funnels both failure modes into the one catch.
+    // biome-ignore lint/nursery/noMisusedPromises: the callback catches all errors itself
     callInNextEventLoop(async () => {
       try {
         await this.jobQueue.push(finalized);
@@ -261,7 +262,8 @@ export class ArchiveStore {
           finalizedEpoch,
           this.chain.clock.currentEpoch,
           this.statePruneFromSlot,
-          this.advanceEarliestAvailableSlot
+          this.advanceEarliestAvailableSlot,
+          MAX_PRUNE_BLOCK_SLOTS_PER_RUN
         );
         this.statePruneFromSlot = stateCutoffSlot;
         timer?.({source: ArchiveStoreTask.PruneHistory});

package/src/chain/archiveStore/utils/archiveBlocks.ts

@@ -56,6 +56,17 @@ async function persistOrphanedBlock(
   await writeIfNotExist(filepath, bytes);
 }
 
+/**
+ * Oldest epoch for which data column sidecars are kept
+ */
+export function getDataColumnSidecarsMinEpoch(
+  config: ChainForkConfig,
+  currentEpoch: Epoch,
+  archiveDataEpochs?: number
+): Epoch {
+  return Math.min(getBlobDataRetentionStartEpoch(config, currentEpoch), currentEpoch - (archiveDataEpochs ?? 0));
+}
+
 /**
  * Archives finalized blocks from active bucket to archive bucket.
  *
@@ -260,10 +271,7 @@ export async function archiveBlocks(
   // Keep only `[min(compute_blob_data_retention_start_epoch(current_epoch), current_epoch - archiveDataEpochs)]`
   if (finalizedPostFulu) {
     if (archiveDataEpochs !== Infinity) {
-      const dataColumnSidecarsMinEpoch = Math.min(
-        getBlobDataRetentionStartEpoch(config, currentEpoch),
-        currentEpoch - (archiveDataEpochs ?? 0)
-      );
+      const dataColumnSidecarsMinEpoch = getDataColumnSidecarsMinEpoch(config, currentEpoch, archiveDataEpochs);
       if (dataColumnSidecarsMinEpoch >= config.FULU_FORK_EPOCH) {
         const columnsPruneSlot = computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch);
         const prunedColumnSlots = await db.dataColumns.pruneBefore(columnsPruneSlot);

package/src/chain/archiveStore/utils/pruneHistory.ts

@@ -1,27 +1,37 @@
-import {ChainConfig} from "@lodestar/config";
+import {ChainForkConfig} from "@lodestar/config";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
 import {Epoch, Slot} from "@lodestar/types";
 import {Logger, prettyPrintIndices} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/interface.js";
 import {Metrics} from "../../../metrics/index.js";
 
+/**
+ * Bounds the blocks pruned per run, on startup and on every finalized checkpoint, so a large backlog is
+ * drained over many runs instead of stalling one. The cutoff can be far ahead when pruning is first enabled
+ * on an existing database, when the retention window shrinks at a fork or when finality resumes after a
+ * long period of non-finality.
+ */
+export const MAX_PRUNE_BLOCK_SLOTS_PER_RUN = 4096;
+
 export async function pruneHistory(
-  config: ChainConfig,
+  config: ChainForkConfig,
   db: IBeaconDb,
   logger: Logger,
   metrics: Metrics | null | undefined,
   finalizedEpoch: Epoch,
   currentEpoch: Epoch,
   statePruneFromSlot: Slot,
-  beforePrune: (blockCutoffSlot: Slot) => Promise<void>
+  beforePrune: (blockCutoffSlot: Slot) => Promise<void>,
+  maxBlockSlots = Infinity
 ): Promise<{blockCutoffSlot: Slot; stateCutoffSlot: Slot}> {
   const blockCutoffEpoch = Math.min(
     // set by config, with underflow protection
-    Math.max(currentEpoch - config.MIN_EPOCHS_FOR_BLOCK_REQUESTS, 0),
+    Math.max(currentEpoch - config.getMinEpochsForBlockRequests(currentEpoch), 0),
     // ensure that during (extremely lol) long periods of non-finality we don't delete unfinalized epoch data
     finalizedEpoch
   );
-  const blockCutoffSlot = computeStartSlotAtEpoch(blockCutoffEpoch);
+  const firstBlockSlot = (await db.blockArchive.firstKey()) ?? 0;
+  const blockCutoffSlot = Math.min(computeStartSlotAtEpoch(blockCutoffEpoch), firstBlockSlot + maxBlockSlots);
   // The latest archived state is the anchor on restart and can trail finalization by a few epochs
   const lastArchivedStateSlot = (await db.stateArchive.lastKey()) ?? 0;
   const stateCutoffSlot = Math.min(computeStartSlotAtEpoch(finalizedEpoch), lastArchivedStateSlot);
@@ -29,7 +39,7 @@ export async function pruneHistory(
   logger.debug("Preparing to prune history", {
     currentEpoch,
     finalizedEpoch,
-    blockCutoffEpoch,
+    blockCutoffSlot,
     statePruneFromSlot,
     stateCutoffSlot,
   });