@lodestar/beacon-node 1.51.0-dev.2593063a82 → 1.51.0-dev.4157a0dec4

package/src/chain/ColumnReconstructionTracker.ts

@@ -1,7 +1,10 @@
 import {ChainForkConfig} from "@lodestar/config";
+import {NUMBER_OF_COLUMNS} from "@lodestar/params";
+import {RootHex} from "@lodestar/types";
 import {Logger, sleep} from "@lodestar/utils";
 import {Metrics} from "../metrics/metrics.js";
 import {DataColumnReconstructionCode, recoverDataColumnSidecars} from "../util/dataColumns.js";
+import {JobItemQueue} from "../util/queue/index.js";
 import {BlockInputColumns} from "./blocks/blockInput/index.js";
 import {PayloadEnvelopeInput} from "./blocks/payloadEnvelopeInput/index.js";
 import {ChainEventEmitter} from "./emitter.js";
@@ -16,15 +19,22 @@ const RECONSTRUCTION_DELAY_MIN_BPS = 667;
  */
 const RECONSTRUCTION_DELAY_MAX_BPS = 1000;
 
+const QUEUE_MAX_LENGTH = 16;
+
+export type ColumnReconstructionInput = BlockInputColumns | PayloadEnvelopeInput;
+
 export type ColumnReconstructionTrackerInit = {
   logger: Logger;
   emitter: ChainEventEmitter;
   metrics: Metrics | null;
   config: ChainForkConfig;
+  signal: AbortSignal;
 };
 
 /**
- * Tracks column reconstruction attempts to avoid duplicate and multiple in-flight calls
+ * Runs column reconstruction one block root at a time, in trigger order. A trigger for a root that is
+ * queued or in flight is a no-op. A root is removed once its attempt finishes so a later column can
+ * retry a failed attempt.
  */
 export class ColumnReconstructionTracker {
   logger: Logger;
@@ -32,62 +42,71 @@ export class ColumnReconstructionTracker {
   metrics: Metrics | null;
   config: ChainForkConfig;
 
-  /**
-   * Track last attempted block root
-   *
-   * This is sufficient to avoid duplicate calls since we only call this
-   * function when we see a new data column sidecar from gossip.
-   */
-  lastBlockRootHex: string | null = null;
-  /** Track if a reconstruction attempt is in-flight */
-  running = false;
+  private readonly signal: AbortSignal;
+  private readonly pending = new Set<RootHex>();
+  private readonly queue: JobItemQueue<[ColumnReconstructionInput, number], void>;
 
   constructor(init: ColumnReconstructionTrackerInit) {
     this.logger = init.logger;
     this.emitter = init.emitter;
     this.metrics = init.metrics;
     this.config = init.config;
+    this.signal = init.signal;
+    this.queue = new JobItemQueue(
+      this.reconstruct,
+      {maxLength: QUEUE_MAX_LENGTH, signal: init.signal},
+      init.metrics?.columnReconstructionQueue
+    );
   }
 
-  triggerColumnReconstruction(input: BlockInputColumns | PayloadEnvelopeInput): void {
-    if (this.running) {
+  triggerColumnReconstruction(input: ColumnReconstructionInput): void {
+    if (this.pending.has(input.blockRootHex)) {
       return;
     }
 
-    if (this.lastBlockRootHex === input.blockRootHex) {
+    if (input.getAllColumns().length < NUMBER_OF_COLUMNS / 2) {
       return;
     }
 
-    // We don't care about the outcome of this call,
-    // just that it has been triggered for this block root.
-    this.running = true;
-    this.lastBlockRootHex = input.blockRootHex;
+    this.pending.add(input.blockRootHex);
+    void this.enqueue(input);
+  }
+
+  private async enqueue(input: ColumnReconstructionInput): Promise<void> {
+    try {
+      await this.queue.push(input, Date.now());
+    } catch (e) {
+      this.logger.debug(
+        "Data column sidecar reconstruction dropped",
+        {slot: input.slot, root: input.blockRootHex},
+        e as Error
+      );
+    } finally {
+      this.pending.delete(input.blockRootHex);
+    }
+  }
+
+  private reconstruct = async (input: ColumnReconstructionInput, queuedAtMs: number): Promise<void> => {
     const fork = this.config.getForkName(input.slot);
     const minDelayMs = this.config.getSlotComponentDurationMs(fork, RECONSTRUCTION_DELAY_MIN_BPS);
     const maxDelayMs = this.config.getSlotComponentDurationMs(fork, RECONSTRUCTION_DELAY_MAX_BPS);
     const delay = minDelayMs + Math.random() * (maxDelayMs - minDelayMs);
-    sleep(delay)
-      .then(() => {
-        const logCtx = {slot: input.slot, root: input.blockRootHex};
-        this.logger.debug("Attempting data column sidecar reconstruction", logCtx);
-        recoverDataColumnSidecars(input, this.emitter, this.metrics)
-          .then((result) => {
-            this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({result});
-            this.logger.debug("Data column sidecar reconstruction complete", {...logCtx, result});
-          })
-          .catch((e) => {
-            this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({
-              result: DataColumnReconstructionCode.Failed,
-            });
-            this.logger.debug("Error during data column sidecar reconstruction", logCtx, e as Error);
-          })
-          .finally(() => {
-            this.logger.debug("Data column sidecar reconstruction attempt finished", logCtx);
-            this.running = false;
-          });
-      })
-      .catch((err) => {
-        this.logger.debug("ColumnReconstructionTracker unreachable error", {}, err);
-      });
-  }
+
+    // The delay gives gossip a chance to deliver the remaining columns, so time already spent in the
+    // queue behind another root counts towards it
+    await sleep(delay - (Date.now() - queuedAtMs), this.signal);
+
+    const logCtx = {slot: input.slot, root: input.blockRootHex};
+    this.logger.debug("Attempting data column sidecar reconstruction", logCtx);
+    try {
+      const result = await recoverDataColumnSidecars(input, this.emitter, this.metrics);
+      this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({result});
+      this.logger.debug("Data column sidecar reconstruction complete", {...logCtx, result});
+    } catch (e) {
+      this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({result: DataColumnReconstructionCode.Failed});
+      this.logger.debug("Error during data column sidecar reconstruction", logCtx, e as Error);
+    } finally {
+      this.logger.debug("Data column sidecar reconstruction attempt finished", logCtx);
+    }
+  };
 }

package/src/chain/archiveStore/utils/archiveBlocks.ts

@@ -3,7 +3,7 @@ import {ChainForkConfig} from "@lodestar/config";
 import {KeyValue} from "@lodestar/db";
 import {CheckpointWithHex, ExecutionStatus, IForkChoice, PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
 import {ForkSeq, SLOTS_PER_EPOCH} from "@lodestar/params";
-import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {computeEpochAtSlot, computeStartSlotAtEpoch, getBlobDataRetentionStartEpoch} from "@lodestar/state-transition";
 import {Epoch, Slot} from "@lodestar/types";
 import {Logger, fromAsync, fromHex, prettyPrintIndices, toRootHex} from "@lodestar/utils";
 import {IBeaconDb} from "../../../db/index.js";
@@ -253,14 +253,13 @@ export async function archiveBlocks(
   }
 
   // Delete expired data column sidecars
-  // Keep only `[current_epoch - max(MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS, archiveDataEpochs)]`
+  // Keep only `[min(compute_blob_data_retention_start_epoch(current_epoch), current_epoch - archiveDataEpochs)]`
   if (finalizedPostFulu) {
     if (archiveDataEpochs !== Infinity) {
-      const dataColumnSidecarsArchiveWindow = Math.max(
-        config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS,
-        archiveDataEpochs ?? 0
+      const dataColumnSidecarsMinEpoch = Math.min(
+        getBlobDataRetentionStartEpoch(config, currentEpoch),
+        currentEpoch - (archiveDataEpochs ?? 0)
       );
-      const dataColumnSidecarsMinEpoch = currentEpoch - dataColumnSidecarsArchiveWindow;
       if (dataColumnSidecarsMinEpoch >= config.FULU_FORK_EPOCH) {
         const columnsPruneSlot = computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch);
         const prunedColumnSlots = await db.dataColumns.pruneBefore(columnsPruneSlot);
@@ -439,7 +438,7 @@ async function migrateDataColumnSidecarsFromHotToColdDb(
       const blockEpoch = computeEpochAtSlot(block.slot);
       if (
         config.getForkSeq(block.slot) < ForkSeq.fulu ||
-        blockEpoch < currentEpoch - config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS
+        blockEpoch < getBlobDataRetentionStartEpoch(config, currentEpoch)
       ) {
         continue;
       }

package/src/chain/blocks/blockInput/utils.ts

@@ -1,6 +1,6 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkName, isForkPostDeneb, isForkPostFulu} from "@lodestar/params";
-import {computeEpochAtSlot} from "@lodestar/state-transition";
+import {computeEpochAtSlot, getBlobDataRetentionStartEpoch} from "@lodestar/state-transition";
 import {Epoch, Slot} from "@lodestar/types";
 
 export function isDaOutOfRange(
@@ -10,7 +10,7 @@ export function isDaOutOfRange(
   currentEpoch: Epoch
 ): boolean {
   if (isForkPostFulu(forkName)) {
-    return computeEpochAtSlot(blockSlot) < currentEpoch - config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS;
+    return computeEpochAtSlot(blockSlot) < getBlobDataRetentionStartEpoch(config, currentEpoch);
   }
 
   if (isForkPostDeneb(forkName)) {

package/src/chain/chain.ts

@@ -509,6 +509,7 @@ export class BeaconChain implements IBeaconChain {
       emitter,
       metrics,
       config,
+      signal,
     });
 
     this.archiveStore = new ArchiveStore(

package/src/metrics/metrics/lodestar.ts

@@ -330,6 +330,31 @@ export function createLodestarMetrics(
       }),
     },
 
+    columnReconstructionQueue: {
+      length: register.gauge({
+        name: "lodestar_column_reconstruction_queue_length",
+        help: "Count of total column reconstruction queue length",
+      }),
+      droppedJobs: register.gauge({
+        name: "lodestar_column_reconstruction_queue_dropped_jobs_total",
+        help: "Count of total column reconstruction queue dropped jobs",
+      }),
+      jobTime: register.histogram({
+        name: "lodestar_column_reconstruction_queue_job_time_seconds",
+        help: "Time to process column reconstruction queue job in seconds",
+        buckets: [0.01, 0.1, 1, 4, 12],
+      }),
+      jobWaitTime: register.histogram({
+        name: "lodestar_column_reconstruction_queue_job_wait_time_seconds",
+        help: "Time from job added to the column reconstruction queue to starting in seconds",
+        buckets: [0.01, 0.1, 1, 4, 12],
+      }),
+      concurrency: register.gauge({
+        name: "lodestar_column_reconstruction_queue_concurrency",
+        help: "Current concurrency of column reconstruction queue",
+      }),
+    },
+
     engineHttpProcessorQueue: {
       length: register.gauge({
         name: "lodestar_engine_http_processor_queue_length",

package/src/network/gossip/gossipsub.ts

@@ -21,6 +21,7 @@ import {
   SLOTS_PER_EPOCH,
   SYNC_COMMITTEE_SUBNET_COUNT,
 } from "@lodestar/params";
+import {getSlotDurations} from "@lodestar/state-transition";
 import {SubnetID} from "@lodestar/types";
 import {Logger, Map2d, Map2dArr} from "@lodestar/utils";
 import {RegistryMetricCreator} from "../../metrics/index.js";
@@ -172,7 +173,8 @@ export class Eth2Gossipsub {
       mcacheLength: 6,
       mcacheGossip: 3,
       // this should be in ms
-      seenTTL: config.SLOT_DURATION_MS * SLOTS_PER_EPOCH * 2,
+      // use the largest configured slot duration so the TTL stays conservative across duration changes
+      seenTTL: Math.max(...getSlotDurations(config).map(([, slotDurationMs]) => slotDurationMs)) * SLOTS_PER_EPOCH * 2,
       scoreParams,
       scoreThresholds: gossipScoreThresholds,
       // For a single stream, await processing each RPC before processing the next

package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts

@@ -3,7 +3,7 @@ import {ChainConfig} from "@lodestar/config";
 import {PayloadStatus} from "@lodestar/fork-choice";
 import {ForkSeq, GENESIS_SLOT, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
 import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
-import {computeEpochAtSlot} from "@lodestar/state-transition";
+import {computeEpochAtSlot, getBlobDataRetentionStartEpoch} from "@lodestar/state-transition";
 import {ColumnIndex, Epoch, RootHex, fulu} from "@lodestar/types";
 import {fromHex, toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../../../chain/index.js";
@@ -216,16 +216,10 @@ export function validateDataColumnSidecarsByRangeRequest(
     throw new ResponseError(RespStatus.INVALID_REQUEST, "startSlot < genesis");
   }
 
-  // Spec: [max(current_epoch - MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS, FULU_FORK_EPOCH), current_epoch]
-  const minimumRequestEpoch = Math.max(
-    currentEpoch - config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS,
-    config.FULU_FORK_EPOCH
-  );
+  // Spec: [max(compute_blob_data_retention_start_epoch(current_epoch), FULU_FORK_EPOCH), current_epoch]
+  const minimumRequestEpoch = Math.max(getBlobDataRetentionStartEpoch(config, currentEpoch), config.FULU_FORK_EPOCH);
   if (computeEpochAtSlot(startSlot) < minimumRequestEpoch) {
-    throw new ResponseError(
-      RespStatus.RESOURCE_UNAVAILABLE,
-      "startSlot is before MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS"
-    );
+    throw new ResponseError(RespStatus.RESOURCE_UNAVAILABLE, "startSlot is before the blob data retention window");
   }
 
   if (count > config.MAX_REQUEST_BLOCKS_DENEB) {

package/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts

@@ -1,6 +1,6 @@
 import {PeerId} from "@libp2p/interface";
 import {ResponseOutgoing} from "@lodestar/reqresp";
-import {computeEpochAtSlot} from "@lodestar/state-transition";
+import {computeEpochAtSlot, getBlobDataRetentionStartEpoch} from "@lodestar/state-transition";
 import {ColumnIndex} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../../../chain/index.js";
@@ -19,10 +19,10 @@ export async function* onDataColumnSidecarsByRoot(
   peerId: PeerId,
   peerClient: string
 ): AsyncIterable<ResponseOutgoing> {
-  // SPEC: minimum_request_epoch = max(current_epoch - MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS, FULU_FORK_EPOCH)
+  // SPEC: minimum_request_epoch = max(compute_blob_data_retention_start_epoch(current_epoch), FULU_FORK_EPOCH)
   const currentEpoch = chain.clock.currentEpoch;
   const minimumRequestEpoch = Math.max(
-    currentEpoch - chain.config.MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS,
+    getBlobDataRetentionStartEpoch(chain.config, currentEpoch),
     chain.config.FULU_FORK_EPOCH
   );