@lodestar/beacon-node 1.51.0-dev.83da04fef3 → 1.51.0-dev.99edacec52

package/src/chain/ColumnReconstructionTracker.ts

@@ -1,7 +1,10 @@
 import {ChainForkConfig} from "@lodestar/config";
+import {NUMBER_OF_COLUMNS} from "@lodestar/params";
+import {RootHex} from "@lodestar/types";
 import {Logger, sleep} from "@lodestar/utils";
 import {Metrics} from "../metrics/metrics.js";
 import {DataColumnReconstructionCode, recoverDataColumnSidecars} from "../util/dataColumns.js";
+import {JobItemQueue} from "../util/queue/index.js";
 import {BlockInputColumns} from "./blocks/blockInput/index.js";
 import {PayloadEnvelopeInput} from "./blocks/payloadEnvelopeInput/index.js";
 import {ChainEventEmitter} from "./emitter.js";
@@ -16,15 +19,22 @@ const RECONSTRUCTION_DELAY_MIN_BPS = 667;
  */
 const RECONSTRUCTION_DELAY_MAX_BPS = 1000;
 
+const QUEUE_MAX_LENGTH = 16;
+
+export type ColumnReconstructionInput = BlockInputColumns | PayloadEnvelopeInput;
+
 export type ColumnReconstructionTrackerInit = {
   logger: Logger;
   emitter: ChainEventEmitter;
   metrics: Metrics | null;
   config: ChainForkConfig;
+  signal: AbortSignal;
 };
 
 /**
- * Tracks column reconstruction attempts to avoid duplicate and multiple in-flight calls
+ * Runs column reconstruction one block root at a time, in trigger order. A trigger for a root that is
+ * queued or in flight is a no-op. A root is removed once its attempt finishes so a later column can
+ * retry a failed attempt.
  */
 export class ColumnReconstructionTracker {
   logger: Logger;
@@ -32,62 +42,71 @@ export class ColumnReconstructionTracker {
   metrics: Metrics | null;
   config: ChainForkConfig;
 
-  /**
-   * Track last attempted block root
-   *
-   * This is sufficient to avoid duplicate calls since we only call this
-   * function when we see a new data column sidecar from gossip.
-   */
-  lastBlockRootHex: string | null = null;
-  /** Track if a reconstruction attempt is in-flight */
-  running = false;
+  private readonly signal: AbortSignal;
+  private readonly pending = new Set<RootHex>();
+  private readonly queue: JobItemQueue<[ColumnReconstructionInput, number], void>;
 
   constructor(init: ColumnReconstructionTrackerInit) {
     this.logger = init.logger;
     this.emitter = init.emitter;
     this.metrics = init.metrics;
     this.config = init.config;
+    this.signal = init.signal;
+    this.queue = new JobItemQueue(
+      this.reconstruct,
+      {maxLength: QUEUE_MAX_LENGTH, signal: init.signal},
+      init.metrics?.columnReconstructionQueue
+    );
   }
 
-  triggerColumnReconstruction(input: BlockInputColumns | PayloadEnvelopeInput): void {
-    if (this.running) {
+  triggerColumnReconstruction(input: ColumnReconstructionInput): void {
+    if (this.pending.has(input.blockRootHex)) {
       return;
     }
 
-    if (this.lastBlockRootHex === input.blockRootHex) {
+    if (input.getAllColumns().length < NUMBER_OF_COLUMNS / 2) {
       return;
     }
 
-    // We don't care about the outcome of this call,
-    // just that it has been triggered for this block root.
-    this.running = true;
-    this.lastBlockRootHex = input.blockRootHex;
+    this.pending.add(input.blockRootHex);
+    void this.enqueue(input);
+  }
+
+  private async enqueue(input: ColumnReconstructionInput): Promise<void> {
+    try {
+      await this.queue.push(input, Date.now());
+    } catch (e) {
+      this.logger.debug(
+        "Data column sidecar reconstruction dropped",
+        {slot: input.slot, root: input.blockRootHex},
+        e as Error
+      );
+    } finally {
+      this.pending.delete(input.blockRootHex);
+    }
+  }
+
+  private reconstruct = async (input: ColumnReconstructionInput, queuedAtMs: number): Promise<void> => {
     const fork = this.config.getForkName(input.slot);
     const minDelayMs = this.config.getSlotComponentDurationMs(fork, RECONSTRUCTION_DELAY_MIN_BPS);
     const maxDelayMs = this.config.getSlotComponentDurationMs(fork, RECONSTRUCTION_DELAY_MAX_BPS);
     const delay = minDelayMs + Math.random() * (maxDelayMs - minDelayMs);
-    sleep(delay)
-      .then(() => {
-        const logCtx = {slot: input.slot, root: input.blockRootHex};
-        this.logger.debug("Attempting data column sidecar reconstruction", logCtx);
-        recoverDataColumnSidecars(input, this.emitter, this.metrics)
-          .then((result) => {
-            this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({result});
-            this.logger.debug("Data column sidecar reconstruction complete", {...logCtx, result});
-          })
-          .catch((e) => {
-            this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({
-              result: DataColumnReconstructionCode.Failed,
-            });
-            this.logger.debug("Error during data column sidecar reconstruction", logCtx, e as Error);
-          })
-          .finally(() => {
-            this.logger.debug("Data column sidecar reconstruction attempt finished", logCtx);
-            this.running = false;
-          });
-      })
-      .catch((err) => {
-        this.logger.debug("ColumnReconstructionTracker unreachable error", {}, err);
-      });
-  }
+
+    // The delay gives gossip a chance to deliver the remaining columns, so time already spent in the
+    // queue behind another root counts towards it
+    await sleep(delay - (Date.now() - queuedAtMs), this.signal);
+
+    const logCtx = {slot: input.slot, root: input.blockRootHex};
+    this.logger.debug("Attempting data column sidecar reconstruction", logCtx);
+    try {
+      const result = await recoverDataColumnSidecars(input, this.emitter, this.metrics);
+      this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({result});
+      this.logger.debug("Data column sidecar reconstruction complete", {...logCtx, result});
+    } catch (e) {
+      this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({result: DataColumnReconstructionCode.Failed});
+      this.logger.debug("Error during data column sidecar reconstruction", logCtx, e as Error);
+    } finally {
+      this.logger.debug("Data column sidecar reconstruction attempt finished", logCtx);
+    }
+  };
 }

package/src/chain/chain.ts

@@ -509,6 +509,7 @@ export class BeaconChain implements IBeaconChain {
       emitter,
       metrics,
       config,
+      signal,
     });
 
     this.archiveStore = new ArchiveStore(

package/src/metrics/metrics/lodestar.ts

@@ -330,6 +330,31 @@ export function createLodestarMetrics(
       }),
     },
 
+    columnReconstructionQueue: {
+      length: register.gauge({
+        name: "lodestar_column_reconstruction_queue_length",
+        help: "Count of total column reconstruction queue length",
+      }),
+      droppedJobs: register.gauge({
+        name: "lodestar_column_reconstruction_queue_dropped_jobs_total",
+        help: "Count of total column reconstruction queue dropped jobs",
+      }),
+      jobTime: register.histogram({
+        name: "lodestar_column_reconstruction_queue_job_time_seconds",
+        help: "Time to process column reconstruction queue job in seconds",
+        buckets: [0.01, 0.1, 1, 4, 12],
+      }),
+      jobWaitTime: register.histogram({
+        name: "lodestar_column_reconstruction_queue_job_wait_time_seconds",
+        help: "Time from job added to the column reconstruction queue to starting in seconds",
+        buckets: [0.01, 0.1, 1, 4, 12],
+      }),
+      concurrency: register.gauge({
+        name: "lodestar_column_reconstruction_queue_concurrency",
+        help: "Current concurrency of column reconstruction queue",
+      }),
+    },
+
     engineHttpProcessorQueue: {
       length: register.gauge({
         name: "lodestar_engine_http_processor_queue_length",

package/src/network/peers/utils/enrSubnetsDeserialize.ts

@@ -17,10 +17,13 @@ export function deserializeEnrSubnets(bytes: Uint8Array, subnetCount: number): b
     return getUint8ByteToBitBooleanArray(bytes[0] ?? 0);
   }
 
-  let boolsArr: boolean[] = [];
   const byteCount = Math.ceil(subnetCount / 8);
+  const boolsArr = new Array<boolean>(byteCount * 8);
   for (let i = 0; i < byteCount; i++) {
-    boolsArr = boolsArr.concat(getUint8ByteToBitBooleanArray(bytes[i] ?? 0));
+    const bits = getUint8ByteToBitBooleanArray(bytes[i] ?? 0);
+    for (let j = 0; j < 8; j++) {
+      boolsArr[i * 8 + j] = bits[j];
+    }
   }
 
   return boolsArr;

package/src/network/processor/index.ts

@@ -63,7 +63,7 @@ export type NetworkProcessorOpts = GossipHandlerOpts & {
 /**
  * Keep up to 3 slot of unknown roots, so we don't always emit to UnknownBlock sync.
  */
-const MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE = 3;
+export const MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE = 3;
 
 /**
  * Max distinct roots we BUFFER messages for, per slot. Kept tight to avoid the OOM
@@ -665,6 +665,7 @@ export class NetworkProcessor {
    *
    * For DOS protection, the message is ignored (reject metric incremented) when any cap is hit:
    * - MAX_QUEUED_UNKNOWN_BLOCK_GOSSIP_OBJECTS: global awaiting-block message count
+   * - MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE: message slot already pruned by onClockSlot, so not bounded by the per-slot root budget
    * - MAX_BUFFERED_ROOTS_PER_SLOT: distinct roots we buffer messages for, per slot
    * - MAX_AWAITING_MESSAGES_PER_ROOT[topic]: messages buffered per (root, topic)
    */
@@ -675,6 +676,10 @@ export class NetworkProcessor {
       metric?.reject.inc({reason: ReprocessRejectReason.reached_limit, topic: topicType});
       return;
     }
+    if (slot <= this.chain.clock.currentSlot - MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE) {
+      metric?.reject.inc({reason: ReprocessRejectReason.expired, topic: topicType});
+      return;
+    }
     // per-(root, topic) message cap: e.g. <= NUMBER_OF_COLUMNS columns per root
     const perRootCap = MAX_AWAITING_MESSAGES_PER_ROOT[topicType];
     if (
@@ -701,6 +706,10 @@ export class NetworkProcessor {
       metric?.reject.inc({reason: ReprocessRejectReason.reached_limit, topic: topicType});
       return;
     }
+    if (slot <= this.chain.clock.currentSlot - MAX_UNKNOWN_ROOTS_SLOT_CACHE_SIZE) {
+      metric?.reject.inc({reason: ReprocessRejectReason.expired, topic: topicType});
+      return;
+    }
     const perRootCap = MAX_AWAITING_MESSAGES_PER_ROOT[topicType];
     if (
       perRootCap !== undefined &&