@lodestar/beacon-node 1.51.0-dev.c99cf49add → 1.51.0-dev.d2b5232209

package/lib/chain/ColumnReconstructionTracker.js

@@ -1,5 +1,7 @@
+import { NUMBER_OF_COLUMNS } from "@lodestar/params";
 import { sleep } from "@lodestar/utils";
 import { DataColumnReconstructionCode, recoverDataColumnSidecars } from "../util/dataColumns.js";
+import { JobItemQueue } from "../util/queue/index.js";
 /**
  * Minimum time to wait before attempting reconstruction
  */
@@ -8,67 +10,71 @@ const RECONSTRUCTION_DELAY_MIN_BPS = 667;
  * Maximum time to wait before attempting reconstruction
  */
 const RECONSTRUCTION_DELAY_MAX_BPS = 1000;
+const QUEUE_MAX_LENGTH = 16;
 /**
- * Tracks column reconstruction attempts to avoid duplicate and multiple in-flight calls
+ * Runs column reconstruction one block root at a time, in trigger order. A trigger for a root that is
+ * queued or in flight is a no-op. A root is removed once its attempt finishes so a later column can
+ * retry a failed attempt.
  */
 export class ColumnReconstructionTracker {
     logger;
     emitter;
     metrics;
     config;
-    /**
-     * Track last attempted block root
-     *
-     * This is sufficient to avoid duplicate calls since we only call this
-     * function when we see a new data column sidecar from gossip.
-     */
-    lastBlockRootHex = null;
-    /** Track if a reconstruction attempt is in-flight */
-    running = false;
+    signal;
+    pending = new Set();
+    queue;
     constructor(init) {
         this.logger = init.logger;
         this.emitter = init.emitter;
         this.metrics = init.metrics;
         this.config = init.config;
+        this.signal = init.signal;
+        this.queue = new JobItemQueue(this.reconstruct, { maxLength: QUEUE_MAX_LENGTH, signal: init.signal }, init.metrics?.columnReconstructionQueue);
     }
     triggerColumnReconstruction(input) {
-        if (this.running) {
+        if (this.pending.has(input.blockRootHex)) {
             return;
         }
-        if (this.lastBlockRootHex === input.blockRootHex) {
+        if (input.getAllColumns().length < NUMBER_OF_COLUMNS / 2) {
             return;
         }
-        // We don't care about the outcome of this call,
-        // just that it has been triggered for this block root.
-        this.running = true;
-        this.lastBlockRootHex = input.blockRootHex;
+        this.pending.add(input.blockRootHex);
+        void this.enqueue(input);
+    }
+    async enqueue(input) {
+        try {
+            await this.queue.push(input, Date.now());
+        }
+        catch (e) {
+            this.logger.debug("Data column sidecar reconstruction dropped", { slot: input.slot, root: input.blockRootHex }, e);
+        }
+        finally {
+            this.pending.delete(input.blockRootHex);
+        }
+    }
+    reconstruct = async (input, queuedAtMs) => {
         const fork = this.config.getForkName(input.slot);
         const minDelayMs = this.config.getSlotComponentDurationMs(fork, RECONSTRUCTION_DELAY_MIN_BPS);
         const maxDelayMs = this.config.getSlotComponentDurationMs(fork, RECONSTRUCTION_DELAY_MAX_BPS);
         const delay = minDelayMs + Math.random() * (maxDelayMs - minDelayMs);
-        sleep(delay)
-            .then(() => {
-            const logCtx = { slot: input.slot, root: input.blockRootHex };
-            this.logger.debug("Attempting data column sidecar reconstruction", logCtx);
-            recoverDataColumnSidecars(input, this.emitter, this.metrics)
-                .then((result) => {
-                this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({ result });
-                this.logger.debug("Data column sidecar reconstruction complete", { ...logCtx, result });
-            })
-                .catch((e) => {
-                this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({
-                    result: DataColumnReconstructionCode.Failed,
-                });
-                this.logger.debug("Error during data column sidecar reconstruction", logCtx, e);
-            })
-                .finally(() => {
-                this.logger.debug("Data column sidecar reconstruction attempt finished", logCtx);
-                this.running = false;
-            });
-        })
-            .catch((err) => {
-            this.logger.debug("ColumnReconstructionTracker unreachable error", {}, err);
-        });
-    }
+        // The delay gives gossip a chance to deliver the remaining columns, so time already spent in the
+        // queue behind another root counts towards it
+        await sleep(delay - (Date.now() - queuedAtMs), this.signal);
+        const logCtx = { slot: input.slot, root: input.blockRootHex };
+        this.logger.debug("Attempting data column sidecar reconstruction", logCtx);
+        try {
+            const result = await recoverDataColumnSidecars(input, this.emitter, this.metrics);
+            this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({ result });
+            this.logger.debug("Data column sidecar reconstruction complete", { ...logCtx, result });
+        }
+        catch (e) {
+            this.metrics?.recoverDataColumnSidecars.reconstructionResult.inc({ result: DataColumnReconstructionCode.Failed });
+            this.logger.debug("Error during data column sidecar reconstruction", logCtx, e);
+        }
+        finally {
+            this.logger.debug("Data column sidecar reconstruction attempt finished", logCtx);
+        }
+    };
 }
 //# sourceMappingURL=ColumnReconstructionTracker.js.map

package/lib/chain/ColumnReconstructionTracker.js.map

@@ -1 +1 @@
-{"version":3,"file":"ColumnReconstructionTracker.js","sourceRoot":"","sources":["../../src/chain/ColumnReconstructionTracker.ts"],"names":[],"mappings":"AACA,OAAO,EAAS,KAAK,EAAC,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAC,4BAA4B,EAAE,yBAAyB,EAAC,MAAM,wBAAwB,CAAC;AAK/F;;GAEG;AACH,MAAM,4BAA4B,GAAG,GAAG,CAAC;AAEzC;;GAEG;AACH,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAS1C;;GAEG;AACH,MAAM,OAAO,2BAA2B;IACtC,MAAM,CAAS;IACf,OAAO,CAAoB;IAC3B,OAAO,CAAiB;IACxB,MAAM,CAAkB;IAExB;;;;;OAKG;IACH,gBAAgB,GAAkB,IAAI,CAAC;IACvC,qDAAqD;IACrD,OAAO,GAAG,KAAK,CAAC;IAEhB,YAAY,IAAqC;QAC/C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,2BAA2B,CAAC,KAA+C;QACzE,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;QAED,IAAI,IAAI,CAAC,gBAAgB,KAAK,KAAK,CAAC,YAAY,EAAE,CAAC;YACjD,OAAO;QACT,CAAC;QAED,gDAAgD;QAChD,uDAAuD;QACvD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC,YAAY,CAAC;QAC3C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,IAAI,EAAE,4BAA4B,CAAC,CAAC;QAC9F,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,IAAI,EAAE,4BAA4B,CAAC,CAAC;QAC9F,MAAM,KAAK,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC;QACrE,KAAK,CAAC,KAAK,CAAC;aACT,IAAI,CAAC,GAAG,EAAE;YACT,MAAM,MAAM,GAAG,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,EAAC,CAAC;YAC5D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+CAA+C,EAAE,MAAM,CAAC,CAAC;YAC3E,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC;iBACzD,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;gBACf,IAAI,CAAC,OAAO,EAAE,yBAAyB,CAAC,oBAAoB,CAAC,GAAG,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;gBAC3E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6CAA6C,EAAE,EAAC,GAAG,MAAM,EAAE,MAAM,EAAC,CAAC,CAAC;YACxF,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;gBACX,IAAI,CAAC,OAAO,EAAE,yBAAyB,CAAC,oBAAoB,CAAC,GAAG,CAAC;oBAC/D,MAAM,EAAE,4BAA4B,CAAC,MAAM;iBAC5C,CAAC,CAAC;gBACH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,MAAM,EAAE,CAAU,CAAC,CAAC;YAC3F,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qDAAqD,EAAE,MAAM,CAAC,CAAC;gBACjF,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;YACvB,CAAC,CAAC,CAAC;QACP,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+CAA+C,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;QAC9E,CAAC,CAAC,CAAC;IACP,CAAC;CACF"}
+{"version":3,"file":"ColumnReconstructionTracker.js","sourceRoot":"","sources":["../../src/chain/ColumnReconstructionTracker.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,iBAAiB,EAAC,MAAM,kBAAkB,CAAC;AAEnD,OAAO,EAAS,KAAK,EAAC,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAC,4BAA4B,EAAE,yBAAyB,EAAC,MAAM,wBAAwB,CAAC;AAC/F,OAAO,EAAC,YAAY,EAAC,MAAM,wBAAwB,CAAC;AAKpD;;GAEG;AACH,MAAM,4BAA4B,GAAG,GAAG,CAAC;AAEzC;;GAEG;AACH,MAAM,4BAA4B,GAAG,IAAI,CAAC;AAE1C,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAY5B;;;;GAIG;AACH,MAAM,OAAO,2BAA2B;IACtC,MAAM,CAAS;IACf,OAAO,CAAoB;IAC3B,OAAO,CAAiB;IACxB,MAAM,CAAkB;IAEP,MAAM,CAAc;IACpB,OAAO,GAAG,IAAI,GAAG,EAAW,CAAC;IAC7B,KAAK,CAA0D;IAEhF,YAAY,IAAqC;QAC/C,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,YAAY,CAC3B,IAAI,CAAC,WAAW,EAChB,EAAC,SAAS,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAC,EAClD,IAAI,CAAC,OAAO,EAAE,yBAAyB,CACxC,CAAC;IACJ,CAAC;IAED,2BAA2B,CAAC,KAAgC;QAC1D,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;YACzC,OAAO;QACT,CAAC;QAED,IAAI,KAAK,CAAC,aAAa,EAAE,CAAC,MAAM,GAAG,iBAAiB,GAAG,CAAC,EAAE,CAAC;YACzD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACrC,KAAK,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAgC;QACpD,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,4CAA4C,EAC5C,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,EAAC,EAC5C,CAAU,CACX,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IAEO,WAAW,GAAG,KAAK,EAAE,KAAgC,EAAE,UAAkB,EAAiB,EAAE;QAClG,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,IAAI,EAAE,4BAA4B,CAAC,CAAC;QAC9F,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,0BAA0B,CAAC,IAAI,EAAE,4BAA4B,CAAC,CAAC;QAC9F,MAAM,KAAK,GAAG,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,UAAU,GAAG,UAAU,CAAC,CAAC;QAErE,iGAAiG;QACjG,8CAA8C;QAC9C,MAAM,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAE5D,MAAM,MAAM,GAAG,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,EAAC,CAAC;QAC5D,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,+CAA+C,EAAE,MAAM,CAAC,CAAC;QAC3E,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YAClF,IAAI,CAAC,OAAO,EAAE,yBAAyB,CAAC,oBAAoB,CAAC,GAAG,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;YAC3E,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6CAA6C,EAAE,EAAC,GAAG,MAAM,EAAE,MAAM,EAAC,CAAC,CAAC;QACxF,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,OAAO,EAAE,yBAAyB,CAAC,oBAAoB,CAAC,GAAG,CAAC,EAAC,MAAM,EAAE,4BAA4B,CAAC,MAAM,EAAC,CAAC,CAAC;YAChH,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iDAAiD,EAAE,MAAM,EAAE,CAAU,CAAC,CAAC;QAC3F,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,qDAAqD,EAAE,MAAM,CAAC,CAAC;QACnF,CAAC;IACH,CAAC,CAAC;CACH"}

package/lib/chain/GetBlobsTracker.d.ts

@@ -5,6 +5,9 @@ import { Metrics } from "../metrics/metrics.js";
 import { IBlockInput } from "./blocks/blockInput/index.js";
 import { PayloadEnvelopeInput } from "./blocks/payloadEnvelopeInput/index.js";
 import { ChainEventEmitter } from "./emitter.js";
+/** A `null` answer only changes if the EL receives the transactions later, so retries are few and spaced out */
+export declare const MAX_GET_BLOBS_ATTEMPTS = 3;
+export declare const GET_BLOBS_RETRY_INTERVAL_MS = 1000;
 export type GetBlobsTrackerInit = {
     logger: Logger;
     executionEngine: IExecutionEngine;
@@ -22,11 +25,19 @@ export declare class GetBlobsTracker {
     metrics: Metrics | null;
     config: ChainForkConfig;
     activeReconstructions: Set<string>;
+    failedAttempts: Map<string, {
+        count: number;
+        lastAttemptMs: number;
+        retryTimer?: NodeJS.Timeout;
+    }>;
+    pendingTriggers: Set<string>;
     blobsAndProofsBuffers: {
         buffers: Uint8Array[];
         inUse: boolean;
     }[];
     constructor(init: GetBlobsTrackerInit);
     triggerGetBlobs(input: IBlockInput | PayloadEnvelopeInput): void;
+    private retryPendingTrigger;
+    private recordFailedAttempt;
 }
 //# sourceMappingURL=GetBlobsTracker.d.ts.map

package/lib/chain/GetBlobsTracker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"GetBlobsTracker.d.ts","sourceRoot":"","sources":["../../src/chain/GetBlobsTracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAC,MAAM,EAAC,MAAM,iBAAiB,CAAC;AAEvC,OAAO,EAAC,gBAAgB,EAAC,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAO9C,OAAO,EAAC,WAAW,EAAoB,MAAM,8BAA8B,CAAC;AAC5E,OAAO,EAAC,oBAAoB,EAAC,MAAM,wCAAwC,CAAC;AAC5E,OAAO,EAAC,iBAAiB,EAAC,MAAM,cAAc,CAAC;AAE/C,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,gBAAgB,CAAC;IAClC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,eAAe,CAAC;CACzB,CAAC;AAEF;;GAEG;AACH,qBAAa,eAAe;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,gBAAgB,CAAC;IAClC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,eAAe,CAAC;IACxB,qBAAqB,cAAqB;IAG1C,qBAAqB,EAAE;QAAC,OAAO,EAAE,UAAU,EAAE,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAC,EAAE,CAAM;IAEtE,YAAY,IAAI,EAAE,mBAAmB,EAMpC;IAED,eAAe,CAAC,KAAK,EAAE,WAAW,GAAG,oBAAoB,GAAG,IAAI,CAoE/D;CACF"}
+{"version":3,"file":"GetBlobsTracker.d.ts","sourceRoot":"","sources":["../../src/chain/GetBlobsTracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAC,MAAM,EAAgB,MAAM,iBAAiB,CAAC;AAEtD,OAAO,EAAC,gBAAgB,EAAC,MAAM,uBAAuB,CAAC;AACvD,OAAO,EAAC,OAAO,EAAC,MAAM,uBAAuB,CAAC;AAM9C,OAAO,EAAC,WAAW,EAAoB,MAAM,8BAA8B,CAAC;AAC5E,OAAO,EAAC,oBAAoB,EAAC,MAAM,wCAAwC,CAAC;AAC5E,OAAO,EAAC,iBAAiB,EAAC,MAAM,cAAc,CAAC;AAE/C,gHAAgH;AAChH,eAAO,MAAM,sBAAsB,IAAI,CAAC;AACxC,eAAO,MAAM,2BAA2B,OAAO,CAAC;AAGhD,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,gBAAgB,CAAC;IAClC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,eAAe,CAAC;CACzB,CAAC;AAEF;;GAEG;AACH,qBAAa,eAAe;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,gBAAgB,CAAC;IAClC,OAAO,EAAE,iBAAiB,CAAC;IAC3B,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IACxB,MAAM,EAAE,eAAe,CAAC;IACxB,qBAAqB,cAAqB;IAC1C,cAAc;eAA2B,MAAM;uBAAiB,MAAM;qBAAe,MAAM,CAAC,OAAO;OAAK;IACxG,eAAe,cAAqB;IAGpC,qBAAqB,EAAE;QAAC,OAAO,EAAE,UAAU,EAAE,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAC,EAAE,CAAM;IAEtE,YAAY,IAAI,EAAE,mBAAmB,EAMpC;IAED,eAAe,CAAC,KAAK,EAAE,WAAW,GAAG,oBAAoB,GAAG,IAAI,CAwG/D;IAED,OAAO,CAAC,mBAAmB;IAQ3B,OAAO,CAAC,mBAAmB;CAM5B"}

package/lib/chain/GetBlobsTracker.js

@@ -1,9 +1,13 @@
 import { computeEpochAtSlot } from "@lodestar/state-transition";
+import { pruneSetToMax } from "@lodestar/utils";
 import { BLOB_AND_PROOF_V2_RPC_BYTES } from "../execution/engine/types.js";
-import { callInNextEventLoop } from "../util/eventLoop.js";
 import { DataColumnEngineResult, getBlobSidecarsFromExecution, getDataColumnSidecarsFromExecution, } from "../util/execution.js";
 import { isBlockInputBlobs } from "./blocks/blockInput/index.js";
 import { PayloadEnvelopeInput } from "./blocks/payloadEnvelopeInput/index.js";
+/** A `null` answer only changes if the EL receives the transactions later, so retries are few and spaced out */
+export const MAX_GET_BLOBS_ATTEMPTS = 3;
+export const GET_BLOBS_RETRY_INTERVAL_MS = 1000;
+const MAX_TRACKED_BLOCK_ROOTS = 64;
 /**
  * Tracks getBlobsV2 calls to the execution engine to avoid duplicate and multiple in-flight calls
  */
@@ -14,6 +18,8 @@ export class GetBlobsTracker {
     metrics;
     config;
     activeReconstructions = new Set();
+    failedAttempts = new Map();
+    pendingTriggers = new Set();
     // Preallocate buffers for getBlobsV2 RPC calls
     // See https://github.com/ChainSafe/lodestar/pull/8282 for context
     blobsAndProofsBuffers = [];
@@ -26,19 +32,52 @@ export class GetBlobsTracker {
     }
     triggerGetBlobs(input) {
         if (this.activeReconstructions.has(input.blockRootHex)) {
+            this.pendingTriggers.add(input.blockRootHex);
             return;
         }
+        const failed = this.failedAttempts.get(input.blockRootHex);
+        if (failed) {
+            if (failed.count >= MAX_GET_BLOBS_ATTEMPTS) {
+                return;
+            }
+            const waitMs = GET_BLOBS_RETRY_INTERVAL_MS - (Date.now() - failed.lastAttemptMs);
+            if (waitMs > 0) {
+                // Columns tend to arrive in one burst, so keep a single trailing retry instead of relying on a later trigger
+                failed.retryTimer ??= setTimeout(() => {
+                    failed.retryTimer = undefined;
+                    if (this.failedAttempts.get(input.blockRootHex) === failed && !input.hasAllData()) {
+                        this.triggerGetBlobs(input);
+                    }
+                }, waitMs).unref();
+                return;
+            }
+        }
+        // The request is sent right away, before block processing issues newPayload: ELs that drop a
+        // transaction's blobs once the payload is validated can only answer a request that gets there first
         if (!(input instanceof PayloadEnvelopeInput) && isBlockInputBlobs(input)) {
             // there is not preallocation for blob sidecars like there is for columns sidecars so no need to
             // store the index for the preallocated buffers
             this.activeReconstructions.add(input.blockRootHex);
-            callInNextEventLoop(() => {
-                const logCtx = { slot: input.slot, root: input.blockRootHex };
-                this.logger.verbose("Trigger getBlobsV1 for block", logCtx);
-                getBlobSidecarsFromExecution(this.config, this.executionEngine, this.metrics, this.emitter, input).finally(() => {
-                    this.logger.verbose("Completed getBlobsV1 for block", logCtx);
-                    this.activeReconstructions.delete(input.blockRootHex);
-                });
+            const logCtx = { slot: input.slot, root: input.blockRootHex };
+            this.logger.verbose("Trigger getBlobsV1 for block", logCtx);
+            let requestFailed = false;
+            getBlobSidecarsFromExecution(this.config, this.executionEngine, this.metrics, this.emitter, input)
+                .then(() => {
+                // EL errors and misses are handled inside the helper, blobs still missing means the attempt failed
+                if (!input.hasAllData()) {
+                    requestFailed = true;
+                    this.recordFailedAttempt(input.blockRootHex);
+                }
+            })
+                .catch((error) => {
+                this.logger.debug("Error during getBlobsV1 for block", logCtx, error);
+                requestFailed = true;
+                this.recordFailedAttempt(input.blockRootHex);
+            })
+                .finally(() => {
+                this.logger.verbose("Completed getBlobsV1 for block", logCtx);
+                this.activeReconstructions.delete(input.blockRootHex);
+                this.retryPendingTrigger(input, requestFailed);
             });
             return;
         }
@@ -56,28 +95,45 @@ export class GetBlobsTracker {
             }
         }
         timer?.();
-        // We don't care about the outcome of this call,
-        // just that it has been triggered for this block root.
         this.activeReconstructions.add(input.blockRootHex);
         this.blobsAndProofsBuffers[freeIndex].inUse = true;
-        callInNextEventLoop(() => {
-            const logCtx = { slot: input.slot, root: input.blockRootHex };
-            this.logger.verbose("Trigger getBlobsV2 for block", logCtx);
-            getDataColumnSidecarsFromExecution(this.config, this.executionEngine, this.emitter, input, this.metrics, this.blobsAndProofsBuffers[freeIndex].buffers)
-                .then((result) => {
-                this.logger.debug("getBlobsV2 result for block", { ...logCtx, result });
-                this.metrics?.dataColumns.dataColumnEngineResult.inc({ result });
-            })
-                .catch((error) => {
-                this.logger.debug("Error during getBlobsV2 for block", logCtx, error);
-                this.metrics?.dataColumns.dataColumnEngineResult.inc({ result: DataColumnEngineResult.Failed });
-            })
-                .finally(() => {
-                this.logger.verbose("Completed getBlobsV2 for block", logCtx);
-                this.activeReconstructions.delete(input.blockRootHex);
-                this.blobsAndProofsBuffers[freeIndex].inUse = false;
-            });
+        const logCtx = { slot: input.slot, root: input.blockRootHex };
+        this.logger.verbose("Trigger getBlobsV2 for block", logCtx);
+        let requestFailed = false;
+        getDataColumnSidecarsFromExecution(this.config, this.executionEngine, this.emitter, input, this.metrics, this.blobsAndProofsBuffers[freeIndex].buffers)
+            .then((result) => {
+            this.logger.debug("getBlobsV2 result for block", { ...logCtx, result });
+            this.metrics?.dataColumns.dataColumnEngineResult.inc({ result });
+            if (result === DataColumnEngineResult.NullResponse) {
+                requestFailed = true;
+                this.recordFailedAttempt(input.blockRootHex);
+            }
+        })
+            .catch((error) => {
+            this.logger.debug("Error during getBlobsV2 for block", logCtx, error);
+            this.metrics?.dataColumns.dataColumnEngineResult.inc({ result: DataColumnEngineResult.Failed });
+            requestFailed = true;
+            this.recordFailedAttempt(input.blockRootHex);
+        })
+            .finally(() => {
+            this.logger.verbose("Completed getBlobsV2 for block", logCtx);
+            this.activeReconstructions.delete(input.blockRootHex);
+            this.blobsAndProofsBuffers[freeIndex].inUse = false;
+            this.retryPendingTrigger(input, requestFailed);
         });
     }
+    retryPendingTrigger(input, requestFailed) {
+        // Triggers dropped while the request was in flight re-enter once it failed, the attempt that was just
+        // recorded makes this arm the trailing retry instead of sending another request right away
+        if (this.pendingTriggers.delete(input.blockRootHex) && requestFailed) {
+            this.triggerGetBlobs(input);
+        }
+    }
+    recordFailedAttempt(blockRootHex) {
+        const failed = this.failedAttempts.get(blockRootHex);
+        clearTimeout(failed?.retryTimer);
+        this.failedAttempts.set(blockRootHex, { count: (failed?.count ?? 0) + 1, lastAttemptMs: Date.now() });
+        pruneSetToMax(this.failedAttempts, MAX_TRACKED_BLOCK_ROOTS);
+    }
 }
 //# sourceMappingURL=GetBlobsTracker.js.map

package/lib/chain/GetBlobsTracker.js.map

@@ -1 +1 @@
-{"version":3,"file":"GetBlobsTracker.js","sourceRoot":"","sources":["../../src/chain/GetBlobsTracker.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,kBAAkB,EAAC,MAAM,4BAA4B,CAAC;AAE9D,OAAO,EAAC,2BAA2B,EAAC,MAAM,8BAA8B,CAAC;AAGzE,OAAO,EAAC,mBAAmB,EAAC,MAAM,sBAAsB,CAAC;AACzD,OAAO,EACL,sBAAsB,EACtB,4BAA4B,EAC5B,kCAAkC,GACnC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAc,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AAC5E,OAAO,EAAC,oBAAoB,EAAC,MAAM,wCAAwC,CAAC;AAW5E;;GAEG;AACH,MAAM,OAAO,eAAe;IAC1B,MAAM,CAAS;IACf,eAAe,CAAmB;IAClC,OAAO,CAAoB;IAC3B,OAAO,CAAiB;IACxB,MAAM,CAAkB;IACxB,qBAAqB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1C,+CAA+C;IAC/C,kEAAkE;IAClE,qBAAqB,GAA8C,EAAE,CAAC;IAEtE,YAAY,IAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;QAC5C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,eAAe,CAAC,KAAyC;QACvD,IAAI,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;YACvD,OAAO;QACT,CAAC;QAED,IAAI,CAAC,CAAC,KAAK,YAAY,oBAAoB,CAAC,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;YACzE,gGAAgG;YAChG,+CAA+C;YAC/C,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACnD,mBAAmB,CAAC,GAAG,EAAE;gBACvB,MAAM,MAAM,GAAG,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,EAAC,CAAC;gBAC5D,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,8BAA8B,EAAE,MAAM,CAAC,CAAC;gBAC5D,4BAA4B,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,OAAO,CACxG,GAAG,EAAE;oBACH,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,gCAAgC,EAAE,MAAM,CAAC,CAAC;oBAC9D,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBACxD,CAAC,CACF,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,OAAO;QACT,CAAC;QAED,IAAI,SAAS,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,EAAC,KAAK,EAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QAC1E,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,SAAS,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC;YAC9C,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,GAAG,EAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAC,CAAC;QACtE,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACjF,4GAA4G;QAC5G,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,2BAA2B,CAAC,UAAU,EAAE,CAAC;QAC7E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;gBACnE,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,UAAU,CAAC,2BAA2B,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;QACD,KAAK,EAAE,EAAE,CAAC;QAEV,gDAAgD;QAChD,uDAAuD;QACvD,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACnD,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC;QACnD,mBAAmB,CAAC,GAAG,EAAE;YACvB,MAAM,MAAM,GAAG,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,EAAC,CAAC;YAC5D,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,8BAA8B,EAAE,MAAM,CAAC,CAAC;YAC5D,kCAAkC,CAChC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,OAAO,EACZ,KAAK,EACL,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,OAAO,CAC9C;iBACE,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAC,GAAG,MAAM,EAAE,MAAM,EAAC,CAAC,CAAC;gBACtE,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,sBAAsB,CAAC,GAAG,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;YACjE,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE,MAAM,EAAE,KAAc,CAAC,CAAC;gBAC/E,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,sBAAsB,CAAC,GAAG,CAAC,EAAC,MAAM,EAAE,sBAAsB,CAAC,MAAM,EAAC,CAAC,CAAC;YAChG,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,gCAAgC,EAAE,MAAM,CAAC,CAAC;gBAC9D,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBACtD,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;YACtD,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;IACL,CAAC;CACF"}
+{"version":3,"file":"GetBlobsTracker.js","sourceRoot":"","sources":["../../src/chain/GetBlobsTracker.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,kBAAkB,EAAC,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAS,aAAa,EAAC,MAAM,iBAAiB,CAAC;AACtD,OAAO,EAAC,2BAA2B,EAAC,MAAM,8BAA8B,CAAC;AAGzE,OAAO,EACL,sBAAsB,EACtB,4BAA4B,EAC5B,kCAAkC,GACnC,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAc,iBAAiB,EAAC,MAAM,8BAA8B,CAAC;AAC5E,OAAO,EAAC,oBAAoB,EAAC,MAAM,wCAAwC,CAAC;AAG5E,gHAAgH;AAChH,MAAM,CAAC,MAAM,sBAAsB,GAAG,CAAC,CAAC;AACxC,MAAM,CAAC,MAAM,2BAA2B,GAAG,IAAI,CAAC;AAChD,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAUnC;;GAEG;AACH,MAAM,OAAO,eAAe;IAC1B,MAAM,CAAS;IACf,eAAe,CAAmB;IAClC,OAAO,CAAoB;IAC3B,OAAO,CAAiB;IACxB,MAAM,CAAkB;IACxB,qBAAqB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1C,cAAc,GAAG,IAAI,GAAG,EAA+E,CAAC;IACxG,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;IACpC,+CAA+C;IAC/C,kEAAkE;IAClE,qBAAqB,GAA8C,EAAE,CAAC;IAEtE,YAAY,IAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,eAAe,CAAC;QAC5C,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAC5B,CAAC;IAED,eAAe,CAAC,KAAyC;QACvD,IAAI,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;YACvD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAC7C,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC3D,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,MAAM,CAAC,KAAK,IAAI,sBAAsB,EAAE,CAAC;gBAC3C,OAAO;YACT,CAAC;YACD,MAAM,MAAM,GAAG,2BAA2B,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC;YACjF,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,6GAA6G;gBAC7G,MAAM,CAAC,UAAU,KAAK,UAAU,CAAC,GAAG,EAAE;oBACpC,MAAM,CAAC,UAAU,GAAG,SAAS,CAAC;oBAC9B,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC;wBAClF,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;oBAC9B,CAAC;gBACH,CAAC,EAAE,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC;gBACnB,OAAO;YACT,CAAC;QACH,CAAC;QAED,6FAA6F;QAC7F,oGAAoG;QACpG,IAAI,CAAC,CAAC,KAAK,YAAY,oBAAoB,CAAC,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;YACzE,gGAAgG;YAChG,+CAA+C;YAC/C,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,EAAC,CAAC;YAC5D,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,8BAA8B,EAAE,MAAM,CAAC,CAAC;YAC5D,IAAI,aAAa,GAAG,KAAK,CAAC;YAC1B,4BAA4B,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC;iBAC/F,IAAI,CAAC,GAAG,EAAE;gBACT,mGAAmG;gBACnG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,EAAE,CAAC;oBACxB,aAAa,GAAG,IAAI,CAAC;oBACrB,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBAC/C,CAAC;YACH,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE,MAAM,EAAE,KAAc,CAAC,CAAC;gBAC/E,aAAa,GAAG,IAAI,CAAC;gBACrB,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAC/C,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,gCAAgC,EAAE,MAAM,CAAC,CAAC;gBAC9D,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;gBACtD,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;YACjD,CAAC,CAAC,CAAC;YAEL,OAAO;QACT,CAAC;QAED,IAAI,SAAS,GAAG,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,EAAC,KAAK,EAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QAC1E,IAAI,SAAS,KAAK,CAAC,CAAC,EAAE,CAAC;YACrB,SAAS,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC;YAC9C,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,GAAG,EAAC,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAC,CAAC;QACtE,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,mBAAmB,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QACjF,4GAA4G;QAC5G,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,2BAA2B,CAAC,UAAU,EAAE,CAAC;QAC7E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;gBACnE,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,UAAU,CAAC,2BAA2B,CAAC,CAAC;YACjG,CAAC;QACH,CAAC;QACD,KAAK,EAAE,EAAE,CAAC;QAEV,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACnD,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC;QACnD,MAAM,MAAM,GAAG,EAAC,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,KAAK,CAAC,YAAY,EAAC,CAAC;QAC5D,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,8BAA8B,EAAE,MAAM,CAAC,CAAC;QAC5D,IAAI,aAAa,GAAG,KAAK,CAAC;QAC1B,kCAAkC,CAChC,IAAI,CAAC,MAAM,EACX,IAAI,CAAC,eAAe,EACpB,IAAI,CAAC,OAAO,EACZ,KAAK,EACL,IAAI,CAAC,OAAO,EACZ,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,OAAO,CAC9C;aACE,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,6BAA6B,EAAE,EAAC,GAAG,MAAM,EAAE,MAAM,EAAC,CAAC,CAAC;YACtE,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,sBAAsB,CAAC,GAAG,CAAC,EAAC,MAAM,EAAC,CAAC,CAAC;YAC/D,IAAI,MAAM,KAAK,sBAAsB,CAAC,YAAY,EAAE,CAAC;gBACnD,aAAa,GAAG,IAAI,CAAC;gBACrB,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACf,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,mCAAmC,EAAE,MAAM,EAAE,KAAc,CAAC,CAAC;YAC/E,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,sBAAsB,CAAC,GAAG,CAAC,EAAC,MAAM,EAAE,sBAAsB,CAAC,MAAM,EAAC,CAAC,CAAC;YAC9F,aAAa,GAAG,IAAI,CAAC;YACrB,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC/C,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE;YACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,gCAAgC,EAAE,MAAM,CAAC,CAAC;YAC9D,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;YACtD,IAAI,CAAC,qBAAqB,CAAC,SAAS,CAAC,CAAC,KAAK,GAAG,KAAK,CAAC;YACpD,IAAI,CAAC,mBAAmB,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;QACjD,CAAC,CAAC,CAAC;IACP,CAAC;IAEO,mBAAmB,CAAC,KAAyC,EAAE,aAAsB;QAC3F,sGAAsG;QACtG,2FAA2F;QAC3F,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,aAAa,EAAE,CAAC;YACrE,IAAI,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,YAAoB;QAC9C,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QACrD,YAAY,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QACjC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,YAAY,EAAE,EAAC,KAAK,EAAE,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,aAAa,EAAE,IAAI,CAAC,GAAG,EAAE,EAAC,CAAC,CAAC;QACpG,aAAa,CAAC,IAAI,CAAC,cAAc,EAAE,uBAAuB,CAAC,CAAC;IAC9D,CAAC;CACF"}

package/lib/chain/archiveStore/utils/archiveBlocks.d.ts

@@ -12,6 +12,10 @@ export declare enum LateCanonicalBlockReason {
     SlowImport = "slow_import",
     LateReceive = "late_receive"
 }
+/**
+ * Oldest epoch for which data column sidecars are kept
+ */
+export declare function getDataColumnSidecarsMinEpoch(config: ChainForkConfig, currentEpoch: Epoch, archiveDataEpochs?: number): Epoch;
 /**
  * Archives finalized blocks from active bucket to archive bucket.
  *

package/lib/chain/archiveStore/utils/archiveBlocks.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"archiveBlocks.d.ts","sourceRoot":"","sources":["../../../../src/chain/archiveStore/utils/archiveBlocks.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAC,iBAAiB,EAAmB,WAAW,EAAiB,UAAU,EAAC,MAAM,uBAAuB,CAAC;AAGjH,OAAO,EAAC,KAAK,EAAE,IAAI,EAAC,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAC,MAAM,EAAoD,MAAM,iBAAiB,CAAC;AAC1F,OAAO,EAAC,SAAS,EAAC,MAAM,sBAAsB,CAAC;AAE/C,OAAO,EAAC,OAAO,EAAC,MAAM,6BAA6B,CAAC;AAIpD,OAAO,EAAC,iBAAiB,EAAC,MAAM,4BAA4B,CAAC;AAU7D;;GAEG;AACH,oBAAY,wBAAwB;IAElC,UAAU,gBAAgB;IAE1B,WAAW,iBAAiB;CAC7B;AAmBD;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,eAAe,EACvB,EAAE,EAAE,SAAS,EACb,UAAU,EAAE,WAAW,EACvB,iBAAiB,EAAE,iBAAiB,GAAG,SAAS,EAChD,MAAM,EAAE,MAAM,EACd,mBAAmB,EAAE,iBAAiB,EACtC,YAAY,EAAE,KAAK,EACnB,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,YAAY,EAAE,OAAO,EACrB,iBAAiB,CAAC,EAAE,MAAM,EAC1B,qBAAqB,CAAC,EAAE,OAAO,EAC/B,wBAAwB,CAAC,EAAE,MAAM,EACjC,cAAc,UAAO,GACpB,OAAO,CAAC,IAAI,CAAC,CAkOf;AAgLD;;;;;;;GAOG;AACH,wBAAsB,+CAA+C,CACnE,MAAM,EAAE,eAAe,EACvB,EAAE,EAAE,SAAS,EACb,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,UAAU,EAAE,EAC7B,cAAc,EAAE,OAAO,GACtB,OAAO,CAAC,IAAI,EAAE,CAAC,CAmDjB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,UAAU,GAAG,UAAU,CAE1E;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,SAAS;IAAC,IAAI,EAAE,IAAI,CAAA;CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CA+C/E"}
+{"version":3,"file":"archiveBlocks.d.ts","sourceRoot":"","sources":["../../../../src/chain/archiveStore/utils/archiveBlocks.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EACL,iBAAiB,EAEjB,WAAW,EAEX,UAAU,EAEX,MAAM,uBAAuB,CAAC;AAG/B,OAAO,EAAC,KAAK,EAAE,IAAI,EAAC,MAAM,iBAAiB,CAAC;AAC5C,OAAO,EAAC,MAAM,EAAyC,MAAM,iBAAiB,CAAC;AAC/E,OAAO,EAAC,SAAS,EAAC,MAAM,sBAAsB,CAAC;AAE/C,OAAO,EAAC,OAAO,EAAC,MAAM,6BAA6B,CAAC;AAIpD,OAAO,EAAC,iBAAiB,EAAC,MAAM,4BAA4B,CAAC;AAU7D;;GAEG;AACH,oBAAY,wBAAwB;IAElC,UAAU,gBAAgB;IAE1B,WAAW,iBAAiB;CAC7B;AAmBD;;GAEG;AACH,wBAAgB,6BAA6B,CAC3C,MAAM,EAAE,eAAe,EACvB,YAAY,EAAE,KAAK,EACnB,iBAAiB,CAAC,EAAE,MAAM,GACzB,KAAK,CAEP;AAED;;;;;;;;GAQG;AACH,wBAAsB,aAAa,CACjC,MAAM,EAAE,eAAe,EACvB,EAAE,EAAE,SAAS,EACb,UAAU,EAAE,WAAW,EACvB,iBAAiB,EAAE,iBAAiB,GAAG,SAAS,EAChD,MAAM,EAAE,MAAM,EACd,mBAAmB,EAAE,iBAAiB,EACtC,YAAY,EAAE,KAAK,EACnB,OAAO,EAAE,OAAO,GAAG,IAAI,EACvB,YAAY,EAAE,OAAO,EACrB,iBAAiB,CAAC,EAAE,MAAM,EAC1B,qBAAqB,CAAC,EAAE,OAAO,EAC/B,wBAAwB,CAAC,EAAE,MAAM,EACjC,cAAc,UAAO,GACpB,OAAO,CAAC,IAAI,CAAC,CA4Nf;AA8GD;;;;;;;GAOG;AACH,wBAAsB,+CAA+C,CACnE,MAAM,EAAE,eAAe,EACvB,EAAE,EAAE,SAAS,EACb,MAAM,EAAE,MAAM,EACd,eAAe,EAAE,UAAU,EAAE,EAC7B,cAAc,EAAE,OAAO,GACtB,OAAO,CAAC,IAAI,EAAE,CAAC,CAmDjB;AAED;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,4BAA4B,CAAC,KAAK,EAAE,UAAU,GAAG,UAAU,CAE1E;AAED;;;;GAIG;AACH,wBAAgB,sBAAsB,CAAC,CAAC,SAAS;IAAC,IAAI,EAAE,IAAI,CAAA;CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,CA+C/E"}

package/lib/chain/archiveStore/utils/archiveBlocks.js

@@ -1,8 +1,8 @@
 import path from "node:path";
-import { ExecutionStatus, PayloadStatus } from "@lodestar/fork-choice";
+import { ExecutionStatus, PayloadStatus, isGloasBlock, } from "@lodestar/fork-choice";
 import { ForkSeq, SLOTS_PER_EPOCH } from "@lodestar/params";
 import { computeEpochAtSlot, computeStartSlotAtEpoch, getBlobDataRetentionStartEpoch } from "@lodestar/state-transition";
-import { fromAsync, fromHex, prettyPrintIndices, toRootHex } from "@lodestar/utils";
+import { fromHex, prettyPrintIndices, toRootHex } from "@lodestar/utils";
 import { encodeArchivedHeaderEnvelope } from "../../../db/repositories/index.js";
 import { ensureDir, writeIfNotExist } from "../../../util/file.js";
 import { toSignedHeaderEnvelope } from "../../../util/headerEnvelope.js";
@@ -30,6 +30,12 @@ async function persistOrphanedBlock(slot, blockRoot, bytes, opts) {
     await ensureDir(dirpath);
     await writeIfNotExist(filepath, bytes);
 }
+/**
+ * Oldest epoch for which data column sidecars are kept
+ */
+export function getDataColumnSidecarsMinEpoch(config, currentEpoch, archiveDataEpochs) {
+    return Math.min(getBlobDataRetentionStartEpoch(config, currentEpoch), currentEpoch - (archiveDataEpochs ?? 0));
+}
 /**
  * Archives finalized blocks from active bucket to archive bucket.
  *
@@ -90,16 +96,6 @@ export async function archiveBlocks(config, db, forkChoice, lightclientServer, l
             const migratedEntries = await migrateBlobSidecarsFromHotToColdDb(config, db, logger, finalizedCanonicalBlockRoots, currentEpoch);
             logger.verbose("Migrated blobSidecars from hot DB to cold DB", { ...logCtx, migratedEntries });
         }
-        // Keep the normal LevelDB hot-to-cold flow for legacy columns. New columns are written
-        // directly to flat files, so this only moves fallback data left by older versions.
-        if (finalizedPostFulu) {
-            const migratedSlots = await migrateDataColumnSidecarsFromHotToColdDb(config, db, logger, finalizedCanonicalBlocks, currentEpoch);
-            logger.verbose("Migrated legacy dataColumnSidecars from hot DB to cold DB", {
-                ...logCtx,
-                migratedEntries: migratedSlots.length,
-                slotRange: prettyPrintIndices(migratedSlots),
-            });
-        }
         if (finalizedPostGloas) {
             const migratedSlots = await migrateExecutionPayloadEnvelopesFromHotToColdDb(config, db, logger, finalizedCanonicalBlocks, dedupePayloads);
             logger.verbose("Migrated executionPayloadEnvelopes from hot DB to cold DB", {
@@ -109,38 +105,31 @@ export async function archiveBlocks(config, db, forkChoice, lightclientServer, l
             });
         }
     }
-    // deleteNonCanonicalBlocks
-    // loop through forkchoice single time
-    const nonCanonicalBlockRoots = finalizedNonCanonicalBlocks.map((summary) => fromHex(summary.blockRoot));
-    if (nonCanonicalBlockRoots.length > 0) {
-        if (persistOrphanedBlocks) {
-            // Persist orphaned blocks to disk before deleting them from hot db
-            await Promise.all(nonCanonicalBlockRoots.map(async (root, index) => {
-                const block = finalizedNonCanonicalBlocks[index];
-                const blockBytes = await db.block.getBinary(root);
-                const blockLogCtx = { slot: block.slot, root: block.blockRoot };
-                if (blockBytes) {
-                    await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
-                        persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
-                    });
-                    logger.verbose("Persisted orphaned block", { ...logCtx, ...blockLogCtx });
-                }
-                else {
-                    logger.warn("Tried to persist orphaned block but no block found", { ...logCtx, ...blockLogCtx });
-                }
-            }));
+    const canonicalRoots = new Set(finalizedCanonicalBlocks.map((block) => block.blockRoot));
+    const orphanedBlocks = [];
+    const orphanedPayloads = [];
+    for (const block of finalizedNonCanonicalBlocks) {
+        if (block.payloadStatus === PayloadStatus.FULL) {
+            orphanedPayloads.push(block);
+        }
+        // Every Gloas block gets EMPTY before FULL. If only FULL survives pruning, an earlier archive run
+        // already handled the root's block-level data.
+        const isBlockNode = !isGloasBlock(block) || block.payloadStatus === PayloadStatus.EMPTY;
+        if (isBlockNode && !canonicalRoots.has(block.blockRoot)) {
+            orphanedBlocks.push(block);
         }
-        const nonCanonicalSlots = finalizedNonCanonicalBlocks.map((summary) => summary.slot).sort((a, b) => a - b);
-        const nonCanonicalLogCtx = {
+    }
+    const orphanedBlockRoots = orphanedBlocks.map((block) => fromHex(block.blockRoot));
+    if (orphanedPayloads.length > 0) {
+        const orphanedPayloadRoots = orphanedPayloads.map((block) => fromHex(block.blockRoot));
+        const orphanedPayloadLogCtx = {
             ...logCtx,
-            count: nonCanonicalBlockRoots.length,
-            slotRange: prettyPrintIndices(nonCanonicalSlots),
+            count: orphanedPayloads.length,
+            slotRange: prettyPrintIndices(orphanedPayloads.map((block) => block.slot).sort((a, b) => a - b)),
         };
-        const columnItems = finalizedNonCanonicalBlocks
-            // Gloas EMPTY and FULL variants share a block root. EMPTY has no sidecars, so deleting by its root could
-            // remove the canonical FULL variant's columns. Pre-Gloas blocks are always FULL.
-            .filter((summary) => config.getForkSeq(summary.slot) >= ForkSeq.fulu && summary.payloadStatus === PayloadStatus.FULL)
-            .map((summary) => ({ slot: summary.slot, blockRoot: summary.blockRoot }));
+        const columnItems = orphanedPayloads
+            .filter((block) => config.getForkSeq(block.slot) >= ForkSeq.fulu)
+            .map((block) => ({ slot: block.slot, blockRoot: block.blockRoot }));
         if (columnItems.length > 0) {
             // Delete sidecars first so their block roots remain available to retry cleanup after a failure or crash.
             await db.dataColumns.deleteMany(columnItems);
@@ -150,16 +139,38 @@ export async function archiveBlocks(config, db, forkChoice, lightclientServer, l
                 slotRange: prettyPrintIndices(columnItems.map(({ slot }) => slot).sort((a, b) => a - b)),
             });
         }
-        await db.block.batchDelete(nonCanonicalBlockRoots);
-        logger.verbose("Deleted non canonical blocks from hot DB", nonCanonicalLogCtx);
         if (finalizedPostDeneb) {
-            await db.blobSidecars.batchDelete(nonCanonicalBlockRoots);
-            logger.verbose("Deleted non canonical blobSidecars from hot DB", nonCanonicalLogCtx);
+            await db.blobSidecars.batchDelete(orphanedPayloadRoots);
+            logger.verbose("Deleted non canonical blobSidecars from hot DB", orphanedPayloadLogCtx);
         }
         if (finalizedPostGloas) {
-            await db.executionPayloadEnvelope.batchDelete(nonCanonicalBlockRoots);
-            logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", nonCanonicalLogCtx);
+            await db.executionPayloadEnvelope.batchDelete(orphanedPayloadRoots);
+            logger.verbose("Deleted non canonical executionPayloadEnvelopes from hot DB", orphanedPayloadLogCtx);
+        }
+    }
+    if (orphanedBlocks.length > 0) {
+        if (persistOrphanedBlocks) {
+            // Persist orphaned blocks to disk before deleting them from hot db
+            await Promise.all(orphanedBlocks.map(async (block) => {
+                const blockBytes = await db.block.getBinary(fromHex(block.blockRoot));
+                const blockLogCtx = { slot: block.slot, root: block.blockRoot };
+                if (blockBytes) {
+                    await persistOrphanedBlock(block.slot, block.blockRoot, blockBytes, {
+                        persistOrphanedBlocksDir: persistOrphanedBlocksDir ?? "orphaned_blocks",
+                    });
+                    logger.verbose("Persisted orphaned block", { ...logCtx, ...blockLogCtx });
+                }
+                else {
+                    logger.debug("Tried to persist orphaned block but no block found", { ...logCtx, ...blockLogCtx });
+                }
+            }));
         }
+        await db.block.batchDelete(orphanedBlockRoots);
+        logger.verbose("Deleted non canonical blocks from hot DB", {
+            ...logCtx,
+            count: orphanedBlocks.length,
+            slotRange: prettyPrintIndices(orphanedBlocks.map((block) => block.slot).sort((a, b) => a - b)),
+        });
     }
     // Delete expired blobs
     // Keep only `[current_epoch - max(MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, archiveDataEpochs)]`
@@ -187,7 +198,7 @@ export async function archiveBlocks(config, db, forkChoice, lightclientServer, l
     // Keep only `[min(compute_blob_data_retention_start_epoch(current_epoch), current_epoch - archiveDataEpochs)]`
     if (finalizedPostFulu) {
         if (archiveDataEpochs !== Infinity) {
-            const dataColumnSidecarsMinEpoch = Math.min(getBlobDataRetentionStartEpoch(config, currentEpoch), currentEpoch - (archiveDataEpochs ?? 0));
+            const dataColumnSidecarsMinEpoch = getDataColumnSidecarsMinEpoch(config, currentEpoch, archiveDataEpochs);
             if (dataColumnSidecarsMinEpoch >= config.FULU_FORK_EPOCH) {
                 const columnsPruneSlot = computeStartSlotAtEpoch(dataColumnSidecarsMinEpoch);
                 const prunedColumnSlots = await db.dataColumns.pruneBefore(columnsPruneSlot);
@@ -210,9 +221,9 @@ export async function archiveBlocks(config, db, forkChoice, lightclientServer, l
             logger.verbose("dataColumnSidecars pruning skipped: archiveDataEpochs set to Infinity", logCtx);
         }
     }
-    // Prunning potential checkpoint data
+    // Pruning potential checkpoint data
     const finalizedCanonicalNonCheckpointBlocks = getNonCheckpointBlocks(finalizedCanonicalBlockRoots);
-    const nonCheckpointBlockRoots = [...nonCanonicalBlockRoots];
+    const nonCheckpointBlockRoots = [...orphanedBlockRoots];
     for (const block of finalizedCanonicalNonCheckpointBlocks) {
         nonCheckpointBlockRoots.push(block.root);
     }
@@ -310,53 +321,6 @@ async function migrateBlobSidecarsFromHotToColdDb(config, db, logger, blocks, cu
     }
     return migratedWrappedBlobSidecars;
 }
-// TODO: This function can be simplified further by reducing layers of promises in a loop
-/**
- * Post-gloas the data columns of a Gloas block are tied to its execution payload envelope.
- * Columns only exist once the FULL variant of the block is in the proto-array. Pre-Gloas
- * blocks only have a FULL variant, so the payload-status filter passes them all.
- */
-async function migrateDataColumnSidecarsFromHotToColdDb(config, db, logger, canonicalBlocks, currentEpoch) {
-    const columnBlocks = canonicalBlocks.filter((block) => config.getForkSeq(block.slot) < ForkSeq.gloas || block.payloadStatus === PayloadStatus.FULL);
-    if (columnBlocks.length === 0)
-        return [];
-    const blocks = columnBlocks.map((block) => ({ slot: block.slot, root: fromHex(block.blockRoot) }));
-    const migratedSlots = [];
-    for (let i = 0; i < blocks.length; i += BLOB_SIDECAR_BATCH_SIZE) {
-        const toIdx = Math.min(i + BLOB_SIDECAR_BATCH_SIZE, blocks.length);
-        const batch = blocks.slice(i, toIdx);
-        if (batch.length === 0)
-            break;
-        const promises = [];
-        for (const block of batch) {
-            const blockEpoch = computeEpochAtSlot(block.slot);
-            if (config.getForkSeq(block.slot) < ForkSeq.fulu ||
-                blockEpoch < getBlobDataRetentionStartEpoch(config, currentEpoch)) {
-                continue;
-            }
-            const dataColumnSidecarBytes = await fromAsync(db.dataColumnSidecar.valuesStreamBinary(block.root));
-            if (dataColumnSidecarBytes.length === 0) {
-                logger.debug("DataColumnSidecars in forkchoice but missing in hot db, could be already archived", {
-                    slot: block.slot,
-                    root: toRootHex(block.root),
-                });
-                continue;
-            }
-            logger.verbose("Migrated legacy dataColumnSidecars for block", {
-                currentEpoch,
-                slot: block.slot,
-                root: toRootHex(block.root),
-                numSidecars: dataColumnSidecarBytes.length,
-            });
-            promises.push(db.dataColumnSidecarArchive.putManyBinary(block.slot, dataColumnSidecarBytes.map((entry) => ({ key: entry.id, value: entry.value }))));
-            migratedSlots.push(block.slot);
-        }
-        await Promise.all(promises);
-        await db.dataColumnSidecar.deleteMany(batch.map((block) => block.root));
-    }
-    // Ancestor walk is newest to oldest; sort ascending so `prettyPrintIndices` renders cleanly.
-    return migratedSlots.sort((a, b) => a - b);
-}
 /**
  * Post-gloas given a finalized checkpoint at a block root, payload of that block root
  * is not considered finalized, hence they are archived in the next run.