@lodestar/beacon-node 1.51.0-dev.e95d4a80f2 → 1.51.0-dev.ec1447da07

package/src/db/repositories/dataColumnSidecarArchive.ts

@@ -15,6 +15,8 @@ const SLOT_BYTE_SIZE = 8;
  * Indexed data by `slot` + `columnIndex`
  */
 export class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot, ColumnIndex, DataColumnSidecar> {
+  private prunedBeforeSlot: Slot | null = null;
+
   constructor(config: ChainForkConfig, db: Db) {
     const bucket = Bucket.allForks_dataColumnSidecarsArchive;
     // Type won't be used since we select it dynamically based on fork
@@ -70,4 +72,22 @@ export class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot,
       encodeNumberForDbKey(0, COLUMN_INDEX_BYTE_SIZE),
     ]);
   }
+
+  /**
+   * Deletes all entries before `slot`. Prunes only delete from the start of the bucket, so the first prune
+   * compacts the range to drop tombstones of earlier runs and later prunes resume where the previous one ended.
+   */
+  async pruneBefore(slot: Slot): Promise<Slot[]> {
+    if (this.prunedBeforeSlot === null) {
+      await this.db.compactRange(this.wrapKey(this.getMinKeyRaw(0)), this.wrapKey(this.getMinKeyRaw(slot)));
+    } else if (slot <= this.prunedBeforeSlot) {
+      return [];
+    }
+
+    const keys = await this.keys({gte: {prefix: this.prunedBeforeSlot ?? 0, id: 0}, lt: {prefix: slot, id: 0}});
+    const slots = [...new Set(keys.map(({prefix}) => prefix))];
+    if (slots.length > 0) await this.deleteMany(slots);
+    this.prunedBeforeSlot = slot;
+    return slots;
+  }
 }

package/src/node/nodejs.ts

@@ -13,11 +13,17 @@ import {BeaconApiMethods} from "@lodestar/api/beacon/server";
 import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ZERO_HASH_HEX} from "@lodestar/params";
-import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
+import {
+  IBeaconStateView,
+  computeStartSlotAtEpoch,
+  isStatePostBellatrix,
+  isStatePostGloas,
+} from "@lodestar/state-transition";
 import {Slot} from "@lodestar/types";
 import {sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
 import {BeaconRestApiServer, getApi} from "../api/index.js";
+import {getDataColumnSidecarsMinEpoch} from "../chain/archiveStore/utils/archiveBlocks.js";
 import {BeaconChain, IBeaconChain, initBeaconMetrics} from "../chain/index.js";
 import {ValidatorMonitor, createValidatorMonitor} from "../chain/validatorMonitor.js";
 import {IBeaconDb} from "../db/index.js";
@@ -212,6 +218,17 @@ export class BeaconNode {
 
     const clock = new Clock({config, genesisTime: anchorState.genesisTime, signal});
 
+    // Before any lookup into the legacy column buckets, which can be slow until compacted
+    const columnsMinEpoch =
+      opts.chain.archiveDataEpochs === Infinity
+        ? null
+        : getDataColumnSidecarsMinEpoch(config, clock.currentEpoch, opts.chain.archiveDataEpochs);
+    await db.compactLegacyDataColumns(
+      columnsMinEpoch !== null && columnsMinEpoch >= config.FULU_FORK_EPOCH
+        ? computeStartSlotAtEpoch(columnsMinEpoch)
+        : 0
+    );
+
     await db.init();
 
     // Prune hot db repos