@lodestar/beacon-node 1.51.0-dev.ec493d509d → 1.51.0-dev.efd5633468

package/src/chain/seenCache/seenGossipBlockInput.ts

@@ -4,15 +4,14 @@ import {
   ForkName,
   ForkPostFulu,
   ForkPostGloas,
-  ForkPreDeneb,
+  ForkPreFulu,
   ForkPreGloas,
   SLOTS_PER_EPOCH,
-  isForkPostDeneb,
   isForkPostFulu,
   isForkPostGloas,
 } from "@lodestar/params";
 import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
-import {BLSSignature, RootHex, SignedBeaconBlock, Slot, deneb, fulu} from "@lodestar/types";
+import {BLSSignature, RootHex, SignedBeaconBlock, Slot, fulu} from "@lodestar/types";
 import {LodestarError, Logger, byteArrayEquals, pruneSetToMax} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
 import {MAX_LOOK_AHEAD_EPOCHS} from "../../sync/constants.js";
@@ -21,19 +20,15 @@ import {CustodyConfig} from "../../util/dataColumns.js";
 import {SerializedCache} from "../../util/serializedCache.js";
 import {
   BlockInput,
-  BlockInputBlobs,
   BlockInputColumns,
   BlockInputNoData,
   BlockInputPreData,
   BlockWithSource,
   DAType,
-  ForkBlobsDA,
   IBlockInput,
   LogMetaBasic,
-  LogMetaBlobs,
   LogMetaColumns,
   SourceMeta,
-  isBlockInputBlobs,
   isBlockInputColumns,
   isDaOutOfRange,
 } from "../blocks/blockInput/index.js";
@@ -62,20 +57,17 @@ export type SeenBlockInputCacheModules = {
   logger?: Logger;
 };
 
-export type GetByBlobOptions = {
+export type GetByColumnOptions = {
   throwErrorIfAlreadyKnown?: boolean;
 };
 
 /**
  * Consumers that create BlockInputs or change types of old BlockInputs
  *
- * - gossipHandlers (block and blob)
- * - beaconBlocksMaybeBlobsByRange
- * - unavailableBeaconBlobsByRoot (beaconBlocksMaybeBlobsByRoot)
+ * - gossipHandlers (block and data column)
+ * - cacheByRangeResponses in sync/utils/downloadByRange.ts
+ * - downloadByRoot in sync/utils/downloadByRoot.ts
  * - publishBlock in the beacon/blocks/index.ts API
- *   https://github.com/ChainSafe/lodestar/blob/unstable/packages/beacon-node/src/api/impl/beacon/blocks/index.ts#L62
- * - maybeValidateBlobs in verifyBlocksDataAvailability (is_data_available spec function)
- *   https://github.com/ChainSafe/lodestar/blob/unstable/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts#L111
  *
  *
  * Pruning management for SeenBlockInputCache
@@ -238,17 +230,6 @@ export class SeenBlockInput {
           seenTimestampSec,
           peerIdStr,
         });
-      } else if (!isForkPostDeneb(forkName)) {
-        // Pre-deneb
-        blockInput = BlockInputPreData.createFromBlock({
-          block: block as SignedBeaconBlock<ForkPreDeneb>,
-          blockRootHex,
-          daOutOfRange,
-          forkName,
-          source,
-          seenTimestampSec,
-          peerIdStr,
-        });
       } else if (isForkPostFulu(forkName)) {
         // Fulu Only
         blockInput = BlockInputColumns.createFromBlock({
@@ -263,9 +244,8 @@ export class SeenBlockInput {
           peerIdStr,
         });
       } else {
-        // Deneb and Electra
-        blockInput = BlockInputBlobs.createFromBlock({
-          block: block as SignedBeaconBlock<ForkBlobsDA>,
+        blockInput = BlockInputPreData.createFromBlock({
+          block: block as SignedBeaconBlock<ForkPreFulu>,
           blockRootHex,
           daOutOfRange,
           forkName,
@@ -288,66 +268,6 @@ export class SeenBlockInput {
     return blockInput as BlockInput;
   }
 
-  getByBlob(
-    {
-      blockRootHex,
-      blobSidecar,
-      source,
-      seenTimestampSec,
-      peerIdStr,
-    }: SourceMeta & {blockRootHex: RootHex; blobSidecar: deneb.BlobSidecar},
-    opts: GetByBlobOptions = {}
-  ): BlockInputBlobs {
-    // TODO(peerDAS): Why is it necessary to static cast this here. All conditional paths result in a valid value so should be defined correctly below
-    let blockInput = this.blockInputs.get(blockRootHex) as IBlockInput;
-    let created = false;
-    if (!blockInput) {
-      created = true;
-      const {forkName, daOutOfRange} = this.buildCommonProps(blobSidecar.signedBlockHeader.message.slot);
-      blockInput = BlockInputBlobs.createFromBlob({
-        blobSidecar,
-        blockRootHex,
-        daOutOfRange,
-        forkName,
-        source,
-        seenTimestampSec,
-        peerIdStr,
-      });
-      this.metrics?.seenCache.blockInput.createdByBlob.inc();
-      this.blockInputs.set(blockRootHex, blockInput);
-    }
-
-    if (!isBlockInputBlobs(blockInput)) {
-      throw new SeenBlockInputCacheError(
-        {
-          code: SeenBlockInputCacheErrorCode.WRONG_BLOCK_INPUT_TYPE,
-          cachedType: blockInput.type,
-          requestedType: DAType.Blobs,
-          ...blockInput.getLogMeta(),
-        },
-        `BlockInputType mismatch adding blobIndex=${blobSidecar.index}`
-      );
-    }
-
-    if (!blockInput.hasBlob(blobSidecar.index)) {
-      blockInput.addBlob({blobSidecar, blockRootHex, source, seenTimestampSec, peerIdStr});
-    } else if (!created) {
-      this.logger?.debug(
-        `Attempt to cache blob index #${blobSidecar.index} but is already cached on BlockInput`,
-        blockInput.getLogMeta()
-      );
-      this.metrics?.seenCache.blockInput.duplicateBlobCount.inc({source});
-      if (opts.throwErrorIfAlreadyKnown) {
-        throw new SeenBlockInputCacheError({
-          code: SeenBlockInputCacheErrorCode.GOSSIP_BLOB_ALREADY_KNOWN,
-          ...blockInput.getLogMeta(),
-        });
-      }
-    }
-
-    return blockInput;
-  }
-
   getByColumn(
     {
       blockRootHex,
@@ -356,7 +276,7 @@ export class SeenBlockInput {
       source,
       peerIdStr,
     }: SourceMeta & {blockRootHex: RootHex; columnSidecar: fulu.DataColumnSidecar},
-    opts: GetByBlobOptions = {}
+    opts: GetByColumnOptions = {}
   ): BlockInputColumns {
     let blockInput = this.blockInputs.get(blockRootHex);
     let created = false;
@@ -473,7 +393,6 @@ export class SeenBlockInput {
 
 enum SeenBlockInputCacheErrorCode {
   WRONG_BLOCK_INPUT_TYPE = "BLOCK_INPUT_CACHE_ERROR_WRONG_BLOCK_INPUT_TYPE",
-  GOSSIP_BLOB_ALREADY_KNOWN = "BLOCK_INPUT_CACHE_ERROR_GOSSIP_BLOB_ALREADY_KNOWN",
   GOSSIP_COLUMN_ALREADY_KNOWN = "BLOCK_INPUT_CACHE_ERROR_GOSSIP_COLUMN_ALREADY_KNOWN",
 }
 
@@ -483,9 +402,6 @@ type SeenBlockInputCacheErrorType =
       cachedType: DAType;
       requestedType: DAType;
     })
-  | (LogMetaBlobs & {
-      code: SeenBlockInputCacheErrorCode.GOSSIP_BLOB_ALREADY_KNOWN;
-    })
   | (LogMetaColumns & {
       code: SeenBlockInputCacheErrorCode.GOSSIP_COLUMN_ALREADY_KNOWN;
     });

package/src/chain/validation/aggregateAndProof.ts

@@ -72,7 +72,7 @@ async function validateAggregateAndProof(
   const attSlot = attData.slot;
 
   let committeeIndex: number | null;
-  if (ForkSeq[fork] >= ForkSeq.gloas) {
+  if (ForkSeq[fork] >= ForkSeq.gloas && chain.config.getForkSeq(attSlot) >= ForkSeq.gloas) {
     // [REJECT] `aggregate.data.index < 2`.
     if (attData.index >= 2) {
       throw new AttestationError(GossipAction.REJECT, {

package/src/chain/validation/attestation.ts

@@ -291,7 +291,7 @@ async function validateAttestationNoSignatureCheck(
       // api or first time validation of a gossip attestation
       committeeIndex = attestationOrCache.attestation.committeeIndex;
 
-      if (isForkPostGloas(fork)) {
+      if (isForkPostGloas(fork) && isForkPostGloas(chain.config.getForkName(attSlot))) {
         // [REJECT] `attestation.data.index < 2`.
         if (attData.index >= 2) {
           throw new AttestationError(GossipAction.REJECT, {

package/src/chain/validation/executionPayloadBid.ts

@@ -179,6 +179,20 @@ async function validateExecutionPayloadBid(
     });
   }
 
+  // [IGNORE] this bid is the highest value bid seen for the tuple
+  // `(bid.slot, bid.parent_block_hash, bid.parent_block_root)`.
+  // As a DoS prevention measure, the bid must also exceed the current highest bid by a minimum
+  // increment, see https://github.com/ethereum/consensus-specs/pull/4831. This prevents spam
+  // from builders submitting numerous bids with minimal value increments.
+  const bestBid = chain.executionPayloadBidPool.getBestBid(bid.slot, bidParentBlockHash, bidParentBlockRoot);
+  if (bestBid !== null && bid.value < getMinBidValue(bestBid.signedBid.message.value)) {
+    throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
+      code: ExecutionPayloadBidErrorCode.BID_TOO_LOW,
+      bidValue: bid.value,
+      currentHighestBid: bestBid.signedBid.message.value,
+    });
+  }
+
   // [IGNORE] `bid.parent_block_root` is the hash tree root of a known beacon block in fork choice.
   // Moved earlier than the spec ordering so we can derive the proposer dependent root for the
   // proposer-preferences lookup below from a known fork-choice block.
@@ -352,19 +366,6 @@ async function validateExecutionPayloadBid(
     });
   }
 
-  // [IGNORE] this bid is the highest value bid seen for the tuple
-  // `(bid.slot, bid.parent_block_hash, bid.parent_block_root)`.
-  // As a DoS prevention measure, the bid must also exceed the current highest bid by a minimum
-  // increment, see https://github.com/ethereum/consensus-specs/pull/4831. This prevents spam
-  // from builders submitting numerous bids with minimal value increments.
-  const bestBid = chain.executionPayloadBidPool.getBestBid(bid.slot, bidParentBlockHash, bidParentBlockRoot);
-  if (bestBid !== null && bid.value < getMinBidValue(bestBid.signedBid.message.value)) {
-    throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
-      code: ExecutionPayloadBidErrorCode.BID_TOO_LOW,
-      bidValue: bid.value,
-      currentHighestBid: bestBid.signedBid.message.value,
-    });
-  }
   // [IGNORE] `bid.value` is less or equal than the builder's excess balance --
   // i.e. `can_builder_cover_bid(state, builder_index, amount)` returns `True`.
   if (!state.canBuilderCoverBid(bid.builderIndex, bid.value)) {

package/src/chain/validation/executionPayloadEnvelope.ts

@@ -1,4 +1,4 @@
-import {PayloadStatus} from "@lodestar/fork-choice";
+import {PayloadStatus, ProtoBlock} from "@lodestar/fork-choice";
 import {
   computeStartSlotAtEpoch,
   getExecutionPayloadEnvelopeSignatureSet,
@@ -6,9 +6,15 @@ import {
 } from "@lodestar/state-transition";
 import {gloas, ssz} from "@lodestar/types";
 import {byteArrayEquals, toRootHex} from "@lodestar/utils";
+import {
+  getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized,
+  getSlotFromExecutionPayloadEnvelopeSerialized,
+  getTransactionCountFromExecutionPayloadEnvelopeSerialized,
+} from "../../util/sszBytes.js";
 import {ExecutionPayloadEnvelopeError, ExecutionPayloadEnvelopeErrorCode, GossipAction} from "../errors/index.js";
 import {IBeaconChain} from "../index.js";
 import {RegenCaller} from "../regen/index.js";
+import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js";
 
 export async function validateApiExecutionPayloadEnvelope(
   chain: IBeaconChain,
@@ -24,6 +30,40 @@ export async function validateGossipExecutionPayloadEnvelope(
   return validateExecutionPayloadEnvelope(chain, executionPayloadEnvelope);
 }
 
+/**
+ * Applies the transaction count check to the serialized envelope so that an oversized envelope is dropped
+ * before it is deserialized. Defers to validateGossipExecutionPayloadEnvelope if the data is malformed
+ * or the block is not known.
+ */
+export function validateGossipExecutionPayloadEnvelopeSerialized(
+  chain: IBeaconChain,
+  serializedData: Uint8Array
+): void {
+  const blockRootHex = getBeaconBlockRootFromExecutionPayloadEnvelopeSerialized(serializedData);
+  const slot = getSlotFromExecutionPayloadEnvelopeSerialized(serializedData);
+  const transactionCount = getTransactionCountFromExecutionPayloadEnvelopeSerialized(serializedData);
+  if (blockRootHex === null || slot === null || transactionCount === null) {
+    return;
+  }
+
+  const block = chain.forkChoice.getBlockHexDefaultStatus(blockRootHex);
+  const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
+  if (!block || !payloadInput) {
+    return;
+  }
+
+  const maxTransactionCount = getMaxTransactionCount(chain, block, payloadInput);
+  if (BigInt(transactionCount) > maxTransactionCount) {
+    throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, {
+      code: ExecutionPayloadEnvelopeErrorCode.TOO_MANY_TRANSACTIONS,
+      slot,
+      root: blockRootHex,
+      transactionCount,
+      maxTransactionCount: Number(maxTransactionCount),
+    });
+  }
+}
+
 async function validateExecutionPayloadEnvelope(
   chain: IBeaconChain,
   executionPayloadEnvelope: gloas.SignedExecutionPayloadEnvelope
@@ -123,6 +163,20 @@ async function validateExecutionPayloadEnvelope(
     });
   }
 
+  // [IGNORE] The number of transactions is within the limit implied by the parent payload's gas limit
+  // TODO GLOAS: [REJECT] once https://github.com/ethereum/consensus-specs/pull/5802 is released,
+  // also in validateGossipExecutionPayloadEnvelopeSerialized
+  const maxTransactionCount = getMaxTransactionCount(chain, block, payloadInput);
+  if (BigInt(payload.transactions.length) > maxTransactionCount) {
+    throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, {
+      code: ExecutionPayloadEnvelopeErrorCode.TOO_MANY_TRANSACTIONS,
+      slot: payload.slotNumber,
+      root: blockRootHex,
+      transactionCount: payload.transactions.length,
+      maxTransactionCount: Number(maxTransactionCount),
+    });
+  }
+
   // Get the block state to verify the builder's signature.
   const blockState = await chain.regen
     .getState(block.stateRoot, RegenCaller.validateGossipPayloadEnvelope)
@@ -154,3 +208,16 @@ async function validateExecutionPayloadEnvelope(
     });
   }
 }
+
+/**
+ * The bid's gas limit is only tied to the parent payload by bid gossip, so the bound is the gas limit the
+ * EIP-1559 rule allows on top of the parent payload's. The bid is the only bound if that gas limit is unknown.
+ */
+function getMaxTransactionCount(chain: IBeaconChain, block: ProtoBlock, payloadInput: PayloadEnvelopeInput): bigint {
+  const parentGasLimit = block.executionPayloadBlockHash === null ? 0n : BigInt(block.executionPayloadGasLimit);
+  const maxGasLimit =
+    parentGasLimit === 0n
+      ? payloadInput.getBid().gasLimit
+      : parentGasLimit + (parentGasLimit / 1024n > 1n ? parentGasLimit / 1024n : 1n) - 1n;
+  return maxGasLimit / BigInt(chain.config.MIN_TRANSACTION_GAS);
+}

package/src/chain/validation/syncCommittee.ts

@@ -1,7 +1,8 @@
 import {SYNC_COMMITTEE_SUBNET_COUNT, SYNC_COMMITTEE_SUBNET_SIZE} from "@lodestar/params";
 import {IBeaconStateView, isStatePostAltair} from "@lodestar/state-transition";
-import {SubnetID, altair} from "@lodestar/types";
+import {RootHex, Slot, SubnetID, ValidatorIndex, altair} from "@lodestar/types";
 import {toRootHex} from "@lodestar/utils";
+import {Metrics} from "../../metrics/metrics.js";
 import {GossipAction, SyncCommitteeError, SyncCommitteeErrorCode} from "../errors/index.js";
 import {IBeaconChain} from "../interface.js";
 import {getSyncCommitteeSignatureSet} from "./signatureSets/index.js";
@@ -32,6 +33,33 @@ export async function validateGossipSyncCommittee(
 
   // [IGNORE] There has been no other valid sync committee signature for the declared slot for the validator referenced
   // by sync_committee_signature.validator_index.
+  assertSyncCommitteeMessageNotKnown(chain, slot, subnet, validatorIndex, messageRoot, chain.metrics);
+
+  // [REJECT] The subnet_id is valid for the given validator, i.e. subnet_id in compute_subnets_for_sync_committee(state, sync_committee_signature.validator_index).
+  // Note this validation implies the validator is part of the broader current sync committee along with the correct subcommittee.
+  // > Checked in validateGossipSyncCommitteeExceptSig()
+
+  // [REJECT] The signature is valid for the message beacon_block_root for the validator referenced by validator_index.
+  await validateSyncCommitteeSigOnly(chain, headState, syncCommittee);
+
+  // Re-check after the await: concurrent messages from the same validator all pass the check above.
+  // Metrics were already counted by the first check.
+  assertSyncCommitteeMessageNotKnown(chain, slot, subnet, validatorIndex, messageRoot, null);
+
+  // Register this valid item as seen
+  chain.seenSyncCommitteeMessages.add(slot, subnet, validatorIndex, messageRoot);
+
+  return {indicesInSubcommittee};
+}
+
+function assertSyncCommitteeMessageNotKnown(
+  chain: IBeaconChain,
+  slot: Slot,
+  subnet: SubnetID,
+  validatorIndex: ValidatorIndex,
+  messageRoot: RootHex,
+  metrics: Metrics | null
+): void {
   const prevRoot = chain.seenSyncCommitteeMessages.get(slot, subnet, validatorIndex);
   if (prevRoot) {
     let shouldIgnore = false;
@@ -39,9 +67,9 @@ export async function validateGossipSyncCommittee(
       shouldIgnore = true;
     } else {
       const headRoot = chain.forkChoice.getHeadRoot();
-      chain.metrics?.gossipSyncCommittee.equivocationCount.inc();
+      metrics?.gossipSyncCommittee.equivocationCount.inc();
       if (messageRoot === headRoot) {
-        chain.metrics?.gossipSyncCommittee.equivocationToHeadCount.inc();
+        metrics?.gossipSyncCommittee.equivocationToHeadCount.inc();
       } else {
         shouldIgnore = true;
       }
@@ -57,18 +85,6 @@ export async function validateGossipSyncCommittee(
       });
     }
   }
-
-  // [REJECT] The subnet_id is valid for the given validator, i.e. subnet_id in compute_subnets_for_sync_committee(state, sync_committee_signature.validator_index).
-  // Note this validation implies the validator is part of the broader current sync committee along with the correct subcommittee.
-  // > Checked in validateGossipSyncCommitteeExceptSig()
-
-  // [REJECT] The signature is valid for the message beacon_block_root for the validator referenced by validator_index.
-  await validateSyncCommitteeSigOnly(chain, headState, syncCommittee);
-
-  // Register this valid item as seen
-  chain.seenSyncCommitteeMessages.add(slot, subnet, validatorIndex, messageRoot);
-
-  return {indicesInSubcommittee};
 }
 
 export async function validateApiSyncCommittee(

package/src/chain/validation/syncCommitteeContributionAndProof.ts

@@ -92,6 +92,19 @@ export async function validateSyncCommitteeGossipContributionAndProof(
     });
   }
 
+  // Re-check after the await: concurrent contributions from the same aggregator all pass the checks above
+  if (chain.seenContributionAndProof.isAggregatorKnown(slot, subcommitteeIndex, aggregatorIndex)) {
+    throw new SyncCommitteeError(GossipAction.IGNORE, {
+      code: SyncCommitteeErrorCode.SYNC_COMMITTEE_AGGREGATOR_ALREADY_KNOWN,
+    });
+  }
+
+  if (!skipValidationKnownParticipants && chain.seenContributionAndProof.participantsKnown(contribution)) {
+    throw new SyncCommitteeError(GossipAction.IGNORE, {
+      code: SyncCommitteeErrorCode.SYNC_COMMITTEE_PARTICIPANTS_ALREADY_KNOWN,
+    });
+  }
+
   // no need to add to seenSyncCommittteeContributionCache here, gossip handler will do that
   chain.seenContributionAndProof.add(contributionAndProof, syncCommitteeParticipantIndices.length);
 

package/src/chain/validatorMonitor.ts

@@ -21,7 +21,6 @@ import {
   SubnetID,
   ValidatorIndex,
   altair,
-  deneb,
   gloas,
 } from "@lodestar/types";
 import {
@@ -70,7 +69,6 @@ export type ValidatorMonitor = {
     balances?: number[]
   ): void;
   registerBeaconBlock(src: OpSource, delaySec: Seconds, block: BeaconBlock): void;
-  registerBlobSidecar(src: OpSource, seenTimestampSec: Seconds, blob: deneb.BlobSidecar): void;
   registerExecutionPayloadEnvelope(
     src: OpSource,
     delaySec: Seconds,
@@ -434,10 +432,6 @@ export function createValidatorMonitor(
       }
     },
 
-    registerBlobSidecar(_src, _seenTimestampSec, _blob) {
-      //TODO: freetheblobs
-    },
-
     registerExecutionPayloadEnvelope(_src, _delaySec, _envelope) {
       // TODO GLOAS: implement execution payload envelope monitoring
     },

package/src/db/beacon.ts

@@ -30,6 +30,9 @@ import {
   VoluntaryExitRepository,
 } from "./repositories/index.js";
 
+/** Below this size, seeks over leftover legacy column data stay cheap */
+const LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES = 1024 ** 3;
+
 export type BeaconDbOpts = {
   dataColumnDir: string;
   logger: Logger;
@@ -67,6 +70,7 @@ export class BeaconDb implements IBeaconDb {
   readonly dataColumns: IDataColumnStore;
   lastLegacyArchiveSlot: Slot | null = null;
   private readonly flatFileStore: FlatFileStore;
+  private readonly logger: Logger;
 
   constructor(
     config: ChainForkConfig,
@@ -100,6 +104,7 @@ export class BeaconDb implements IBeaconDb {
     this.syncCommittee = new SyncCommitteeRepository(config, db);
     this.syncCommitteeWitness = new SyncCommitteeWitnessRepository(config, db);
 
+    this.logger = opts.logger;
     this.flatFileStore = new FlatFileStore(opts.dataColumnDir, config, opts.logger, opts.metrics);
     this.dataColumns = new LegacyDataColumnStore(
       this.flatFileStore,
@@ -125,6 +130,40 @@ export class BeaconDb implements IBeaconDb {
     this.flatFileStore.setMetrics(flatFileStoreMetrics);
   }
 
+  /**
+   * Deletes made while an iterator held a snapshot can stay in the bottom LevelDB level, which `compactRange` never
+   * rewrites, so every seek into the legacy column buckets walks them. Writing a key at each end of the pruned range
+   * makes the compaction overlap those files and rewrite them.
+   */
+  async compactLegacyDataColumns(
+    prunedBeforeSlot: Slot,
+    minSizeBytes = LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES
+  ): Promise<void> {
+    const archiveBucket = Bucket.allForks_dataColumnSidecarsArchive;
+    const start = encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(0));
+    const end = encodeKey(archiveBucket, this.dataColumnSidecarArchive.getMinKeyRaw(prunedBeforeSlot));
+    const sizeBytes = await this.db.approximateSize(start, end);
+    if (sizeBytes < minSizeBytes) return;
+
+    // Column indices stay below 0xffff and hot column keys are shorter, so no sentinel matches a real key
+    const upperSentinel =
+      prunedBeforeSlot > 0
+        ? encodeKey(archiveBucket, this.dataColumnSidecarArchive.encodeKeyRaw(prunedBeforeSlot - 1, 0xffff))
+        : encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(35, 0xff));
+    const sentinels = [start, upperSentinel];
+
+    this.logger.info("Compacting legacy data columns, this may take a while", {sizeBytes});
+    const startTime = Date.now();
+    await this.db.batchPut(sentinels.map((key) => ({key, value: new Uint8Array()})));
+    await this.db.compactRange(start, end);
+    await this.db.batchDelete(sentinels);
+    await this.db.compactRange(start, end);
+    this.logger.verbose("Compacted legacy data columns", {
+      durationMs: Date.now() - startTime,
+      sizeBytes: await this.db.approximateSize(start, end),
+    });
+  }
+
   async pruneHotDb(): Promise<void> {
     // Prune all hot blobs
     await this.blobSidecars.batchDelete(await this.blobSidecars.keys());

package/src/db/dataColumnStore.ts

@@ -13,7 +13,7 @@ export type IndexedDataColumnBytes = {index: ColumnIndex; data: Uint8Array};
 type LegacyHotDataColumnStore = Pick<DataColumnSidecarRepository, "values" | "getManyBinary" | "deleteMany">;
 type LegacyArchiveDataColumnStore = Pick<
   DataColumnSidecarArchiveRepository,
-  "values" | "getManyBinary" | "keys" | "deleteMany"
+  "values" | "getManyBinary" | "pruneBefore"
 >;
 type BlockRootIndex = Pick<BlockArchiveRepository, "getSlotByRoot">;
 
@@ -86,9 +86,7 @@ export class LegacyDataColumnStore implements IDataColumnStore {
 
   async pruneBefore(slot: Slot): Promise<Slot[]> {
     const flatFileSlots = await this.flatFiles.pruneBefore(slot);
-    const prefixedKeys = await this.legacyArchive.keys({lt: {prefix: slot, id: 0}});
-    const slots = [...new Set(prefixedKeys.map(({prefix}) => prefix))];
-    if (slots.length > 0) await this.legacyArchive.deleteMany(slots);
+    const slots = await this.legacyArchive.pruneBefore(slot);
     return [...new Set([...flatFileSlots, ...slots])].sort((a, b) => a - b);
   }
 }

package/src/db/flatFileStore/flatFileStore.ts

@@ -1,4 +1,4 @@
-import fs from "node:fs";
+import {type FileHandle, open, readFile, rm} from "node:fs/promises";
 import path from "node:path";
 import {ChainForkConfig} from "@lodestar/config";
 import {ForkPostFulu} from "@lodestar/params";
@@ -172,7 +172,7 @@ export class FlatFileStore implements IFlatFileStore {
       for (const slot of slotsToPrune) {
         const release = await this.acquireMutationLock(slot);
         try {
-          await fs.promises.rm(path.join(this.dataColumnDir, padSlot(slot)), {recursive: true, force: true});
+          await rm(path.join(this.dataColumnDir, padSlot(slot)), {recursive: true, force: true});
           this.slotIndex.remove(slot);
           this.metrics?.prunedDirectories.inc();
         } finally {
@@ -212,7 +212,7 @@ export class FlatFileStore implements IFlatFileStore {
 
   private async readFile(slot: Slot, rootHex: RootHex): Promise<Uint8Array | null> {
     try {
-      return await fs.promises.readFile(this.filePath(slot, rootHex));
+      return await readFile(this.filePath(slot, rootHex));
     } catch (e) {
       if (!isFsNotFoundError(e)) throw e;
       return null;
@@ -224,9 +224,9 @@ export class FlatFileStore implements IFlatFileStore {
     rootHex: RootHex,
     indices: number[]
   ): Promise<(Uint8Array | undefined)[] | null> {
-    let fd: fs.promises.FileHandle;
+    let fd: FileHandle;
     try {
-      fd = await fs.promises.open(this.filePath(slot, rootHex), "r");
+      fd = await open(this.filePath(slot, rootHex), "r");
     } catch (e) {
       if (!isFsNotFoundError(e)) throw e;
       return null;
@@ -270,7 +270,7 @@ export class FlatFileStore implements IFlatFileStore {
     await observeFlatFileStoreOperation(this.metrics, FlatFileStoreOperation.delete, async () => {
       const release = await this.acquireMutationLock(slot);
       try {
-        await fs.promises.rm(this.filePath(slot, rootHex), {force: true});
+        await rm(this.filePath(slot, rootHex), {force: true});
       } finally {
         release();
       }
@@ -294,7 +294,7 @@ function validateDcolHeader(header: DcolHeader, slot: Slot, rootHex: RootHex): v
   }
 }
 
-async function readExactly(fd: fs.promises.FileHandle, buffer: Uint8Array, position: number): Promise<void> {
+async function readExactly(fd: FileHandle, buffer: Uint8Array, position: number): Promise<void> {
   let totalRead = 0;
   while (totalRead < buffer.length) {
     const {bytesRead} = await fd.read(buffer, totalRead, buffer.length - totalRead, position + totalRead);

package/src/db/flatFileStore/slotIndex.ts

@@ -1,4 +1,5 @@
-import fs from "node:fs";
+import type {Dirent} from "node:fs";
+import {readdir} from "node:fs/promises";
 import {Slot} from "@lodestar/types";
 import {isFsNotFoundError} from "./errors.js";
 import {padSlot} from "./path.js";
@@ -33,9 +34,9 @@ export class SlotIndex {
 
   async rebuildFromDisk(columnsDir: string): Promise<SlotIndexRebuildStats> {
     this.slots.clear();
-    let entries: fs.Dirent[];
+    let entries: Dirent[];
     try {
-      entries = await fs.promises.readdir(columnsDir, {withFileTypes: true});
+      entries = await readdir(columnsDir, {withFileTypes: true});
     } catch (e) {
       if (!isFsNotFoundError(e)) throw e;
       return {slots: 0, ignoredEntries: 0};

package/src/db/interface.ts

@@ -68,6 +68,8 @@ export interface IBeaconDb {
   readonly lastLegacyArchiveSlot: Slot | null;
   init(): Promise<void>;
 
+  compactLegacyDataColumns(prunedBeforeSlot: Slot, minSizeBytes?: number): Promise<void>;
+
   pruneHotDb(): Promise<void>;
 
   deleteDeprecatedEth1Data(): Promise<void>;

package/src/db/repositories/blobSidecars.ts

@@ -14,7 +14,6 @@ export const blobSidecarsWrapperSsz = new ContainerType(
 );
 
 export type BlobSidecarsWrapper = ValueOf<typeof blobSidecarsWrapperSsz>;
-export const BLOB_SIDECARS_IN_WRAPPER_INDEX = 44;
 
 /**
  * blobSidecarsWrapper by block root (= hash_tree_root(SignedBeaconBlock.message))