@lodestar/beacon-node 1.48.0-dev.fd70e5fc15 → 1.48.0-dev.ff9749be2c

package/src/chain/validation/block.ts

@@ -134,8 +134,9 @@ export async function validateGossipBlock(
   if (parentBlock.slot >= blockSlot) {
     throw new BlockGossipError(GossipAction.REJECT, {
       code: BlockErrorCode.NOT_LATER_THAN_PARENT,
-      parentSlot: parentBlock.slot,
       slot: blockSlot,
+      root: blockRoot,
+      parentSlot: parentBlock.slot,
     });
   }
 
@@ -150,6 +151,8 @@ export async function validateGossipBlock(
     if (blobKzgCommitmentsLen > maxBlobsPerBlock) {
       throw new BlockGossipError(GossipAction.REJECT, {
         code: BlockErrorCode.TOO_MANY_KZG_COMMITMENTS,
+        slot: blockSlot,
+        root: blockRoot,
         blobKzgCommitmentsLen,
         commitmentLimit: maxBlobsPerBlock,
       });
@@ -167,6 +170,8 @@ export async function validateGossipBlock(
     if (blobKzgCommitmentsLen > maxBlobsPerBlock) {
       throw new BlockGossipError(GossipAction.REJECT, {
         code: BlockErrorCode.TOO_MANY_KZG_COMMITMENTS,
+        slot: blockSlot,
+        root: blockRoot,
         blobKzgCommitmentsLen,
         commitmentLimit: maxBlobsPerBlock,
       });
@@ -176,19 +181,13 @@ export async function validateGossipBlock(
     if (!byteArrayEquals(bid.parentBlockRoot, block.parentRoot)) {
       throw new BlockGossipError(GossipAction.REJECT, {
         code: BlockErrorCode.BID_PARENT_ROOT_MISMATCH,
+        slot: blockSlot,
+        root: blockRoot,
         bidParentRoot: toRootHex(bid.parentBlockRoot),
         blockParentRoot: parentRoot,
       });
     }
 
-    // [REJECT] The block contains no deposits.
-    if (body.deposits.length !== 0) {
-      throw new BlockGossipError(GossipAction.REJECT, {
-        code: BlockErrorCode.NON_ZERO_DEPOSITS,
-        count: body.deposits.length,
-      });
-    }
-
     // TODO GLOAS: [REJECT] The block's execution payload parent (defined by bid.parent_block_hash) passes all validation
     // This requires execution engine integration to verify the parent block hash
   }
@@ -249,6 +248,8 @@ export async function validateGossipBlock(
       if (executionPayload.timestamp !== computeTimeAtSlot(config, blockSlot, chain.genesisTime)) {
         throw new BlockGossipError(GossipAction.REJECT, {
           code: BlockErrorCode.INCORRECT_TIMESTAMP,
+          slot: blockSlot,
+          root: blockRoot,
           timestamp: executionPayload.timestamp,
           expectedTimestamp,
         });
@@ -258,7 +259,12 @@ export async function validateGossipBlock(
 
   // [REJECT] The proposer index is a valid validator index
   if (proposerIndex >= state.validatorCount) {
-    throw new BlockGossipError(GossipAction.REJECT, {code: BlockErrorCode.UNKNOWN_PROPOSER, proposerIndex});
+    throw new BlockGossipError(GossipAction.REJECT, {
+      code: BlockErrorCode.UNKNOWN_PROPOSER,
+      slot: blockSlot,
+      root: blockRoot,
+      proposerIndex,
+    });
   }
 
   // [REJECT] The proposer signature, signed_beacon_block.signature, is valid with respect to the proposer_index pubkey.
@@ -270,7 +276,12 @@ export async function validateGossipBlock(
   // shuffling, the block MAY be queued for later processing while proposers for the block's branch are calculated --
   // in such a case do not REJECT, instead IGNORE this message.
   if (state.getBeaconProposer(blockSlot) !== proposerIndex) {
-    throw new BlockGossipError(GossipAction.REJECT, {code: BlockErrorCode.INCORRECT_PROPOSER, proposerIndex});
+    throw new BlockGossipError(GossipAction.REJECT, {
+      code: BlockErrorCode.INCORRECT_PROPOSER,
+      slot: blockSlot,
+      root: blockRoot,
+      proposerIndex,
+    });
   }
 
   // Simple implementation of a pending block queue. Keeping the block here recycles the queue logic, and keeps the
@@ -315,7 +326,8 @@ export async function verifyBlockProposerSignature(
   if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: opts.verifyOnMainThread ?? true}))) {
     throw new BlockGossipError(GossipAction.REJECT, {
       code: BlockErrorCode.PROPOSAL_SIGNATURE_INVALID,
-      blockSlot,
+      slot: blockSlot,
+      root: blockRoot,
     });
   }
 

package/src/chain/validation/dataColumnSidecar.ts

@@ -11,7 +11,7 @@ import {
   computeStartSlotAtEpoch,
   getBlockHeaderProposerSignatureSetByHeaderSlot,
 } from "@lodestar/state-transition";
-import {DataColumnSidecar, Root, Slot, SubnetID, ValidatorIndex, fulu, gloas, ssz} from "@lodestar/types";
+import {DataColumnSidecar, Root, RootHex, Slot, SubnetID, ValidatorIndex, fulu, gloas, ssz} from "@lodestar/types";
 import {byteArrayEquals, toRootHex, verifyMerkleBranch} from "@lodestar/utils";
 import {BeaconMetrics} from "../../metrics/metrics/beacon.js";
 import {Metrics} from "../../metrics/metrics.js";
@@ -39,12 +39,14 @@ export async function validateGossipFuluDataColumnSidecar(
   const blockRootHex = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader));
 
   // 1) [REJECT] The sidecar is valid as verified by verify_data_column_sidecar
-  verifyFuluDataColumnSidecar(chain.config, dataColumnSidecar);
+  verifyFuluDataColumnSidecar(chain.config, dataColumnSidecar, blockRootHex);
 
   // 2) [REJECT] The sidecar is for the correct subnet -- i.e. compute_subnet_for_data_column_sidecar(sidecar.index) == subnet_id
   if (computeSubnetForDataColumnSidecar(chain.config, dataColumnSidecar) !== gossipSubnet) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_SUBNET,
+      slot: blockHeader.slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
       gossipSubnet: gossipSubnet,
     });
@@ -100,8 +102,9 @@ export async function validateGossipFuluDataColumnSidecar(
   if (parentBlock.slot >= blockHeader.slot) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.NOT_LATER_THAN_PARENT,
-      parentSlot: parentBlock.slot,
       slot: blockHeader.slot,
+      root: blockRootHex,
+      parentSlot: parentBlock.slot,
     });
   }
 
@@ -140,6 +143,8 @@ export async function validateGossipFuluDataColumnSidecar(
   if (proposerIndex !== expectedProposerIndex) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INCORRECT_PROPOSER,
+      slot: blockHeader.slot,
+      root: blockRootHex,
       actualProposerIndex: proposerIndex,
       expectedProposerIndex,
     });
@@ -161,9 +166,9 @@ export async function validateGossipFuluDataColumnSidecar(
     ) {
       throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
         code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
-        blockRoot: blockRootHex,
-        index: dataColumnSidecar.index,
         slot: blockHeader.slot,
+        root: blockRootHex,
+        index: dataColumnSidecar.index,
       });
     }
 
@@ -185,7 +190,8 @@ export async function validateGossipFuluDataColumnSidecar(
   if (!valid) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID,
-      slot: dataColumnSidecar.signedBlockHeader.message.slot,
+      slot: blockHeader.slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   }
@@ -204,6 +210,7 @@ export async function validateGossipFuluDataColumnSidecar(
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF,
       slot: blockHeader.slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   } finally {
@@ -240,6 +247,7 @@ export async function validateGossipGloasDataColumnSidecar(
   if (block.slot !== dataColumnSidecar.slot) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_SLOT,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
       expected: block.slot,
       actual: dataColumnSidecar.slot,
@@ -248,12 +256,14 @@ export async function validateGossipGloasDataColumnSidecar(
 
   // [REJECT] The sidecar must pass verify_data_column_sidecar against the block commitments
   const kzgCommitments = payloadInput.getBlobKzgCommitments();
-  verifyGloasDataColumnSidecar(dataColumnSidecar, kzgCommitments);
+  verifyGloasDataColumnSidecar(dataColumnSidecar, kzgCommitments, blockRootHex);
 
   // [REJECT] The sidecar must be on the correct subnet
   if (computeSubnetForDataColumnSidecar(chain.config, dataColumnSidecar) !== gossipSubnet) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_SUBNET,
+      slot: dataColumnSidecar.slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
       gossipSubnet,
     });
@@ -272,6 +282,7 @@ export async function validateGossipGloasDataColumnSidecar(
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF,
       slot: dataColumnSidecar.slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   } finally {
@@ -283,11 +294,17 @@ export async function validateGossipGloasDataColumnSidecar(
  * SPEC FUNCTION
  * https://github.com/ethereum/consensus-specs/blob/v1.6.0-alpha.4/specs/fulu/p2p-interface.md#verify_data_column_sidecar
  */
-function verifyFuluDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar: fulu.DataColumnSidecar): void {
+function verifyFuluDataColumnSidecar(
+  config: ChainForkConfig,
+  dataColumnSidecar: fulu.DataColumnSidecar,
+  blockRootHex: RootHex
+): void {
+  const slot = dataColumnSidecar.signedBlockHeader.message.slot;
   if (dataColumnSidecar.index >= NUMBER_OF_COLUMNS) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_INDEX,
-      slot: dataColumnSidecar.signedBlockHeader.message.slot,
+      slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   }
@@ -295,18 +312,20 @@ function verifyFuluDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar:
   if (dataColumnSidecar.kzgCommitments.length === 0) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.NO_COMMITMENTS,
-      slot: dataColumnSidecar.signedBlockHeader.message.slot,
+      slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   }
 
-  const epoch = computeEpochAtSlot(dataColumnSidecar.signedBlockHeader.message.slot);
+  const epoch = computeEpochAtSlot(slot);
   const maxBlobsPerBlock = config.getMaxBlobsPerBlock(epoch);
 
   if (dataColumnSidecar.kzgCommitments.length > maxBlobsPerBlock) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.TOO_MANY_KZG_COMMITMENTS,
-      slot: dataColumnSidecar.signedBlockHeader.message.slot,
+      slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
       count: dataColumnSidecar.kzgCommitments.length,
       limit: maxBlobsPerBlock,
@@ -319,6 +338,8 @@ function verifyFuluDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar:
   ) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.MISMATCHED_LENGTHS,
+      slot,
+      root: blockRootHex,
       columnLength: dataColumnSidecar.column.length,
       commitmentsLength: dataColumnSidecar.kzgCommitments.length,
       proofsLength: dataColumnSidecar.kzgProofs.length,
@@ -330,12 +351,17 @@ function verifyFuluDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar:
  * SPEC FUNCTION
  * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.3/specs/gloas/p2p-interface.md#modified-verify_data_column_sidecar
  */
-function verifyGloasDataColumnSidecar(dataColumnSidecar: gloas.DataColumnSidecar, kzgCommitments: Uint8Array[]): void {
+function verifyGloasDataColumnSidecar(
+  dataColumnSidecar: gloas.DataColumnSidecar,
+  kzgCommitments: Uint8Array[],
+  blockRootHex: RootHex
+): void {
   const slot = getDataColumnSidecarSlot(dataColumnSidecar);
   if (dataColumnSidecar.index >= NUMBER_OF_COLUMNS) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_INDEX,
       slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   }
@@ -344,6 +370,7 @@ function verifyGloasDataColumnSidecar(dataColumnSidecar: gloas.DataColumnSidecar
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.NO_COMMITMENTS,
       slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   }
@@ -354,6 +381,8 @@ function verifyGloasDataColumnSidecar(dataColumnSidecar: gloas.DataColumnSidecar
   ) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.MISMATCHED_LENGTHS,
+      slot,
+      root: blockRootHex,
       columnLength: dataColumnSidecar.column.length,
       commitmentsLength: kzgCommitments.length,
       proofsLength: dataColumnSidecar.kzgProofs.length,
@@ -465,8 +494,8 @@ export async function validateFuluBlockDataColumnSidecars(
         ) {
           throw new DataColumnSidecarValidationError({
             code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
-            blockRoot: rootHex,
             slot: blockSlot,
+            root: rootHex,
             index: dataColumnSidecars[0].index,
           });
         }
@@ -499,6 +528,7 @@ export async function validateFuluBlockDataColumnSidecars(
           {
             code: DataColumnSidecarErrorCode.INVALID_INDEX,
             slot: blockSlot,
+            root: toRootHex(blockRoot),
             columnIndex: columnSidecar.index,
           },
           "DataColumnSidecar has invalid index"
@@ -543,6 +573,7 @@ export async function validateFuluBlockDataColumnSidecars(
           {
             code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID,
             slot: blockSlot,
+            root: toRootHex(blockRoot),
             columnIndex: columnSidecar.index,
           },
           "DataColumnSidecar has invalid inclusion proof"
@@ -623,6 +654,7 @@ export async function validateGloasBlockDataColumnSidecars(
       if (columnSidecar.slot !== blockSlot) {
         throw new DataColumnSidecarValidationError({
           code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_SLOT,
+          root: toRootHex(blockRoot),
           columnIndex: columnSidecar.index,
           expected: blockSlot,
           actual: columnSidecar.slot,
@@ -644,6 +676,7 @@ export async function validateGloasBlockDataColumnSidecars(
           {
             code: DataColumnSidecarErrorCode.INVALID_INDEX,
             slot: blockSlot,
+            root: toRootHex(blockRoot),
             columnIndex: columnSidecar.index,
           },
           "DataColumnSidecar has invalid index"

package/src/chain/validation/executionPayloadEnvelope.ts

@@ -82,6 +82,8 @@ async function validateExecutionPayloadEnvelope(
   if (block.slot !== payload.slotNumber) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
       code: ExecutionPayloadEnvelopeErrorCode.SLOT_MISMATCH,
+      slot: payload.slotNumber,
+      root: blockRootHex,
       envelopeSlot: payload.slotNumber,
       blockSlot: block.slot,
     });
@@ -91,6 +93,8 @@ async function validateExecutionPayloadEnvelope(
   if (envelope.builderIndex !== payloadInput.getBuilderIndex()) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
       code: ExecutionPayloadEnvelopeErrorCode.BUILDER_INDEX_MISMATCH,
+      slot: payload.slotNumber,
+      root: blockRootHex,
       envelopeBuilderIndex: envelope.builderIndex,
       bidBuilderIndex: payloadInput.getBuilderIndex(),
     });
@@ -100,6 +104,8 @@ async function validateExecutionPayloadEnvelope(
   if (toRootHex(payload.blockHash) !== payloadInput.getBlockHashHex()) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
       code: ExecutionPayloadEnvelopeErrorCode.BLOCK_HASH_MISMATCH,
+      slot: payload.slotNumber,
+      root: blockRootHex,
       envelopeBlockHash: toRootHex(payload.blockHash),
       bidBlockHash: payloadInput.getBlockHashHex(),
     });
@@ -110,6 +116,8 @@ async function validateExecutionPayloadEnvelope(
   if (!byteArrayEquals(requestsRoot, payloadInput.getBid().executionRequestsRoot)) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
       code: ExecutionPayloadEnvelopeErrorCode.EXECUTION_REQUESTS_ROOT_MISMATCH,
+      slot: payload.slotNumber,
+      root: blockRootHex,
       envelopeRequestsRoot: toRootHex(requestsRoot),
       bidRequestsRoot: toRootHex(payloadInput.getBid().executionRequestsRoot),
     });
@@ -141,6 +149,8 @@ async function validateExecutionPayloadEnvelope(
   if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
       code: ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE,
+      slot: payload.slotNumber,
+      root: blockRootHex,
     });
   }
 }

package/src/db/beacon.ts

@@ -1,6 +1,11 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {Db, LevelDbControllerMetrics, encodeKey} from "@lodestar/db";
+import {Slot} from "@lodestar/types";
+import {Logger} from "@lodestar/utils";
 import {Bucket} from "./buckets.js";
+import {type IDataColumnStore, LegacyDataColumnStore} from "./dataColumnStore.js";
+import {FlatFileStore} from "./flatFileStore/flatFileStore.js";
+import type {FlatFileStoreMetrics} from "./flatFileStore/metrics.js";
 import {IBeaconDb} from "./interface.js";
 import {CheckpointStateRepository} from "./repositories/checkpointState.js";
 import {
@@ -24,9 +29,10 @@ import {
   VoluntaryExitRepository,
 } from "./repositories/index.js";
 
-export type BeaconDbModules = {
-  config: ChainForkConfig;
-  db: Db;
+export type BeaconDbOpts = {
+  dataColumnDir: string;
+  logger: Logger;
+  metrics?: FlatFileStoreMetrics | null;
 };
 
 export class BeaconDb implements IBeaconDb {
@@ -57,9 +63,14 @@ export class BeaconDb implements IBeaconDb {
 
   backfilledRanges: BackfilledRanges;
 
+  readonly dataColumns: IDataColumnStore;
+  lastLegacyArchiveSlot: Slot | null = null;
+  private readonly flatFileStore: FlatFileStore;
+
   constructor(
     config: ChainForkConfig,
-    protected readonly db: Db
+    protected readonly db: Db,
+    opts: BeaconDbOpts
   ) {
     // Warning: If code is ever run in the constructor, must change this stub to not extend 'packages/beacon-node/test/utils/stub/beaconDb.ts' -
     this.block = new BlockRepository(config, db);
@@ -87,14 +98,30 @@ export class BeaconDb implements IBeaconDb {
     this.syncCommitteeWitness = new SyncCommitteeWitnessRepository(config, db);
 
     this.backfilledRanges = new BackfilledRanges(config, db);
+
+    this.flatFileStore = new FlatFileStore(opts.dataColumnDir, config, opts.logger, opts.metrics);
+    this.dataColumns = new LegacyDataColumnStore(
+      this.flatFileStore,
+      this.dataColumnSidecar,
+      this.dataColumnSidecarArchive,
+      this.blockArchive
+    );
+  }
+
+  async init(): Promise<void> {
+    await this.flatFileStore.init();
+    const [lastLegacyKey] = await this.dataColumnSidecarArchive.keys({reverse: true, limit: 1});
+    this.lastLegacyArchiveSlot = lastLegacyKey?.prefix ?? null;
   }
 
-  close(): Promise<void> {
+  async close(): Promise<void> {
+    await this.flatFileStore.close();
     return this.db.close();
   }
 
-  setMetrics(metrics: LevelDbControllerMetrics): void {
+  setMetrics(metrics: LevelDbControllerMetrics, flatFileStoreMetrics: FlatFileStoreMetrics | null = null): void {
     this.db.setMetrics(metrics);
+    this.flatFileStore.setMetrics(flatFileStoreMetrics);
   }
 
   async pruneHotDb(): Promise<void> {

package/src/db/dataColumnStore.ts

@@ -0,0 +1,113 @@
+import {ColumnIndex, DataColumnSidecar, RootHex, Slot} from "@lodestar/types";
+import {fromHex} from "@lodestar/utils";
+import type {IFlatFileStore} from "./flatFileStore/interface.js";
+import type {
+  BlockArchiveRepository,
+  DataColumnSidecarArchiveRepository,
+  DataColumnSidecarRepository,
+} from "./repositories/index.js";
+
+export type DataColumnKey = {slot: Slot; blockRoot: RootHex};
+export type IndexedDataColumnBytes = {index: ColumnIndex; data: Uint8Array};
+
+type LegacyHotDataColumnStore = Pick<DataColumnSidecarRepository, "values" | "getManyBinary" | "deleteMany">;
+type LegacyArchiveDataColumnStore = Pick<
+  DataColumnSidecarArchiveRepository,
+  "values" | "getManyBinary" | "keys" | "deleteMany"
+>;
+type BlockRootIndex = Pick<BlockArchiveRepository, "getSlotByRoot">;
+
+export interface IDataColumnStore {
+  getAll(key: DataColumnKey): Promise<DataColumnSidecar[]>;
+  getManyBinary(key: DataColumnKey, indices: ColumnIndex[]): Promise<(Uint8Array | undefined)[]>;
+  putManyBinary(key: DataColumnKey, columns: IndexedDataColumnBytes[]): Promise<void>;
+  deleteMany(keys: DataColumnKey[]): Promise<void>;
+  pruneBefore(slot: Slot): Promise<Slot[]>;
+}
+
+/**
+ * Bridges pre-flat-file LevelDB columns during the supported in-place upgrade window.
+ * Remove the legacy repositories after upgrades from the last pre-flat-file release are no longer supported.
+ */
+export class LegacyDataColumnStore implements IDataColumnStore {
+  constructor(
+    private readonly flatFiles: IFlatFileStore,
+    private readonly legacyHot: LegacyHotDataColumnStore,
+    private readonly legacyArchive: LegacyArchiveDataColumnStore,
+    private readonly blockArchive: BlockRootIndex
+  ) {}
+
+  async getAll({slot, blockRoot}: DataColumnKey): Promise<DataColumnSidecar[]> {
+    const sidecars = await this.flatFiles.getDataColumns(slot, blockRoot);
+    if (sidecars.length > 0) return sidecars;
+
+    const sidecarsByIndex = new Map<ColumnIndex, DataColumnSidecar>();
+    const root = fromHex(blockRoot);
+    for (const sidecar of await this.legacyHot.values(root)) {
+      if (!sidecarsByIndex.has(sidecar.index)) sidecarsByIndex.set(sidecar.index, sidecar);
+    }
+
+    if ((await this.blockArchive.getSlotByRoot(root)) === slot) {
+      for (const sidecar of await this.legacyArchive.values(slot)) {
+        if (!sidecarsByIndex.has(sidecar.index)) sidecarsByIndex.set(sidecar.index, sidecar);
+      }
+    }
+
+    return [...sidecarsByIndex.values()].sort((a, b) => a.index - b.index);
+  }
+
+  async getManyBinary({slot, blockRoot}: DataColumnKey, indices: ColumnIndex[]): Promise<(Uint8Array | undefined)[]> {
+    const flatFileSidecars = await this.flatFiles.getDataColumnsBinary(slot, blockRoot, indices);
+    if (flatFileSidecars !== null) return flatFileSidecars;
+    if (indices.length === 0) return [];
+
+    const root = fromHex(blockRoot);
+    const result = await this.legacyHot.getManyBinary(root, indices);
+
+    const missingPositions = getMissingPositions(result);
+    if (missingPositions.length === 0 || (await this.blockArchive.getSlotByRoot(root)) !== slot) return result;
+
+    const archivedSidecars = await this.legacyArchive.getManyBinary(
+      slot,
+      missingPositions.map((position) => indices[position])
+    );
+    fillMissing(result, missingPositions, archivedSidecars);
+    return result;
+  }
+
+  async putManyBinary({slot, blockRoot}: DataColumnKey, columns: IndexedDataColumnBytes[]): Promise<void> {
+    await this.flatFiles.putDataColumnsBinary(slot, blockRoot, columns);
+  }
+
+  async deleteMany(keys: DataColumnKey[]): Promise<void> {
+    await this.flatFiles.deleteMany(keys);
+    await this.legacyHot.deleteMany(keys.map(({blockRoot}) => fromHex(blockRoot)));
+  }
+
+  async pruneBefore(slot: Slot): Promise<Slot[]> {
+    const flatFileSlots = await this.flatFiles.pruneBefore(slot);
+    const prefixedKeys = await this.legacyArchive.keys({lt: {prefix: slot, id: 0}});
+    const slots = [...new Set(prefixedKeys.map(({prefix}) => prefix))];
+    if (slots.length > 0) await this.legacyArchive.deleteMany(slots);
+    return [...new Set([...flatFileSlots, ...slots])].sort((a, b) => a - b);
+  }
+}
+
+function getMissingPositions(sidecars: (Uint8Array | undefined)[]): number[] {
+  const positions: number[] = [];
+  for (let i = 0; i < sidecars.length; i++) {
+    if (sidecars[i] === undefined) positions.push(i);
+  }
+  return positions;
+}
+
+function fillMissing(
+  result: (Uint8Array | undefined)[],
+  missingPositions: number[],
+  sidecars: (Uint8Array | null | undefined)[]
+): void {
+  for (let i = 0; i < missingPositions.length; i++) {
+    const sidecar = sidecars[i];
+    if (sidecar != null) result[missingPositions[i]] = sidecar;
+  }
+}

package/src/db/flatFileStore/atomicWrite.ts

@@ -0,0 +1,65 @@
+import {mkdir, open, rename, rm} from "node:fs/promises";
+import path from "node:path";
+
+const pendingDirectorySyncs = new Map<string, string[]>();
+
+/**
+ * Atomic write: write to `.part` temp file -> datasync -> rename to target.
+ * This guarantees crash-safe writes — if the process crashes mid-write,
+ * only the `.part` file is corrupted, never the target.
+ */
+export async function atomicWrite(targetPath: string, data: Uint8Array): Promise<void> {
+  const dir = path.dirname(targetPath);
+  const partPath = `${targetPath}.part-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`;
+  await mkdirDurable(dir);
+
+  try {
+    const fd = await open(partPath, "wx");
+    try {
+      await fd.writeFile(data);
+      await fd.datasync();
+    } finally {
+      await fd.close();
+    }
+
+    await rename(partPath, targetPath);
+
+    // Ensure rename metadata is durable on crash.
+    await syncDirectory(dir);
+  } catch (e) {
+    await rm(partPath, {force: true}).catch(() => {});
+    throw e;
+  }
+}
+
+export async function mkdirDurable(dir: string): Promise<void> {
+  const targetDir = path.resolve(dir);
+  let directoriesToSync = pendingDirectorySyncs.get(targetDir);
+  if (directoriesToSync === undefined) {
+    const firstCreatedDir = await mkdir(targetDir, {recursive: true});
+    if (firstCreatedDir === undefined) return;
+
+    let createdDir = path.resolve(firstCreatedDir);
+    directoriesToSync = [path.dirname(createdDir)];
+    while (createdDir !== targetDir) {
+      directoriesToSync.push(createdDir);
+      const [nextSegment] = path.relative(createdDir, targetDir).split(path.sep);
+      createdDir = path.join(createdDir, nextSegment);
+    }
+    pendingDirectorySyncs.set(targetDir, directoriesToSync);
+  }
+
+  for (const directory of directoriesToSync) {
+    await syncDirectory(directory);
+  }
+  pendingDirectorySyncs.delete(targetDir);
+}
+
+async function syncDirectory(dir: string): Promise<void> {
+  const dirFd = await open(dir, "r");
+  try {
+    await dirFd.sync();
+  } finally {
+    await dirFd.close();
+  }
+}