@lodestar/beacon-node 1.48.0-dev.b71ab45af3 → 1.48.0-dev.bb3ffba9e9

package/src/chain/validation/dataColumnSidecar.ts

@@ -11,7 +11,7 @@ import {
   computeStartSlotAtEpoch,
   getBlockHeaderProposerSignatureSetByHeaderSlot,
 } from "@lodestar/state-transition";
-import {DataColumnSidecar, Root, Slot, SubnetID, ValidatorIndex, fulu, gloas, ssz} from "@lodestar/types";
+import {DataColumnSidecar, Root, RootHex, Slot, SubnetID, ValidatorIndex, fulu, gloas, ssz} from "@lodestar/types";
 import {byteArrayEquals, toRootHex, verifyMerkleBranch} from "@lodestar/utils";
 import {BeaconMetrics} from "../../metrics/metrics/beacon.js";
 import {Metrics} from "../../metrics/metrics.js";
@@ -39,12 +39,14 @@ export async function validateGossipFuluDataColumnSidecar(
   const blockRootHex = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(blockHeader));
 
   // 1) [REJECT] The sidecar is valid as verified by verify_data_column_sidecar
-  verifyFuluDataColumnSidecar(chain.config, dataColumnSidecar);
+  verifyFuluDataColumnSidecar(chain.config, dataColumnSidecar, blockRootHex);
 
   // 2) [REJECT] The sidecar is for the correct subnet -- i.e. compute_subnet_for_data_column_sidecar(sidecar.index) == subnet_id
   if (computeSubnetForDataColumnSidecar(chain.config, dataColumnSidecar) !== gossipSubnet) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_SUBNET,
+      slot: blockHeader.slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
       gossipSubnet: gossipSubnet,
     });
@@ -100,8 +102,9 @@ export async function validateGossipFuluDataColumnSidecar(
   if (parentBlock.slot >= blockHeader.slot) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.NOT_LATER_THAN_PARENT,
-      parentSlot: parentBlock.slot,
       slot: blockHeader.slot,
+      root: blockRootHex,
+      parentSlot: parentBlock.slot,
     });
   }
 
@@ -140,6 +143,8 @@ export async function validateGossipFuluDataColumnSidecar(
   if (proposerIndex !== expectedProposerIndex) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INCORRECT_PROPOSER,
+      slot: blockHeader.slot,
+      root: blockRootHex,
       actualProposerIndex: proposerIndex,
       expectedProposerIndex,
     });
@@ -161,9 +166,9 @@ export async function validateGossipFuluDataColumnSidecar(
     ) {
       throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
         code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
-        blockRoot: blockRootHex,
-        index: dataColumnSidecar.index,
         slot: blockHeader.slot,
+        root: blockRootHex,
+        index: dataColumnSidecar.index,
       });
     }
 
@@ -185,7 +190,8 @@ export async function validateGossipFuluDataColumnSidecar(
   if (!valid) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID,
-      slot: dataColumnSidecar.signedBlockHeader.message.slot,
+      slot: blockHeader.slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   }
@@ -204,6 +210,7 @@ export async function validateGossipFuluDataColumnSidecar(
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF,
       slot: blockHeader.slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   } finally {
@@ -240,6 +247,7 @@ export async function validateGossipGloasDataColumnSidecar(
   if (block.slot !== dataColumnSidecar.slot) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_SLOT,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
       expected: block.slot,
       actual: dataColumnSidecar.slot,
@@ -248,12 +256,14 @@ export async function validateGossipGloasDataColumnSidecar(
 
   // [REJECT] The sidecar must pass verify_data_column_sidecar against the block commitments
   const kzgCommitments = payloadInput.getBlobKzgCommitments();
-  verifyGloasDataColumnSidecar(dataColumnSidecar, kzgCommitments);
+  verifyGloasDataColumnSidecar(dataColumnSidecar, kzgCommitments, blockRootHex);
 
   // [REJECT] The sidecar must be on the correct subnet
   if (computeSubnetForDataColumnSidecar(chain.config, dataColumnSidecar) !== gossipSubnet) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_SUBNET,
+      slot: dataColumnSidecar.slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
       gossipSubnet,
     });
@@ -272,6 +282,7 @@ export async function validateGossipGloasDataColumnSidecar(
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF,
       slot: dataColumnSidecar.slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   } finally {
@@ -283,11 +294,17 @@ export async function validateGossipGloasDataColumnSidecar(
  * SPEC FUNCTION
  * https://github.com/ethereum/consensus-specs/blob/v1.6.0-alpha.4/specs/fulu/p2p-interface.md#verify_data_column_sidecar
  */
-function verifyFuluDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar: fulu.DataColumnSidecar): void {
+function verifyFuluDataColumnSidecar(
+  config: ChainForkConfig,
+  dataColumnSidecar: fulu.DataColumnSidecar,
+  blockRootHex: RootHex
+): void {
+  const slot = dataColumnSidecar.signedBlockHeader.message.slot;
   if (dataColumnSidecar.index >= NUMBER_OF_COLUMNS) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_INDEX,
-      slot: dataColumnSidecar.signedBlockHeader.message.slot,
+      slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   }
@@ -295,18 +312,20 @@ function verifyFuluDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar:
   if (dataColumnSidecar.kzgCommitments.length === 0) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.NO_COMMITMENTS,
-      slot: dataColumnSidecar.signedBlockHeader.message.slot,
+      slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   }
 
-  const epoch = computeEpochAtSlot(dataColumnSidecar.signedBlockHeader.message.slot);
+  const epoch = computeEpochAtSlot(slot);
   const maxBlobsPerBlock = config.getMaxBlobsPerBlock(epoch);
 
   if (dataColumnSidecar.kzgCommitments.length > maxBlobsPerBlock) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.TOO_MANY_KZG_COMMITMENTS,
-      slot: dataColumnSidecar.signedBlockHeader.message.slot,
+      slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
       count: dataColumnSidecar.kzgCommitments.length,
       limit: maxBlobsPerBlock,
@@ -319,6 +338,8 @@ function verifyFuluDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar:
   ) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.MISMATCHED_LENGTHS,
+      slot,
+      root: blockRootHex,
       columnLength: dataColumnSidecar.column.length,
       commitmentsLength: dataColumnSidecar.kzgCommitments.length,
       proofsLength: dataColumnSidecar.kzgProofs.length,
@@ -330,12 +351,17 @@ function verifyFuluDataColumnSidecar(config: ChainForkConfig, dataColumnSidecar:
  * SPEC FUNCTION
  * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.3/specs/gloas/p2p-interface.md#modified-verify_data_column_sidecar
  */
-function verifyGloasDataColumnSidecar(dataColumnSidecar: gloas.DataColumnSidecar, kzgCommitments: Uint8Array[]): void {
+function verifyGloasDataColumnSidecar(
+  dataColumnSidecar: gloas.DataColumnSidecar,
+  kzgCommitments: Uint8Array[],
+  blockRootHex: RootHex
+): void {
   const slot = getDataColumnSidecarSlot(dataColumnSidecar);
   if (dataColumnSidecar.index >= NUMBER_OF_COLUMNS) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.INVALID_INDEX,
       slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   }
@@ -344,6 +370,7 @@ function verifyGloasDataColumnSidecar(dataColumnSidecar: gloas.DataColumnSidecar
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.NO_COMMITMENTS,
       slot,
+      root: blockRootHex,
       columnIndex: dataColumnSidecar.index,
     });
   }
@@ -354,6 +381,8 @@ function verifyGloasDataColumnSidecar(dataColumnSidecar: gloas.DataColumnSidecar
   ) {
     throw new DataColumnSidecarGossipError(GossipAction.REJECT, {
       code: DataColumnSidecarErrorCode.MISMATCHED_LENGTHS,
+      slot,
+      root: blockRootHex,
       columnLength: dataColumnSidecar.column.length,
       commitmentsLength: kzgCommitments.length,
       proofsLength: dataColumnSidecar.kzgProofs.length,
@@ -465,8 +494,8 @@ export async function validateFuluBlockDataColumnSidecars(
         ) {
           throw new DataColumnSidecarValidationError({
             code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
-            blockRoot: rootHex,
             slot: blockSlot,
+            root: rootHex,
             index: dataColumnSidecars[0].index,
           });
         }
@@ -499,6 +528,7 @@ export async function validateFuluBlockDataColumnSidecars(
           {
             code: DataColumnSidecarErrorCode.INVALID_INDEX,
             slot: blockSlot,
+            root: toRootHex(blockRoot),
             columnIndex: columnSidecar.index,
           },
           "DataColumnSidecar has invalid index"
@@ -543,6 +573,7 @@ export async function validateFuluBlockDataColumnSidecars(
           {
             code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID,
             slot: blockSlot,
+            root: toRootHex(blockRoot),
             columnIndex: columnSidecar.index,
           },
           "DataColumnSidecar has invalid inclusion proof"
@@ -623,6 +654,7 @@ export async function validateGloasBlockDataColumnSidecars(
       if (columnSidecar.slot !== blockSlot) {
         throw new DataColumnSidecarValidationError({
           code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_SLOT,
+          root: toRootHex(blockRoot),
           columnIndex: columnSidecar.index,
           expected: blockSlot,
           actual: columnSidecar.slot,
@@ -644,6 +676,7 @@ export async function validateGloasBlockDataColumnSidecars(
           {
             code: DataColumnSidecarErrorCode.INVALID_INDEX,
             slot: blockSlot,
+            root: toRootHex(blockRoot),
             columnIndex: columnSidecar.index,
           },
           "DataColumnSidecar has invalid index"

package/src/chain/validation/executionPayloadEnvelope.ts

@@ -1,11 +1,4 @@
 import {PayloadStatus} from "@lodestar/fork-choice";
-import {
-  MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD,
-  MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD,
-  MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD,
-  MAX_WITHDRAWALS_PER_PAYLOAD,
-  MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD,
-} from "@lodestar/params";
 import {
   computeStartSlotAtEpoch,
   getExecutionPayloadEnvelopeSignatureSet,
@@ -89,6 +82,8 @@ async function validateExecutionPayloadEnvelope(
   if (block.slot !== payload.slotNumber) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
       code: ExecutionPayloadEnvelopeErrorCode.SLOT_MISMATCH,
+      slot: payload.slotNumber,
+      root: blockRootHex,
       envelopeSlot: payload.slotNumber,
       blockSlot: block.slot,
     });
@@ -98,6 +93,8 @@ async function validateExecutionPayloadEnvelope(
   if (envelope.builderIndex !== payloadInput.getBuilderIndex()) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
       code: ExecutionPayloadEnvelopeErrorCode.BUILDER_INDEX_MISMATCH,
+      slot: payload.slotNumber,
+      root: blockRootHex,
       envelopeBuilderIndex: envelope.builderIndex,
       bidBuilderIndex: payloadInput.getBuilderIndex(),
     });
@@ -107,6 +104,8 @@ async function validateExecutionPayloadEnvelope(
   if (toRootHex(payload.blockHash) !== payloadInput.getBlockHashHex()) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
       code: ExecutionPayloadEnvelopeErrorCode.BLOCK_HASH_MISMATCH,
+      slot: payload.slotNumber,
+      root: blockRootHex,
       envelopeBlockHash: toRootHex(payload.blockHash),
       bidBlockHash: payloadInput.getBlockHashHex(),
     });
@@ -117,41 +116,13 @@ async function validateExecutionPayloadEnvelope(
   if (!byteArrayEquals(requestsRoot, payloadInput.getBid().executionRequestsRoot)) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
       code: ExecutionPayloadEnvelopeErrorCode.EXECUTION_REQUESTS_ROOT_MISMATCH,
+      slot: payload.slotNumber,
+      root: blockRootHex,
       envelopeRequestsRoot: toRootHex(requestsRoot),
       bidRequestsRoot: toRootHex(payloadInput.getBid().executionRequestsRoot),
     });
   }
 
-  // [REJECT] The counts of `execution_requests` are within their respective limits.
-  // New in Gloas:EIP7688 — progressive lists are unbounded at the type level, so bounds
-  // are enforced here in gossip validation.
-  const {executionRequests} = envelope;
-  const requestCountLimits: [string, number, number][] = [
-    ["withdrawals", executionRequests.withdrawals.length, MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD],
-    ["consolidations", executionRequests.consolidations.length, MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD],
-    ["builderDeposits", executionRequests.builderDeposits.length, MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD],
-    ["builderExits", executionRequests.builderExits.length, MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD],
-  ];
-  for (const [name, count, limit] of requestCountLimits) {
-    if (count > limit) {
-      throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
-        code: ExecutionPayloadEnvelopeErrorCode.EXECUTION_REQUESTS_COUNT_EXCEEDED,
-        name,
-        count,
-        limit,
-      });
-    }
-  }
-
-  // [REJECT] The number of withdrawals is within the limit.
-  if (payload.withdrawals.length > MAX_WITHDRAWALS_PER_PAYLOAD) {
-    throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
-      code: ExecutionPayloadEnvelopeErrorCode.WITHDRAWALS_COUNT_EXCEEDED,
-      count: payload.withdrawals.length,
-      limit: MAX_WITHDRAWALS_PER_PAYLOAD,
-    });
-  }
-
   // Get the block state to verify the builder's signature.
   const blockState = await chain.regen
     .getState(block.stateRoot, RegenCaller.validateGossipPayloadEnvelope)
@@ -178,6 +149,8 @@ async function validateExecutionPayloadEnvelope(
   if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
       code: ExecutionPayloadEnvelopeErrorCode.INVALID_SIGNATURE,
+      slot: payload.slotNumber,
+      root: blockRootHex,
     });
   }
 }

package/src/chain/validatorMonitor.ts

@@ -1,5 +1,5 @@
 import {ChainForkConfig} from "@lodestar/config";
-import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params";
+import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
 import {
   IBeaconStateView,
   ParticipationFlags,
@@ -1117,7 +1117,11 @@ function isCanonical(rootCache: RootHexCache, block: AttestationBlockInclusion):
 
 /** Returns true if root at slot is the same at slot - 1 == there was no new block at slot */
 function isMissedSlot(rootCache: RootHexCache, slot: Slot): boolean {
-  return slot > 0 && rootCache.getBlockRootAtSlot(slot) === rootCache.getBlockRootAtSlot(slot - 1);
+  if (slot <= 0) {
+    return false;
+  }
+  const root = rootCache.getBlockRootAtSlot(slot);
+  return root !== null && root === rootCache.getBlockRootAtSlot(slot - 1);
 }
 
 function renderBlockProposalSummary(
@@ -1131,7 +1135,11 @@ function renderBlockProposalSummary(
     return "not_submitted";
   }
 
-  if (rootCache.getBlockRootAtSlot(proposalSlot) === proposal.blockRoot) {
+  const canonicalRoot = rootCache.getBlockRootAtSlot(proposalSlot);
+  if (canonicalRoot === null) {
+    return "unknown";
+  }
+  if (canonicalRoot === proposal.blockRoot) {
     // Canonical state includes our block
     return "canonical";
   }
@@ -1157,14 +1165,23 @@ function renderBlockProposalSummary(
  * In normal network conditions the same root is read multiple times, specially the target.
  */
 export class RootHexCache {
-  private readonly blockRootSlotCache = new Map<Slot, RootHex>();
+  private readonly blockRootSlotCache = new Map<Slot, RootHex | null>();
 
   constructor(private readonly state: IBeaconStateView) {}
 
-  getBlockRootAtSlot(slot: Slot): RootHex {
+  /**
+   * Block root at `slot`, null if the state does not cover it: slots at or after the state's own slot (an inclusion
+   * in a block the head state does not know yet, e.g. an orphaned block, or the slot after the last attestation of
+   * the epoch) or slots older than `SLOTS_PER_HISTORICAL_ROOT`. Callers treat null as unknown instead of throwing
+   * and aborting the summaries of all remaining validators.
+   */
+  getBlockRootAtSlot(slot: Slot): RootHex | null {
     let root = this.blockRootSlotCache.get(slot);
-    if (!root) {
-      root = toRootHex(this.state.getBlockRootAtSlot(slot));
+    if (root === undefined) {
+      root =
+        slot < this.state.slot && slot >= this.state.slot - SLOTS_PER_HISTORICAL_ROOT
+          ? toRootHex(this.state.getBlockRootAtSlot(slot))
+          : null;
       this.blockRootSlotCache.set(slot, root);
     }
     return root;

package/src/db/beacon.ts

@@ -1,6 +1,11 @@
 import {ChainForkConfig} from "@lodestar/config";
 import {Db, LevelDbControllerMetrics, encodeKey} from "@lodestar/db";
+import {Slot} from "@lodestar/types";
+import {Logger} from "@lodestar/utils";
 import {Bucket} from "./buckets.js";
+import {type IDataColumnStore, LegacyDataColumnStore} from "./dataColumnStore.js";
+import {FlatFileStore} from "./flatFileStore/flatFileStore.js";
+import type {FlatFileStoreMetrics} from "./flatFileStore/metrics.js";
 import {IBeaconDb} from "./interface.js";
 import {CheckpointStateRepository} from "./repositories/checkpointState.js";
 import {
@@ -24,9 +29,10 @@ import {
   VoluntaryExitRepository,
 } from "./repositories/index.js";
 
-export type BeaconDbModules = {
-  config: ChainForkConfig;
-  db: Db;
+export type BeaconDbOpts = {
+  dataColumnDir: string;
+  logger: Logger;
+  metrics?: FlatFileStoreMetrics | null;
 };
 
 export class BeaconDb implements IBeaconDb {
@@ -57,9 +63,14 @@ export class BeaconDb implements IBeaconDb {
 
   backfilledRanges: BackfilledRanges;
 
+  readonly dataColumns: IDataColumnStore;
+  lastLegacyArchiveSlot: Slot | null = null;
+  private readonly flatFileStore: FlatFileStore;
+
   constructor(
     config: ChainForkConfig,
-    protected readonly db: Db
+    protected readonly db: Db,
+    opts: BeaconDbOpts
   ) {
     // Warning: If code is ever run in the constructor, must change this stub to not extend 'packages/beacon-node/test/utils/stub/beaconDb.ts' -
     this.block = new BlockRepository(config, db);
@@ -87,14 +98,30 @@ export class BeaconDb implements IBeaconDb {
     this.syncCommitteeWitness = new SyncCommitteeWitnessRepository(config, db);
 
     this.backfilledRanges = new BackfilledRanges(config, db);
+
+    this.flatFileStore = new FlatFileStore(opts.dataColumnDir, config, opts.logger, opts.metrics);
+    this.dataColumns = new LegacyDataColumnStore(
+      this.flatFileStore,
+      this.dataColumnSidecar,
+      this.dataColumnSidecarArchive,
+      this.blockArchive
+    );
+  }
+
+  async init(): Promise<void> {
+    await this.flatFileStore.init();
+    const [lastLegacyKey] = await this.dataColumnSidecarArchive.keys({reverse: true, limit: 1});
+    this.lastLegacyArchiveSlot = lastLegacyKey?.prefix ?? null;
   }
 
-  close(): Promise<void> {
+  async close(): Promise<void> {
+    await this.flatFileStore.close();
     return this.db.close();
   }
 
-  setMetrics(metrics: LevelDbControllerMetrics): void {
+  setMetrics(metrics: LevelDbControllerMetrics, flatFileStoreMetrics: FlatFileStoreMetrics | null = null): void {
     this.db.setMetrics(metrics);
+    this.flatFileStore.setMetrics(flatFileStoreMetrics);
   }
 
   async pruneHotDb(): Promise<void> {

package/src/db/dataColumnStore.ts

@@ -0,0 +1,113 @@
+import {ColumnIndex, DataColumnSidecar, RootHex, Slot} from "@lodestar/types";
+import {fromHex} from "@lodestar/utils";
+import type {IFlatFileStore} from "./flatFileStore/interface.js";
+import type {
+  BlockArchiveRepository,
+  DataColumnSidecarArchiveRepository,
+  DataColumnSidecarRepository,
+} from "./repositories/index.js";
+
+export type DataColumnKey = {slot: Slot; blockRoot: RootHex};
+export type IndexedDataColumnBytes = {index: ColumnIndex; data: Uint8Array};
+
+type LegacyHotDataColumnStore = Pick<DataColumnSidecarRepository, "values" | "getManyBinary" | "deleteMany">;
+type LegacyArchiveDataColumnStore = Pick<
+  DataColumnSidecarArchiveRepository,
+  "values" | "getManyBinary" | "keys" | "deleteMany"
+>;
+type BlockRootIndex = Pick<BlockArchiveRepository, "getSlotByRoot">;
+
+export interface IDataColumnStore {
+  getAll(key: DataColumnKey): Promise<DataColumnSidecar[]>;
+  getManyBinary(key: DataColumnKey, indices: ColumnIndex[]): Promise<(Uint8Array | undefined)[]>;
+  putManyBinary(key: DataColumnKey, columns: IndexedDataColumnBytes[]): Promise<void>;
+  deleteMany(keys: DataColumnKey[]): Promise<void>;
+  pruneBefore(slot: Slot): Promise<Slot[]>;
+}
+
+/**
+ * Bridges pre-flat-file LevelDB columns during the supported in-place upgrade window.
+ * Remove the legacy repositories after upgrades from the last pre-flat-file release are no longer supported.
+ */
+export class LegacyDataColumnStore implements IDataColumnStore {
+  constructor(
+    private readonly flatFiles: IFlatFileStore,
+    private readonly legacyHot: LegacyHotDataColumnStore,
+    private readonly legacyArchive: LegacyArchiveDataColumnStore,
+    private readonly blockArchive: BlockRootIndex
+  ) {}
+
+  async getAll({slot, blockRoot}: DataColumnKey): Promise<DataColumnSidecar[]> {
+    const sidecars = await this.flatFiles.getDataColumns(slot, blockRoot);
+    if (sidecars.length > 0) return sidecars;
+
+    const sidecarsByIndex = new Map<ColumnIndex, DataColumnSidecar>();
+    const root = fromHex(blockRoot);
+    for (const sidecar of await this.legacyHot.values(root)) {
+      if (!sidecarsByIndex.has(sidecar.index)) sidecarsByIndex.set(sidecar.index, sidecar);
+    }
+
+    if ((await this.blockArchive.getSlotByRoot(root)) === slot) {
+      for (const sidecar of await this.legacyArchive.values(slot)) {
+        if (!sidecarsByIndex.has(sidecar.index)) sidecarsByIndex.set(sidecar.index, sidecar);
+      }
+    }
+
+    return [...sidecarsByIndex.values()].sort((a, b) => a.index - b.index);
+  }
+
+  async getManyBinary({slot, blockRoot}: DataColumnKey, indices: ColumnIndex[]): Promise<(Uint8Array | undefined)[]> {
+    const flatFileSidecars = await this.flatFiles.getDataColumnsBinary(slot, blockRoot, indices);
+    if (flatFileSidecars !== null) return flatFileSidecars;
+    if (indices.length === 0) return [];
+
+    const root = fromHex(blockRoot);
+    const result = await this.legacyHot.getManyBinary(root, indices);
+
+    const missingPositions = getMissingPositions(result);
+    if (missingPositions.length === 0 || (await this.blockArchive.getSlotByRoot(root)) !== slot) return result;
+
+    const archivedSidecars = await this.legacyArchive.getManyBinary(
+      slot,
+      missingPositions.map((position) => indices[position])
+    );
+    fillMissing(result, missingPositions, archivedSidecars);
+    return result;
+  }
+
+  async putManyBinary({slot, blockRoot}: DataColumnKey, columns: IndexedDataColumnBytes[]): Promise<void> {
+    await this.flatFiles.putDataColumnsBinary(slot, blockRoot, columns);
+  }
+
+  async deleteMany(keys: DataColumnKey[]): Promise<void> {
+    await this.flatFiles.deleteMany(keys);
+    await this.legacyHot.deleteMany(keys.map(({blockRoot}) => fromHex(blockRoot)));
+  }
+
+  async pruneBefore(slot: Slot): Promise<Slot[]> {
+    const flatFileSlots = await this.flatFiles.pruneBefore(slot);
+    const prefixedKeys = await this.legacyArchive.keys({lt: {prefix: slot, id: 0}});
+    const slots = [...new Set(prefixedKeys.map(({prefix}) => prefix))];
+    if (slots.length > 0) await this.legacyArchive.deleteMany(slots);
+    return [...new Set([...flatFileSlots, ...slots])].sort((a, b) => a - b);
+  }
+}
+
+function getMissingPositions(sidecars: (Uint8Array | undefined)[]): number[] {
+  const positions: number[] = [];
+  for (let i = 0; i < sidecars.length; i++) {
+    if (sidecars[i] === undefined) positions.push(i);
+  }
+  return positions;
+}
+
+function fillMissing(
+  result: (Uint8Array | undefined)[],
+  missingPositions: number[],
+  sidecars: (Uint8Array | null | undefined)[]
+): void {
+  for (let i = 0; i < missingPositions.length; i++) {
+    const sidecar = sidecars[i];
+    if (sidecar != null) result[missingPositions[i]] = sidecar;
+  }
+}

package/src/db/flatFileStore/atomicWrite.ts

@@ -0,0 +1,65 @@
+import {mkdir, open, rename, rm} from "node:fs/promises";
+import path from "node:path";
+
+const pendingDirectorySyncs = new Map<string, string[]>();
+
+/**
+ * Atomic write: write to `.part` temp file -> datasync -> rename to target.
+ * This guarantees crash-safe writes — if the process crashes mid-write,
+ * only the `.part` file is corrupted, never the target.
+ */
+export async function atomicWrite(targetPath: string, data: Uint8Array): Promise<void> {
+  const dir = path.dirname(targetPath);
+  const partPath = `${targetPath}.part-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`;
+  await mkdirDurable(dir);
+
+  try {
+    const fd = await open(partPath, "wx");
+    try {
+      await fd.writeFile(data);
+      await fd.datasync();
+    } finally {
+      await fd.close();
+    }
+
+    await rename(partPath, targetPath);
+
+    // Ensure rename metadata is durable on crash.
+    await syncDirectory(dir);
+  } catch (e) {
+    await rm(partPath, {force: true}).catch(() => {});
+    throw e;
+  }
+}
+
+export async function mkdirDurable(dir: string): Promise<void> {
+  const targetDir = path.resolve(dir);
+  let directoriesToSync = pendingDirectorySyncs.get(targetDir);
+  if (directoriesToSync === undefined) {
+    const firstCreatedDir = await mkdir(targetDir, {recursive: true});
+    if (firstCreatedDir === undefined) return;
+
+    let createdDir = path.resolve(firstCreatedDir);
+    directoriesToSync = [path.dirname(createdDir)];
+    while (createdDir !== targetDir) {
+      directoriesToSync.push(createdDir);
+      const [nextSegment] = path.relative(createdDir, targetDir).split(path.sep);
+      createdDir = path.join(createdDir, nextSegment);
+    }
+    pendingDirectorySyncs.set(targetDir, directoriesToSync);
+  }
+
+  for (const directory of directoriesToSync) {
+    await syncDirectory(directory);
+  }
+  pendingDirectorySyncs.delete(targetDir);
+}
+
+async function syncDirectory(dir: string): Promise<void> {
+  const dirFd = await open(dir, "r");
+  try {
+    await dirFd.sync();
+  } finally {
+    await dirFd.close();
+  }
+}