@lodestar/beacon-node 1.48.0-dev.e9c9b8835d → 1.48.0-dev.fd70e5fc15

package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts

@@ -1,11 +1,11 @@
 import {PeerId} from "@libp2p/interface";
 import {ChainConfig} from "@lodestar/config";
 import {PayloadStatus} from "@lodestar/fork-choice";
-import {ForkSeq, GENESIS_SLOT, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
+import {ForkSeq, GENESIS_SLOT} from "@lodestar/params";
 import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
-import {ColumnIndex, Epoch, RootHex, fulu} from "@lodestar/types";
-import {fromHex, toRootHex} from "@lodestar/utils";
+import {ColumnIndex, Epoch, fulu} from "@lodestar/types";
+import {fromHex} from "@lodestar/utils";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
 import {prettyPrintPeerId} from "../../util.js";
@@ -50,155 +50,121 @@ export async function* onDataColumnSidecarsByRange(
     );
   }
 
+  const finalized = db.dataColumnSidecarArchive;
   const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
-  // At Gloas, finalizing the beacon block does not finalize its payload. Keep the boundary block in the
-  // fork-choice-backed section until the following finalization, matching ExecutionPayloadEnvelopesByRange.
+  // Columns of the last finalized block live in different DBs depending on fork:
+  // - Pre-gloas (fulu): migrated to dataColumnSidecarArchive in the same finalization run.
+  // - Post-gloas: stay in the hot db (db.dataColumnSidecar) until the next finalization run,
+  //   because the migration filter requires payloadStatus === FULL for gloas blocks.
+  // archiveMaxSlot is the last slot whose columns are served by the archive loop below;
+  // anything above it is served by the headChain loop.
   const isPostGloasFinalized = chain.config.getForkSeq(finalizedSlot) >= ForkSeq.gloas;
   const archiveMaxSlot = isPostGloasFinalized ? finalizedSlot - 1 : finalizedSlot;
-  const headBlock = chain.forkChoice.getHead();
-  const headChain = chain.forkChoice.getAllAncestorBlocks(headBlock.blockRoot, headBlock.payloadStatus);
 
-  for await (const block of resolveCanonicalDataColumnBlocks(
-    chain,
-    db,
-    headChain,
-    startSlot,
-    endSlot,
-    archiveMaxSlot,
-    finalizedSlot
-  )) {
-    const dataColumnSidecars =
-      block.blockRoot === null
-        ? await db.dataColumnSidecarArchive.getManyBinary(block.slot, availableColumns)
-        : await chain.getSerializedDataColumnSidecars(block.slot, block.blockRoot, availableColumns);
-    if (block.blockRoot === null && dataColumnSidecars.every((sidecar) => sidecar === undefined)) continue;
-    const unavailableColumnIndices: ColumnIndex[] = [];
-    for (let i = 0; i < dataColumnSidecars.length; i++) {
-      const dataColumnSidecarBytes = dataColumnSidecars[i];
-      if (dataColumnSidecarBytes) {
-        yield {
-          data: dataColumnSidecarBytes,
-          boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(block.slot)),
-        };
-      } else {
-        unavailableColumnIndices.push(availableColumns[i]);
+  // Finalized range of columns
+  if (startSlot <= archiveMaxSlot) {
+    const archiveEnd = Math.min(endSlot, archiveMaxSlot + 1);
+    for (let slot = startSlot; slot < archiveEnd; slot++) {
+      const dataColumnSidecars = await finalized.getManyBinary(slot, availableColumns);
+
+      const unavailableColumnIndices: ColumnIndex[] = [];
+      for (let i = 0; i < dataColumnSidecars.length; i++) {
+        const dataColumnSidecarBytes = dataColumnSidecars[i];
+        if (dataColumnSidecarBytes) {
+          yield {
+            data: dataColumnSidecarBytes,
+            boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(slot)),
+          };
+        }
+
+        // TODO: Check blobs for that block and respond resource_unavailable
+        // After we have consensus from other teams on the specs
+        else {
+          unavailableColumnIndices.push(availableColumns[i]);
+        }
       }
-    }
-
-    if (unavailableColumnIndices.length > 0) {
-      await handleColumnSidecarUnavailability({
-        chain,
-        db,
-        metrics: chain.metrics,
-        unavailableColumnIndices,
-        blockRoot: block.unavailabilityBlockRoot ? fromHex(block.unavailabilityBlockRoot) : undefined,
-        finalized: block.finalized,
-        slot: block.slot,
-        requestedColumns,
-        availableColumns,
-      });
-    }
-  }
-}
 
-type CanonicalDataColumnBlock = {
-  slot: number;
-  blockRoot: RootHex | null;
-  unavailabilityBlockRoot?: RootHex;
-  finalized: boolean;
-};
-
-async function* resolveCanonicalDataColumnBlocks(
-  chain: IBeaconChain,
-  db: IBeaconDb,
-  headChain: ReturnType<IBeaconChain["forkChoice"]["getAllAncestorBlocks"]>,
-  startSlot: number,
-  endSlot: number,
-  archiveMaxSlot: number,
-  finalizedSlot: number
-): AsyncIterable<CanonicalDataColumnBlock> {
-  const canonicalBlocksBySlot = new Map(headChain.map((block) => [block.slot, block]));
-  const oldestForkChoiceSlot = headChain.at(-1)?.slot ?? Number.POSITIVE_INFINITY;
-  const archiveEnd = Math.min(endSlot, archiveMaxSlot + 1);
-  const recentRoots = getRecentCanonicalRoots(chain, startSlot, Math.min(archiveEnd, oldestForkChoiceSlot));
-
-  for (let slot = startSlot; slot < archiveEnd; slot++) {
-    if (slot >= oldestForkChoiceSlot) {
-      const block = canonicalBlocksBySlot.get(slot);
-      if (block?.payloadStatus === PayloadStatus.FULL) {
-        yield {
+      if (unavailableColumnIndices.length) {
+        await handleColumnSidecarUnavailability({
+          chain,
+          db,
+          metrics: chain.metrics,
+          unavailableColumnIndices,
           slot,
-          blockRoot: block.blockRoot,
-          unavailabilityBlockRoot: block.blockRoot,
-          finalized: true,
-        };
+          requestedColumns,
+          availableColumns,
+        });
       }
-      continue;
-    }
-
-    let blockRoot = recentRoots.get(slot);
-    if (blockRoot === null) continue;
-    if (blockRoot === undefined) {
-      const root = await db.blockArchive.getRootBySlot(slot);
-      blockRoot = root === null ? null : toRootHex(root);
-    }
-    if (blockRoot === null && (db.lastLegacyArchiveSlot === null || slot > db.lastLegacyArchiveSlot)) continue;
-    if (
-      chain.config.getForkSeq(slot) >= ForkSeq.gloas &&
-      !(await hasExecutionPayloadEnvelope(chain, db, slot, blockRoot))
-    ) {
-      continue;
     }
-    yield {
-      slot,
-      blockRoot,
-      unavailabilityBlockRoot: chain.config.getForkSeq(slot) >= ForkSeq.gloas ? (blockRoot ?? undefined) : undefined,
-      finalized: true,
-    };
   }
 
-  for (let i = headChain.length - 1; i >= 0; i--) {
-    const block = headChain[i];
-    if (block.slot >= endSlot) break;
-    if (block.slot <= archiveMaxSlot || block.slot < startSlot || block.payloadStatus !== PayloadStatus.FULL) continue;
-    yield {
-      slot: block.slot,
-      blockRoot: block.blockRoot,
-      unavailabilityBlockRoot: block.blockRoot,
-      finalized: block.slot <= finalizedSlot,
-    };
-  }
-}
-
-/** Copy roots synchronously so serving a request never retains a head-state reference across I/O. */
-function getRecentCanonicalRoots(chain: IBeaconChain, startSlot: number, endSlot: number): Map<number, RootHex | null> {
-  const roots = new Map<number, RootHex | null>();
-  if (startSlot >= endSlot) return roots;
-  const state = chain.getHeadState();
-  // Skipped slots repeat the prior root. At the oldest ring entry, that prior root is unavailable.
-  const start = Math.max(startSlot, state.slot - SLOTS_PER_HISTORICAL_ROOT + 1, 1);
-  const end = Math.min(endSlot, state.slot);
-  if (start >= end) return roots;
-  let previousRoot = toRootHex(state.getBlockRootAtSlot(start - 1));
-  for (let slot = start; slot < end; slot++) {
-    const root = toRootHex(state.getBlockRootAtSlot(slot));
-    roots.set(slot, root === previousRoot ? null : root);
-    previousRoot = root;
-  }
-  return roots;
-}
+  // Non-finalized range of columns
+  if (endSlot > archiveMaxSlot) {
+    const headBlock = chain.forkChoice.getHead();
+    const headRoot = headBlock.blockRoot;
+    // getAllAncestorBlocks includes the last finalized block as its final element.
+    // Skip anything the archive loop above already served via the block.slot > archiveMaxSlot
+    // filter below (pre-gloas this skips finalizedSlot, post-gloas it keeps it).
+    const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
+
+    // Iterate head chain with ascending block numbers
+    for (let i = headChain.length - 1; i >= 0; i--) {
+      const block = headChain[i];
+
+      // Must include only columns in the range requested
+      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
+        // Post-gloas, columns exist only for FULL blocks (pre-gloas blocks are always FULL)
+        if (block.payloadStatus !== PayloadStatus.FULL) {
+          continue;
+        }
+
+        // Note: Here the forkChoice head may change due to a re-org, so the headChain reflects the canonical chain
+        // at the time of the start of the request. Spec is clear the chain of columns must be consistent, but on
+        // re-org there's no need to abort the request
+        // Spec: https://github.com/ethereum/consensus-specs/blob/ad36024441cf910d428d03f87f331fbbd2b3e5f1/specs/fulu/p2p-interface.md#L425-L429
+        const dataColumnSidecars = await chain.getSerializedDataColumnSidecars(
+          block.slot,
+          block.blockRoot,
+          availableColumns
+        );
+
+        const unavailableColumnIndices: ColumnIndex[] = [];
+        for (let i = 0; i < dataColumnSidecars.length; i++) {
+          const dataColumnSidecarBytes = dataColumnSidecars[i];
+          if (dataColumnSidecarBytes) {
+            yield {
+              data: dataColumnSidecarBytes,
+              boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(block.slot)),
+            };
+          }
+
+          // TODO: Check blobs for that block and respond resource_unavailable
+          // After we have consensus from other teams on the specs
+          else {
+            unavailableColumnIndices.push(availableColumns[i]);
+          }
+        }
+
+        if (unavailableColumnIndices.length) {
+          await handleColumnSidecarUnavailability({
+            chain,
+            db,
+            metrics: chain.metrics,
+            unavailableColumnIndices,
+            blockRoot: fromHex(block.blockRoot),
+            slot: block.slot,
+            requestedColumns,
+            availableColumns,
+          });
+        }
+      }
 
-async function hasExecutionPayloadEnvelope(
-  chain: IBeaconChain,
-  db: IBeaconDb,
-  slot: number,
-  blockRoot: RootHex | null
-): Promise<boolean> {
-  if (blockRoot !== null) {
-    if (chain.seenPayloadEnvelopeInputCache.hasPayload(blockRoot)) return true;
-    if ((await db.executionPayloadEnvelope.getBinary(fromHex(blockRoot))) !== null) return true;
+      // If block is after endSlot, stop iterating
+      else if (block.slot >= endSlot) {
+        break;
+      }
+    }
   }
-  return (await db.executionPayloadEnvelopeArchive.getBinary(slot)) !== null;
 }
 
 export function validateDataColumnSidecarsByRangeRequest(

package/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts

@@ -81,14 +81,12 @@ export async function* onDataColumnSidecarsByRoot(
     }
 
     if (unavailableColumnIndices.length) {
-      const finalized = !block || (await db.blockArchive.getSlotByRoot(blockRoot)) === slot;
       await handleColumnSidecarUnavailability({
         chain,
         db,
         metrics: chain.metrics,
         slot,
         blockRoot,
-        finalized,
         unavailableColumnIndices,
         requestedColumns,
         availableColumns,

package/src/network/reqresp/protocols.ts

@@ -1,5 +1,5 @@
 import {BeaconConfig} from "@lodestar/config";
-import {ForkName} from "@lodestar/params";
+import {ForkName, MAX_DATA_COLUMN_SIDECAR_SIZE, isForkPostGloas} from "@lodestar/params";
 import {ContextBytesFactory, ContextBytesType, Encoding, TypeSizes} from "@lodestar/reqresp";
 import {rateLimitQuotas} from "./rateLimit.js";
 import {ProtocolNoHandler, ReqRespMethod, Version, requestSszTypeByMethod, responseSszTypeByMethod} from "./types.js";
@@ -151,18 +151,35 @@ function toProtocol(protocol: ProtocolSummary) {
       encoding: Encoding.SSZ_SNAPPY,
       contextBytes: toContextBytes(protocol.contextBytesType, config),
       inboundRateLimits: rateLimitQuotas(fork, config)[protocol.method],
-      requestSizes: requestType === null ? null : clampTypeSizes(requestType, config),
-      responseSizes: (fork) => clampTypeSizes(responseSszTypeByMethod[protocol.method](fork, protocol.version), config),
+      requestSizes: requestType === null ? null : clampTypeSizes(requestType, protocol.method, fork, config),
+      responseSizes: (fork) =>
+        clampTypeSizes(responseSszTypeByMethod[protocol.method](fork, protocol.version), protocol.method, fork, config),
     };
   };
 }
 
 /**
- * Length-prefix must be within the SSZ type bounds or MAX_PAYLOAD_SIZE, whichever is smaller
- * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.12/specs/phase0/p2p-interface.md#encoding-strategies
+ * Bound the sizes accepted from the ssz-snappy length-prefix. Gloas progressive containers have broad
+ * theoretical SSZ max sizes so the preset p2p bounds must be used instead.
+ *
+ * The length-prefix must be within the size bounds derived from the payload SSZ type or `MAX_PAYLOAD_SIZE`,
+ * whichever is smaller, see
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.12/specs/phase0/p2p-interface.md#encoding-strategies.
+ * Type-specific SSZ bounds supersede the bounds derived from the SSZ type, see
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.12/specs/gloas/p2p-interface.md#type-specific-ssz-bounds.
  */
-function clampTypeSizes(type: TypeSizes, config: BeaconConfig): TypeSizes {
-  return {minSize: type.minSize, maxSize: Math.min(type.maxSize, config.MAX_PAYLOAD_SIZE)};
+function clampTypeSizes(type: TypeSizes, method: ReqRespMethod, fork: ForkName, config: BeaconConfig): TypeSizes {
+  let typeSpecificBound = config.MAX_PAYLOAD_SIZE;
+  if (isForkPostGloas(fork)) {
+    switch (method) {
+      case ReqRespMethod.DataColumnSidecarsByRange:
+      case ReqRespMethod.DataColumnSidecarsByRoot:
+        typeSpecificBound = MAX_DATA_COLUMN_SIDECAR_SIZE;
+        break;
+    }
+  }
+
+  return {minSize: type.minSize, maxSize: Math.min(type.maxSize, typeSpecificBound)};
 }
 
 function toContextBytes(type: ContextBytesType, config: BeaconConfig): ContextBytesFactory {

package/src/network/reqresp/utils/dataColumnResponseValidation.ts

@@ -16,14 +16,12 @@ export async function handleColumnSidecarUnavailability({
   availableColumns,
   slot,
   blockRoot,
-  finalized,
 }: {
   chain: IBeaconChain;
   db: IBeaconDb;
   metrics: Metrics | null;
   slot: Slot;
   blockRoot?: Uint8Array;
-  finalized: boolean;
   unavailableColumnIndices: ColumnIndex[];
   requestedColumns: ColumnIndex[];
   availableColumns: ColumnIndex[];
@@ -40,23 +38,17 @@ export async function handleColumnSidecarUnavailability({
 
   chain.logger.debug("dataColumnSidecar requested unavailable", logData);
 
-  // Post-gloas, columns exist only for FULL blocks. The envelope may still be hot while finalization
-  // archiving is in progress. Bid blobsCount is unreliable since an EMPTY block's bid may still commit to blobs.
-  if (chain.config.getForkSeq(slot) >= ForkSeq.gloas) {
-    const hasCachedEnvelope = blockRoot ? chain.seenPayloadEnvelopeInputCache.hasPayload(toRootHex(blockRoot)) : false;
-    if (!hasCachedEnvelope) {
-      const hotEnvelopeBytes = blockRoot ? await db.executionPayloadEnvelope.getBinary(blockRoot) : null;
-      const envelopeBytes =
-        hotEnvelopeBytes ?? (finalized ? await db.executionPayloadEnvelopeArchive.getBinary(slot) : null);
-      if (!envelopeBytes) return;
-    }
+  // Post-gloas, columns exist only for FULL blocks; a finalized block is FULL if its envelope was
+  // archived. Bid blobsCount is unreliable here since an EMPTY block's bid may still commit to blobs
+  if (blockRoot === undefined && chain.config.getForkSeq(slot) >= ForkSeq.gloas) {
+    const envelopeBytes = await db.executionPayloadEnvelopeArchive.getBinary(slot);
+    if (!envelopeBytes) return;
   }
 
-  const hotBlockBytes = blockRoot ? await db.block.getBinary(blockRoot) : null;
-  const blockBytes = hotBlockBytes ?? (finalized ? await db.blockArchive.getBinary(slot) : null);
+  const blockBytes = blockRoot ? await db.block.getBinary(blockRoot) : await db.blockArchive.getBinary(slot);
   if (!blockBytes) {
     chain.logger.verbose(
-      `Expected ${finalized ? "finalized" : "unfinalized"} block not found while handling unavailable dataColumnSidecar`,
+      `Expected ${blockRoot ? "unfinalized" : "finalized"} block not found while handling unavailable dataColumnSidecar`,
       {
         slot,
         blockRoot: blockRoot ? toRootHex(blockRoot) : "unknown",

package/src/node/nodejs.ts

@@ -54,7 +54,6 @@ export type BeaconNodeInitModules = {
   processShutdownCallback: ProcessShutdownCallback;
   privateKey: PrivateKey;
   dataDir: string;
-  dataColumnDir: string;
   peerStoreDir?: string;
   anchorState: IBeaconStateView;
   isAnchorStateFinalized: boolean;
@@ -157,7 +156,6 @@ export class BeaconNode {
     processShutdownCallback,
     privateKey,
     dataDir,
-    dataColumnDir,
     peerStoreDir,
     anchorState,
     isAnchorStateFinalized,
@@ -183,7 +181,7 @@ export class BeaconNode {
       metrics = createMetrics(opts.metrics, anchorState.genesisTime, metricsRegistries);
       initBeaconMetrics(metrics, anchorState);
       // Since the db is instantiated before this, metrics must be injected manually afterwards
-      db.setMetrics(metrics.db, metrics.flatFileStore);
+      db.setMetrics(metrics.db);
       signal.addEventListener("abort", metrics.close, {once: true});
     }
 
@@ -200,8 +198,6 @@ export class BeaconNode {
 
     const clock = new Clock({config, genesisTime: anchorState.genesisTime, signal});
 
-    await db.init();
-
     // Prune hot db repos
     // TODO: Should this call be awaited?
     await db.pruneHotDb();
@@ -248,7 +244,6 @@ export class BeaconNode {
       clock,
       pubkeyCache,
       dataDir,
-      dataColumnDir,
       db,
       dbName: opts.db.name,
       logger: logger.child({module: LoggerModule.chain}),

package/src/util/sszBytes.ts

@@ -754,7 +754,7 @@ export function getParentBlockRootFromSignedExecutionPayloadBidSerialized(data:
  *
  * If the high bytes are not zero, return null
  */
-export function getSlotFromOffset(data: Uint8Array, offset: number): Slot | null {
+function getSlotFromOffset(data: Uint8Array, offset: number): Slot | null {
   return checkSlotHighBytes(data, offset) ? getSlotFromOffsetTrusted(data, offset) : null;
 }
 

package/lib/db/dataColumnStore.d.ts

@@ -1,39 +0,0 @@
-import { ColumnIndex, DataColumnSidecar, RootHex, Slot } from "@lodestar/types";
-import type { IFlatFileStore } from "./flatFileStore/interface.js";
-import type { BlockArchiveRepository, DataColumnSidecarArchiveRepository, DataColumnSidecarRepository } from "./repositories/index.js";
-export type DataColumnKey = {
-    slot: Slot;
-    blockRoot: RootHex;
-};
-export type IndexedDataColumnBytes = {
-    index: ColumnIndex;
-    data: Uint8Array;
-};
-type LegacyHotDataColumnStore = Pick<DataColumnSidecarRepository, "values" | "getManyBinary" | "deleteMany">;
-type LegacyArchiveDataColumnStore = Pick<DataColumnSidecarArchiveRepository, "values" | "getManyBinary" | "keys" | "deleteMany">;
-type BlockRootIndex = Pick<BlockArchiveRepository, "getSlotByRoot">;
-export interface IDataColumnStore {
-    getAll(key: DataColumnKey): Promise<DataColumnSidecar[]>;
-    getManyBinary(key: DataColumnKey, indices: ColumnIndex[]): Promise<(Uint8Array | undefined)[]>;
-    putManyBinary(key: DataColumnKey, columns: IndexedDataColumnBytes[]): Promise<void>;
-    deleteMany(keys: DataColumnKey[]): Promise<void>;
-    pruneBefore(slot: Slot): Promise<Slot[]>;
-}
-/**
- * Bridges pre-flat-file LevelDB columns during the supported in-place upgrade window.
- * Remove the legacy repositories after upgrades from the last pre-flat-file release are no longer supported.
- */
-export declare class LegacyDataColumnStore implements IDataColumnStore {
-    private readonly flatFiles;
-    private readonly legacyHot;
-    private readonly legacyArchive;
-    private readonly blockArchive;
-    constructor(flatFiles: IFlatFileStore, legacyHot: LegacyHotDataColumnStore, legacyArchive: LegacyArchiveDataColumnStore, blockArchive: BlockRootIndex);
-    getAll({ slot, blockRoot }: DataColumnKey): Promise<DataColumnSidecar[]>;
-    getManyBinary({ slot, blockRoot }: DataColumnKey, indices: ColumnIndex[]): Promise<(Uint8Array | undefined)[]>;
-    putManyBinary({ slot, blockRoot }: DataColumnKey, columns: IndexedDataColumnBytes[]): Promise<void>;
-    deleteMany(keys: DataColumnKey[]): Promise<void>;
-    pruneBefore(slot: Slot): Promise<Slot[]>;
-}
-export {};
-//# sourceMappingURL=dataColumnStore.d.ts.map

package/lib/db/dataColumnStore.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"dataColumnStore.d.ts","sourceRoot":"","sources":["../../src/db/dataColumnStore.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,WAAW,EAAE,iBAAiB,EAAE,OAAO,EAAE,IAAI,EAAC,MAAM,iBAAiB,CAAC;AAE9E,OAAO,KAAK,EAAC,cAAc,EAAC,MAAM,8BAA8B,CAAC;AACjE,OAAO,KAAK,EACV,sBAAsB,EACtB,kCAAkC,EAClC,2BAA2B,EAC5B,MAAM,yBAAyB,CAAC;AAEjC,MAAM,MAAM,aAAa,GAAG;IAAC,IAAI,EAAE,IAAI,CAAC;IAAC,SAAS,EAAE,OAAO,CAAA;CAAC,CAAC;AAC7D,MAAM,MAAM,sBAAsB,GAAG;IAAC,KAAK,EAAE,WAAW,CAAC;IAAC,IAAI,EAAE,UAAU,CAAA;CAAC,CAAC;AAE5E,KAAK,wBAAwB,GAAG,IAAI,CAAC,2BAA2B,EAAE,QAAQ,GAAG,eAAe,GAAG,YAAY,CAAC,CAAC;AAC7G,KAAK,4BAA4B,GAAG,IAAI,CACtC,kCAAkC,EAClC,QAAQ,GAAG,eAAe,GAAG,MAAM,GAAG,YAAY,CACnD,CAAC;AACF,KAAK,cAAc,GAAG,IAAI,CAAC,sBAAsB,EAAE,eAAe,CAAC,CAAC;AAEpE,MAAM,WAAW,gBAAgB;IAC/B,MAAM,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;IACzD,aAAa,CAAC,GAAG,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC,EAAE,CAAC,CAAC;IAC/F,aAAa,CAAC,GAAG,EAAE,aAAa,EAAE,OAAO,EAAE,sBAAsB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpF,UAAU,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACjD,WAAW,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;CAC1C;AAED;;;GAGG;AACH,qBAAa,qBAAsB,YAAW,gBAAgB;IAE1D,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,SAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,YAAY;IAJ/B,YACmB,SAAS,EAAE,cAAc,EACzB,SAAS,EAAE,wBAAwB,EACnC,aAAa,EAAE,4BAA4B,EAC3C,YAAY,EAAE,cAAc,EAC3C;IAEE,MAAM,CAAC,EAAC,IAAI,EAAE,SAAS,EAAC,EAAE,aAAa,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAiB3E;IAEK,aAAa,CAAC,EAAC,IAAI,EAAE,SAAS,EAAC,EAAE,aAAa,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,CAAC,UAAU,GAAG,SAAS,CAAC,EAAE,CAAC,CAiBjH;IAEK,aAAa,CAAC,EAAC,IAAI,EAAE,SAAS,EAAC,EAAE,aAAa,EAAE,OAAO,EAAE,sBAAsB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAEtG;IAEK,UAAU,CAAC,IAAI,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAGrD;IAEK,WAAW,CAAC,IAAI,EAAE,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAM7C;CACF"}

package/lib/db/dataColumnStore.js

@@ -1,81 +0,0 @@
-import { fromHex } from "@lodestar/utils";
-/**
- * Bridges pre-flat-file LevelDB columns during the supported in-place upgrade window.
- * Remove the legacy repositories after upgrades from the last pre-flat-file release are no longer supported.
- */
-export class LegacyDataColumnStore {
-    flatFiles;
-    legacyHot;
-    legacyArchive;
-    blockArchive;
-    constructor(flatFiles, legacyHot, legacyArchive, blockArchive) {
-        this.flatFiles = flatFiles;
-        this.legacyHot = legacyHot;
-        this.legacyArchive = legacyArchive;
-        this.blockArchive = blockArchive;
-    }
-    async getAll({ slot, blockRoot }) {
-        const sidecars = await this.flatFiles.getDataColumns(slot, blockRoot);
-        if (sidecars.length > 0)
-            return sidecars;
-        const sidecarsByIndex = new Map();
-        const root = fromHex(blockRoot);
-        for (const sidecar of await this.legacyHot.values(root)) {
-            if (!sidecarsByIndex.has(sidecar.index))
-                sidecarsByIndex.set(sidecar.index, sidecar);
-        }
-        if ((await this.blockArchive.getSlotByRoot(root)) === slot) {
-            for (const sidecar of await this.legacyArchive.values(slot)) {
-                if (!sidecarsByIndex.has(sidecar.index))
-                    sidecarsByIndex.set(sidecar.index, sidecar);
-            }
-        }
-        return [...sidecarsByIndex.values()].sort((a, b) => a.index - b.index);
-    }
-    async getManyBinary({ slot, blockRoot }, indices) {
-        const flatFileSidecars = await this.flatFiles.getDataColumnsBinary(slot, blockRoot, indices);
-        if (flatFileSidecars !== null)
-            return flatFileSidecars;
-        if (indices.length === 0)
-            return [];
-        const root = fromHex(blockRoot);
-        const result = await this.legacyHot.getManyBinary(root, indices);
-        const missingPositions = getMissingPositions(result);
-        if (missingPositions.length === 0 || (await this.blockArchive.getSlotByRoot(root)) !== slot)
-            return result;
-        const archivedSidecars = await this.legacyArchive.getManyBinary(slot, missingPositions.map((position) => indices[position]));
-        fillMissing(result, missingPositions, archivedSidecars);
-        return result;
-    }
-    async putManyBinary({ slot, blockRoot }, columns) {
-        await this.flatFiles.putDataColumnsBinary(slot, blockRoot, columns);
-    }
-    async deleteMany(keys) {
-        await this.flatFiles.deleteMany(keys);
-        await this.legacyHot.deleteMany(keys.map(({ blockRoot }) => fromHex(blockRoot)));
-    }
-    async pruneBefore(slot) {
-        const flatFileSlots = await this.flatFiles.pruneBefore(slot);
-        const prefixedKeys = await this.legacyArchive.keys({ lt: { prefix: slot, id: 0 } });
-        const slots = [...new Set(prefixedKeys.map(({ prefix }) => prefix))];
-        if (slots.length > 0)
-            await this.legacyArchive.deleteMany(slots);
-        return [...new Set([...flatFileSlots, ...slots])].sort((a, b) => a - b);
-    }
-}
-function getMissingPositions(sidecars) {
-    const positions = [];
-    for (let i = 0; i < sidecars.length; i++) {
-        if (sidecars[i] === undefined)
-            positions.push(i);
-    }
-    return positions;
-}
-function fillMissing(result, missingPositions, sidecars) {
-    for (let i = 0; i < missingPositions.length; i++) {
-        const sidecar = sidecars[i];
-        if (sidecar != null)
-            result[missingPositions[i]] = sidecar;
-    }
-}
-//# sourceMappingURL=dataColumnStore.js.map

package/lib/db/dataColumnStore.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"dataColumnStore.js","sourceRoot":"","sources":["../../src/db/dataColumnStore.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,OAAO,EAAC,MAAM,iBAAiB,CAAC;AA0BxC;;;GAGG;AACH,MAAM,OAAO,qBAAqB;IAEb,SAAS;IACT,SAAS;IACT,aAAa;IACb,YAAY;IAJ/B,YACmB,SAAyB,EACzB,SAAmC,EACnC,aAA2C,EAC3C,YAA4B;yBAH5B,SAAS;yBACT,SAAS;6BACT,aAAa;4BACb,YAAY;IAC5B,CAAC;IAEJ,KAAK,CAAC,MAAM,CAAC,EAAC,IAAI,EAAE,SAAS,EAAgB;QAC3C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,cAAc,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QACtE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,QAAQ,CAAC;QAEzC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkC,CAAC;QAClE,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,KAAK,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACxD,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC;gBAAE,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACvF,CAAC;QAED,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3D,KAAK,MAAM,OAAO,IAAI,MAAM,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC5D,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC;oBAAE,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,eAAe,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAC,IAAI,EAAE,SAAS,EAAgB,EAAE,OAAsB;QAC1E,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC7F,IAAI,gBAAgB,KAAK,IAAI;YAAE,OAAO,gBAAgB,CAAC;QACvD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEpC,MAAM,IAAI,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAEjE,MAAM,gBAAgB,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACrD,IAAI,gBAAgB,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI;YAAE,OAAO,MAAM,CAAC;QAE3G,MAAM,gBAAgB,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,aAAa,CAC7D,IAAI,EACJ,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CACtD,CAAC;QACF,WAAW,CAAC,MAAM,EAAE,gBAAgB,EAAE,gBAAgB,CAAC,CAAC;QACxD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,EAAC,IAAI,EAAE,SAAS,EAAgB,EAAE,OAAiC;QACrF,MAAM,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;IACtE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,IAAqB;QACpC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;QACtC,MAAM,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAC,SAAS,EAAC,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;IACjF,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,IAAU;QAC1B,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAC,EAAE,EAAE,EAAC,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,EAAC,EAAC,CAAC,CAAC;QAChF,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,EAAC,MAAM,EAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACnE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,IAAI,CAAC,aAAa,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACjE,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,aAAa,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1E,CAAC;CACF;AAED,SAAS,mBAAmB,CAAC,QAAoC;IAC/D,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,QAAQ,CAAC,CAAC,CAAC,KAAK,SAAS;YAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,WAAW,CAClB,MAAkC,EAClC,gBAA0B,EAC1B,QAA2C;IAE3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC5B,IAAI,OAAO,IAAI,IAAI;YAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC;IAC7D,CAAC;AACH,CAAC"}

package/lib/db/flatFileStore/atomicWrite.d.ts

@@ -1,8 +0,0 @@
-/**
- * Atomic write: write to `.part` temp file -> datasync -> rename to target.
- * This guarantees crash-safe writes — if the process crashes mid-write,
- * only the `.part` file is corrupted, never the target.
- */
-export declare function atomicWrite(targetPath: string, data: Uint8Array): Promise<void>;
-export declare function mkdirDurable(dir: string): Promise<void>;
-//# sourceMappingURL=atomicWrite.d.ts.map

package/lib/db/flatFileStore/atomicWrite.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"atomicWrite.d.ts","sourceRoot":"","sources":["../../../src/db/flatFileStore/atomicWrite.ts"],"names":[],"mappings":"AAKA;;;;GAIG;AACH,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAsBrF;AAED,wBAAsB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAqB7D"}

package/lib/db/flatFileStore/atomicWrite.js

@@ -1,61 +0,0 @@
-import { mkdir, open, rename, rm } from "node:fs/promises";
-import path from "node:path";
-const pendingDirectorySyncs = new Map();
-/**
- * Atomic write: write to `.part` temp file -> datasync -> rename to target.
- * This guarantees crash-safe writes — if the process crashes mid-write,
- * only the `.part` file is corrupted, never the target.
- */
-export async function atomicWrite(targetPath, data) {
-    const dir = path.dirname(targetPath);
-    const partPath = `${targetPath}.part-${process.pid}-${Date.now()}-${Math.random().toString(16).slice(2)}`;
-    await mkdirDurable(dir);
-    try {
-        const fd = await open(partPath, "wx");
-        try {
-            await fd.writeFile(data);
-            await fd.datasync();
-        }
-        finally {
-            await fd.close();
-        }
-        await rename(partPath, targetPath);
-        // Ensure rename metadata is durable on crash.
-        await syncDirectory(dir);
-    }
-    catch (e) {
-        await rm(partPath, { force: true }).catch(() => { });
-        throw e;
-    }
-}
-export async function mkdirDurable(dir) {
-    const targetDir = path.resolve(dir);
-    let directoriesToSync = pendingDirectorySyncs.get(targetDir);
-    if (directoriesToSync === undefined) {
-        const firstCreatedDir = await mkdir(targetDir, { recursive: true });
-        if (firstCreatedDir === undefined)
-            return;
-        let createdDir = path.resolve(firstCreatedDir);
-        directoriesToSync = [path.dirname(createdDir)];
-        while (createdDir !== targetDir) {
-            directoriesToSync.push(createdDir);
-            const [nextSegment] = path.relative(createdDir, targetDir).split(path.sep);
-            createdDir = path.join(createdDir, nextSegment);
-        }
-        pendingDirectorySyncs.set(targetDir, directoriesToSync);
-    }
-    for (const directory of directoriesToSync) {
-        await syncDirectory(directory);
-    }
-    pendingDirectorySyncs.delete(targetDir);
-}
-async function syncDirectory(dir) {
-    const dirFd = await open(dir, "r");
-    try {
-        await dirFd.sync();
-    }
-    finally {
-        await dirFd.close();
-    }
-}
-//# sourceMappingURL=atomicWrite.js.map