@lodestar/beacon-node 1.48.0-dev.a427d3dbb5 → 1.48.0-dev.acdde443d2

package/src/db/repositories/blockArchiveIndex.ts

@@ -1,7 +1,6 @@
 import {Db, encodeKey} from "@lodestar/db";
 import {ForkAll} from "@lodestar/params";
 import {Root, SSZTypesFor, SignedBeaconBlock, Slot, ssz} from "@lodestar/types";
-import {intToBytes} from "@lodestar/utils";
 import {Bucket, getBucketNameByValue} from "../buckets.js";
 
 export const rootIndexBucketId = getBucketNameByValue(Bucket.index_blockArchiveRootIndex);
@@ -15,14 +14,6 @@ export async function getParentRootIndex(db: Db, parentRoot: Root): Promise<Uint
   return db.get(getParentRootIndexKey(parentRoot), {bucketId: parentRootIndexBucketId});
 }
 
-export async function storeRootIndex(db: Db, slot: Slot, blockRoot: Root): Promise<void> {
-  return db.put(getRootIndexKey(blockRoot), intToBytes(slot, 8, "be"), {bucketId: rootIndexBucketId});
-}
-
-export async function storeParentRootIndex(db: Db, slot: Slot, parentRoot: Root): Promise<void> {
-  return db.put(getParentRootIndexKey(parentRoot), intToBytes(slot, 8, "be"), {bucketId: parentRootIndexBucketId});
-}
-
 export async function deleteRootIndex(
   db: Db,
   signedBeaconBlockType: SSZTypesFor<ForkAll, "SignedBeaconBlock">,
@@ -43,3 +34,7 @@ export function getParentRootIndexKey(parentRoot: Root): Uint8Array {
 export function getRootIndexKey(root: Root): Uint8Array {
   return encodeKey(Bucket.index_blockArchiveRootIndex, root);
 }
+
+export function getSlotIndexKey(slot: Slot): Uint8Array {
+  return encodeKey(Bucket.index_mainChain, slot);
+}

package/src/metrics/metrics/lodestar.ts

@@ -21,6 +21,7 @@ import {ReprocessStatus} from "../../chain/reprocess.js";
 import {RejectReason} from "../../chain/seenCache/seenAttestationData.js";
 import {CacheItemType} from "../../chain/stateCache/types.js";
 import {OpSource} from "../../chain/validatorMonitor.js";
+import type {FlatFileStoreOperation} from "../../db/flatFileStore/metrics.js";
 import {ExecutionPayloadStatus} from "../../execution/index.js";
 import {GossipType} from "../../network/index.js";
 import {CannotAcceptWorkReason, ReprocessRejectReason} from "../../network/processor/index.js";
@@ -2099,7 +2100,7 @@ export function createLodestarMetrics(
       }),
       dbSizeTotal: register.gauge({
         name: "lodestar_db_size_bytes_total",
-        help: "Approximate number of bytes of file system space used by db",
+        help: "Approximate number of bytes of file system space used by LevelDB",
       }),
       dbApproximateSizeTime: register.histogram({
         name: "lodestar_db_approximate_size_time_seconds",
@@ -2108,6 +2109,61 @@ export function createLodestarMetrics(
       }),
     },
 
+    flatFileStore: {
+      operationDuration: {
+        read: register.histogram({
+          name: "lodestar_flat_file_store_read_duration_seconds",
+          help: "Duration of flat file store reads in seconds",
+          buckets: [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05],
+        }),
+        write: register.histogram({
+          name: "lodestar_flat_file_store_write_duration_seconds",
+          help: "Duration of flat file store writes in seconds",
+          buckets: [0.005, 0.01, 0.05, 0.1, 0.5, 1],
+        }),
+        delete: register.histogram({
+          name: "lodestar_flat_file_store_delete_duration_seconds",
+          help: "Duration of flat file store deletions in seconds",
+          buckets: [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05],
+        }),
+        prune: register.histogram({
+          name: "lodestar_flat_file_store_prune_duration_seconds",
+          help: "Duration of flat file store pruning in seconds",
+          buckets: [0.0005, 0.01, 0.05, 0.5, 5, 30],
+        }),
+      },
+      operationErrors: register.counter<{operation: FlatFileStoreOperation}>({
+        name: "lodestar_flat_file_store_operation_errors_total",
+        help: "Total count of failed flat file store operations",
+        labelNames: ["operation"],
+      }),
+      readBytes: register.counter({
+        name: "lodestar_flat_file_store_read_bytes_total",
+        help: "Total bytes read from flat file storage",
+      }),
+      writeBytes: register.counter({
+        name: "lodestar_flat_file_store_write_bytes_total",
+        help: "Total bytes written to flat file storage",
+      }),
+      prunedDirectories: register.counter({
+        name: "lodestar_flat_file_store_pruned_directories_total",
+        help: "Total count of slot directories pruned from flat file storage",
+      }),
+      slotIndexSize: register.gauge({
+        name: "lodestar_flat_file_store_indexed_slots",
+        help: "Count of indexed flat file slot directories, including empty directories awaiting pruning",
+      }),
+      startupDuration: register.histogram({
+        name: "lodestar_flat_file_store_startup_duration_seconds",
+        help: "Duration of flat file store slot index reconstruction in seconds",
+        buckets: [0.1, 0.5, 1, 5, 10, 30, 60, 300],
+      }),
+      startupErrors: register.counter({
+        name: "lodestar_flat_file_store_startup_errors_total",
+        help: "Total count of flat file store startup failures",
+      }),
+    },
+
     pruneHistory: {
       pruneCount: register.gauge({
         name: "lodestar_prune_history_prune_count_total",

package/src/monitoring/clientStats.ts

@@ -107,7 +107,7 @@ function createBeaconNodeStats(): ClientStats {
       metricName: "lodestar_db_size_bytes_total",
       jsonType: JsonType.Number,
       defaultValue: 0,
-      description: "Amount of bytes consumed on disk by the beacon node's database",
+      description: "Amount of bytes consumed on disk by the beacon node's LevelDB database",
     }),
     networkLibp2pBytesTotalReceive: new MetricProperty({
       jsonKey: "network_libp2p_bytes_total_receive",

package/src/network/reqresp/handlers/dataColumnSidecarsByRange.ts

@@ -1,11 +1,11 @@
 import {PeerId} from "@libp2p/interface";
 import {ChainConfig} from "@lodestar/config";
 import {PayloadStatus} from "@lodestar/fork-choice";
-import {ForkSeq, GENESIS_SLOT} from "@lodestar/params";
+import {ForkSeq, GENESIS_SLOT, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
 import {RespStatus, ResponseError, ResponseOutgoing} from "@lodestar/reqresp";
 import {computeEpochAtSlot} from "@lodestar/state-transition";
-import {ColumnIndex, Epoch, fulu} from "@lodestar/types";
-import {fromHex} from "@lodestar/utils";
+import {ColumnIndex, Epoch, RootHex, fulu} from "@lodestar/types";
+import {fromHex, toRootHex} from "@lodestar/utils";
 import {IBeaconChain} from "../../../chain/index.js";
 import {IBeaconDb} from "../../../db/index.js";
 import {prettyPrintPeerId} from "../../util.js";
@@ -50,121 +50,155 @@ export async function* onDataColumnSidecarsByRange(
     );
   }
 
-  const finalized = db.dataColumnSidecarArchive;
   const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot;
-  // Columns of the last finalized block live in different DBs depending on fork:
-  // - Pre-gloas (fulu): migrated to dataColumnSidecarArchive in the same finalization run.
-  // - Post-gloas: stay in the hot db (db.dataColumnSidecar) until the next finalization run,
-  //   because the migration filter requires payloadStatus === FULL for gloas blocks.
-  // archiveMaxSlot is the last slot whose columns are served by the archive loop below;
-  // anything above it is served by the headChain loop.
+  // At Gloas, finalizing the beacon block does not finalize its payload. Keep the boundary block in the
+  // fork-choice-backed section until the following finalization, matching ExecutionPayloadEnvelopesByRange.
   const isPostGloasFinalized = chain.config.getForkSeq(finalizedSlot) >= ForkSeq.gloas;
   const archiveMaxSlot = isPostGloasFinalized ? finalizedSlot - 1 : finalizedSlot;
+  const headBlock = chain.forkChoice.getHead();
+  const headChain = chain.forkChoice.getAllAncestorBlocks(headBlock.blockRoot, headBlock.payloadStatus);
 
-  // Finalized range of columns
-  if (startSlot <= archiveMaxSlot) {
-    const archiveEnd = Math.min(endSlot, archiveMaxSlot + 1);
-    for (let slot = startSlot; slot < archiveEnd; slot++) {
-      const dataColumnSidecars = await finalized.getManyBinary(slot, availableColumns);
-
-      const unavailableColumnIndices: ColumnIndex[] = [];
-      for (let i = 0; i < dataColumnSidecars.length; i++) {
-        const dataColumnSidecarBytes = dataColumnSidecars[i];
-        if (dataColumnSidecarBytes) {
-          yield {
-            data: dataColumnSidecarBytes,
-            boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(slot)),
-          };
-        }
-
-        // TODO: Check blobs for that block and respond resource_unavailable
-        // After we have consensus from other teams on the specs
-        else {
-          unavailableColumnIndices.push(availableColumns[i]);
-        }
+  for await (const block of resolveCanonicalDataColumnBlocks(
+    chain,
+    db,
+    headChain,
+    startSlot,
+    endSlot,
+    archiveMaxSlot,
+    finalizedSlot
+  )) {
+    const dataColumnSidecars =
+      block.blockRoot === null
+        ? await db.dataColumnSidecarArchive.getManyBinary(block.slot, availableColumns)
+        : await chain.getSerializedDataColumnSidecars(block.slot, block.blockRoot, availableColumns);
+    if (block.blockRoot === null && dataColumnSidecars.every((sidecar) => sidecar === undefined)) continue;
+    const unavailableColumnIndices: ColumnIndex[] = [];
+    for (let i = 0; i < dataColumnSidecars.length; i++) {
+      const dataColumnSidecarBytes = dataColumnSidecars[i];
+      if (dataColumnSidecarBytes) {
+        yield {
+          data: dataColumnSidecarBytes,
+          boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(block.slot)),
+        };
+      } else {
+        unavailableColumnIndices.push(availableColumns[i]);
       }
+    }
 
-      if (unavailableColumnIndices.length) {
-        await handleColumnSidecarUnavailability({
-          chain,
-          db,
-          metrics: chain.metrics,
-          unavailableColumnIndices,
-          slot,
-          requestedColumns,
-          availableColumns,
-        });
-      }
+    if (unavailableColumnIndices.length > 0) {
+      await handleColumnSidecarUnavailability({
+        chain,
+        db,
+        metrics: chain.metrics,
+        unavailableColumnIndices,
+        blockRoot: block.unavailabilityBlockRoot ? fromHex(block.unavailabilityBlockRoot) : undefined,
+        finalized: block.finalized,
+        slot: block.slot,
+        requestedColumns,
+        availableColumns,
+      });
     }
   }
+}
 
-  // Non-finalized range of columns
-  if (endSlot > archiveMaxSlot) {
-    const headBlock = chain.forkChoice.getHead();
-    const headRoot = headBlock.blockRoot;
-    // getAllAncestorBlocks includes the last finalized block as its final element.
-    // Skip anything the archive loop above already served via the block.slot > archiveMaxSlot
-    // filter below (pre-gloas this skips finalizedSlot, post-gloas it keeps it).
-    const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot, headBlock.payloadStatus);
-
-    // Iterate head chain with ascending block numbers
-    for (let i = headChain.length - 1; i >= 0; i--) {
-      const block = headChain[i];
-
-      // Must include only columns in the range requested
-      if (block.slot > archiveMaxSlot && block.slot >= startSlot && block.slot < endSlot) {
-        // Post-gloas, columns exist only for FULL blocks (pre-gloas blocks are always FULL)
-        if (block.payloadStatus !== PayloadStatus.FULL) {
-          continue;
-        }
-
-        // Note: Here the forkChoice head may change due to a re-org, so the headChain reflects the canonical chain
-        // at the time of the start of the request. Spec is clear the chain of columns must be consistent, but on
-        // re-org there's no need to abort the request
-        // Spec: https://github.com/ethereum/consensus-specs/blob/ad36024441cf910d428d03f87f331fbbd2b3e5f1/specs/fulu/p2p-interface.md#L425-L429
-        const dataColumnSidecars = await chain.getSerializedDataColumnSidecars(
-          block.slot,
-          block.blockRoot,
-          availableColumns
-        );
-
-        const unavailableColumnIndices: ColumnIndex[] = [];
-        for (let i = 0; i < dataColumnSidecars.length; i++) {
-          const dataColumnSidecarBytes = dataColumnSidecars[i];
-          if (dataColumnSidecarBytes) {
-            yield {
-              data: dataColumnSidecarBytes,
-              boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(block.slot)),
-            };
-          }
-
-          // TODO: Check blobs for that block and respond resource_unavailable
-          // After we have consensus from other teams on the specs
-          else {
-            unavailableColumnIndices.push(availableColumns[i]);
-          }
-        }
-
-        if (unavailableColumnIndices.length) {
-          await handleColumnSidecarUnavailability({
-            chain,
-            db,
-            metrics: chain.metrics,
-            unavailableColumnIndices,
-            blockRoot: fromHex(block.blockRoot),
-            slot: block.slot,
-            requestedColumns,
-            availableColumns,
-          });
-        }
-      }
+type CanonicalDataColumnBlock = {
+  slot: number;
+  blockRoot: RootHex | null;
+  unavailabilityBlockRoot?: RootHex;
+  finalized: boolean;
+};
 
-      // If block is after endSlot, stop iterating
-      else if (block.slot >= endSlot) {
-        break;
+async function* resolveCanonicalDataColumnBlocks(
+  chain: IBeaconChain,
+  db: IBeaconDb,
+  headChain: ReturnType<IBeaconChain["forkChoice"]["getAllAncestorBlocks"]>,
+  startSlot: number,
+  endSlot: number,
+  archiveMaxSlot: number,
+  finalizedSlot: number
+): AsyncIterable<CanonicalDataColumnBlock> {
+  const canonicalBlocksBySlot = new Map(headChain.map((block) => [block.slot, block]));
+  const oldestForkChoiceSlot = headChain.at(-1)?.slot ?? Number.POSITIVE_INFINITY;
+  const archiveEnd = Math.min(endSlot, archiveMaxSlot + 1);
+  const recentRoots = getRecentCanonicalRoots(chain, startSlot, Math.min(archiveEnd, oldestForkChoiceSlot));
+
+  for (let slot = startSlot; slot < archiveEnd; slot++) {
+    if (slot >= oldestForkChoiceSlot) {
+      const block = canonicalBlocksBySlot.get(slot);
+      if (block?.payloadStatus === PayloadStatus.FULL) {
+        yield {
+          slot,
+          blockRoot: block.blockRoot,
+          unavailabilityBlockRoot: block.blockRoot,
+          finalized: true,
+        };
       }
+      continue;
+    }
+
+    let blockRoot = recentRoots.get(slot);
+    if (blockRoot === null) continue;
+    if (blockRoot === undefined) {
+      const root = await db.blockArchive.getRootBySlot(slot);
+      blockRoot = root === null ? null : toRootHex(root);
     }
+    if (blockRoot === null && (db.lastLegacyArchiveSlot === null || slot > db.lastLegacyArchiveSlot)) continue;
+    if (
+      chain.config.getForkSeq(slot) >= ForkSeq.gloas &&
+      !(await hasExecutionPayloadEnvelope(chain, db, slot, blockRoot))
+    ) {
+      continue;
+    }
+    yield {
+      slot,
+      blockRoot,
+      unavailabilityBlockRoot: chain.config.getForkSeq(slot) >= ForkSeq.gloas ? (blockRoot ?? undefined) : undefined,
+      finalized: true,
+    };
+  }
+
+  for (let i = headChain.length - 1; i >= 0; i--) {
+    const block = headChain[i];
+    if (block.slot >= endSlot) break;
+    if (block.slot <= archiveMaxSlot || block.slot < startSlot || block.payloadStatus !== PayloadStatus.FULL) continue;
+    yield {
+      slot: block.slot,
+      blockRoot: block.blockRoot,
+      unavailabilityBlockRoot: block.blockRoot,
+      finalized: block.slot <= finalizedSlot,
+    };
+  }
+}
+
+/** Copy roots synchronously so serving a request never retains a head-state reference across I/O. */
+function getRecentCanonicalRoots(chain: IBeaconChain, startSlot: number, endSlot: number): Map<number, RootHex | null> {
+  const roots = new Map<number, RootHex | null>();
+  if (startSlot >= endSlot) return roots;
+  const state = chain.getHeadState();
+  // Skipped slots repeat the prior root. At the oldest ring entry, that prior root is unavailable.
+  const start = Math.max(startSlot, state.slot - SLOTS_PER_HISTORICAL_ROOT + 1, 1);
+  const end = Math.min(endSlot, state.slot);
+  if (start >= end) return roots;
+  let previousRoot = toRootHex(state.getBlockRootAtSlot(start - 1));
+  for (let slot = start; slot < end; slot++) {
+    const root = toRootHex(state.getBlockRootAtSlot(slot));
+    roots.set(slot, root === previousRoot ? null : root);
+    previousRoot = root;
+  }
+  return roots;
+}
+
+async function hasExecutionPayloadEnvelope(
+  chain: IBeaconChain,
+  db: IBeaconDb,
+  slot: number,
+  blockRoot: RootHex | null
+): Promise<boolean> {
+  if (blockRoot !== null) {
+    if (chain.seenPayloadEnvelopeInputCache.hasPayload(blockRoot)) return true;
+    if ((await db.executionPayloadEnvelope.getBinary(fromHex(blockRoot))) !== null) return true;
   }
+  return (await db.executionPayloadEnvelopeArchive.getBinary(slot)) !== null;
 }
 
 export function validateDataColumnSidecarsByRangeRequest(

package/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts

@@ -81,12 +81,14 @@ export async function* onDataColumnSidecarsByRoot(
     }
 
     if (unavailableColumnIndices.length) {
+      const finalized = !block || (await db.blockArchive.getSlotByRoot(blockRoot)) === slot;
       await handleColumnSidecarUnavailability({
         chain,
         db,
         metrics: chain.metrics,
         slot,
         blockRoot,
+        finalized,
         unavailableColumnIndices,
         requestedColumns,
         availableColumns,

package/src/network/reqresp/utils/dataColumnResponseValidation.ts

@@ -16,12 +16,14 @@ export async function handleColumnSidecarUnavailability({
   availableColumns,
   slot,
   blockRoot,
+  finalized,
 }: {
   chain: IBeaconChain;
   db: IBeaconDb;
   metrics: Metrics | null;
   slot: Slot;
   blockRoot?: Uint8Array;
+  finalized: boolean;
   unavailableColumnIndices: ColumnIndex[];
   requestedColumns: ColumnIndex[];
   availableColumns: ColumnIndex[];
@@ -38,17 +40,23 @@ export async function handleColumnSidecarUnavailability({
 
   chain.logger.debug("dataColumnSidecar requested unavailable", logData);
 
-  // Post-gloas, columns exist only for FULL blocks; a finalized block is FULL if its envelope was
-  // archived. Bid blobsCount is unreliable here since an EMPTY block's bid may still commit to blobs
-  if (blockRoot === undefined && chain.config.getForkSeq(slot) >= ForkSeq.gloas) {
-    const envelopeBytes = await db.executionPayloadEnvelopeArchive.getBinary(slot);
-    if (!envelopeBytes) return;
+  // Post-gloas, columns exist only for FULL blocks. The envelope may still be hot while finalization
+  // archiving is in progress. Bid blobsCount is unreliable since an EMPTY block's bid may still commit to blobs.
+  if (chain.config.getForkSeq(slot) >= ForkSeq.gloas) {
+    const hasCachedEnvelope = blockRoot ? chain.seenPayloadEnvelopeInputCache.hasPayload(toRootHex(blockRoot)) : false;
+    if (!hasCachedEnvelope) {
+      const hotEnvelopeBytes = blockRoot ? await db.executionPayloadEnvelope.getBinary(blockRoot) : null;
+      const envelopeBytes =
+        hotEnvelopeBytes ?? (finalized ? await db.executionPayloadEnvelopeArchive.getBinary(slot) : null);
+      if (!envelopeBytes) return;
+    }
   }
 
-  const blockBytes = blockRoot ? await db.block.getBinary(blockRoot) : await db.blockArchive.getBinary(slot);
+  const hotBlockBytes = blockRoot ? await db.block.getBinary(blockRoot) : null;
+  const blockBytes = hotBlockBytes ?? (finalized ? await db.blockArchive.getBinary(slot) : null);
   if (!blockBytes) {
     chain.logger.verbose(
-      `Expected ${blockRoot ? "unfinalized" : "finalized"} block not found while handling unavailable dataColumnSidecar`,
+      `Expected ${finalized ? "finalized" : "unfinalized"} block not found while handling unavailable dataColumnSidecar`,
       {
         slot,
         blockRoot: blockRoot ? toRootHex(blockRoot) : "unknown",

package/src/node/nodejs.ts

@@ -54,6 +54,7 @@ export type BeaconNodeInitModules = {
   processShutdownCallback: ProcessShutdownCallback;
   privateKey: PrivateKey;
   dataDir: string;
+  dataColumnDir: string;
   peerStoreDir?: string;
   anchorState: IBeaconStateView;
   isAnchorStateFinalized: boolean;
@@ -156,6 +157,7 @@ export class BeaconNode {
     processShutdownCallback,
     privateKey,
     dataDir,
+    dataColumnDir,
     peerStoreDir,
     anchorState,
     isAnchorStateFinalized,
@@ -181,7 +183,7 @@ export class BeaconNode {
       metrics = createMetrics(opts.metrics, anchorState.genesisTime, metricsRegistries);
       initBeaconMetrics(metrics, anchorState);
       // Since the db is instantiated before this, metrics must be injected manually afterwards
-      db.setMetrics(metrics.db);
+      db.setMetrics(metrics.db, metrics.flatFileStore);
       signal.addEventListener("abort", metrics.close, {once: true});
     }
 
@@ -198,6 +200,8 @@ export class BeaconNode {
 
     const clock = new Clock({config, genesisTime: anchorState.genesisTime, signal});
 
+    await db.init();
+
     // Prune hot db repos
     // TODO: Should this call be awaited?
     await db.pruneHotDb();
@@ -244,6 +248,7 @@ export class BeaconNode {
       clock,
       pubkeyCache,
       dataDir,
+      dataColumnDir,
       db,
       dbName: opts.db.name,
       logger: logger.child({module: LoggerModule.chain}),

package/src/util/sszBytes.ts

@@ -754,7 +754,7 @@ export function getParentBlockRootFromSignedExecutionPayloadBidSerialized(data:
  *
  * If the high bytes are not zero, return null
  */
-function getSlotFromOffset(data: Uint8Array, offset: number): Slot | null {
+export function getSlotFromOffset(data: Uint8Array, offset: number): Slot | null {
   return checkSlotHighBytes(data, offset) ? getSlotFromOffsetTrusted(data, offset) : null;
 }