@lodestar/beacon-node 1.51.0-dev.3da261f88e → 1.51.0-dev.44a2c43710

package/src/chain/blocks/importExecutionPayload.ts

@@ -37,6 +37,7 @@ export enum PayloadErrorCode {
   MISS_BLOCK_STATE = "PAYLOAD_ERROR_MISS_BLOCK_STATE",
   ENVELOPE_VERIFICATION_ERROR = "PAYLOAD_ERROR_ENVELOPE_VERIFICATION_ERROR",
   INVALID_SIGNATURE = "PAYLOAD_ERROR_INVALID_SIGNATURE",
+  DATA_UNAVAILABLE = "PAYLOAD_ERROR_DATA_UNAVAILABLE",
 }
 
 export type PayloadErrorType =
@@ -64,6 +65,9 @@ export type PayloadErrorType =
     }
   | {
       code: PayloadErrorCode.INVALID_SIGNATURE;
+    }
+  | {
+      code: PayloadErrorCode.DATA_UNAVAILABLE;
     };
 
 export class PayloadError extends Error {
@@ -383,6 +387,6 @@ export async function processExecutionPayload(
   signal: AbortSignal,
   opts: ImportPayloadOpts = {}
 ): Promise<void> {
-  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability([payloadInput], signal);
+  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(this.config, [payloadInput], signal);
   await importExecutionPayload.call(this, payloadInput, dataAvailabilityStatuses[0], opts);
 }

package/src/chain/blocks/index.ts

@@ -147,7 +147,7 @@ export async function processBlocks(
         proposerBalanceDelta: proposerBalanceDeltas[i],
         indexedAttestations: indexedAttestationsByBlock[i],
         // TODO: Make this param mandatory and capture in gossip
-        seenTimestampSec: opts.seenTimestampSec ?? Math.floor(Date.now() / 1000),
+        seenTimestampSec: opts.seenTimestampSec ?? Date.now() / 1000,
       });
     }
 
@@ -292,7 +292,11 @@ async function importPayloadEnvelopesOfKnownBlocks(
     return result;
   }
 
-  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(payloadInputs, new AbortController().signal);
+  const {dataAvailabilityStatuses} = await verifyPayloadsDataAvailability(
+    this.config,
+    payloadInputs,
+    new AbortController().signal
+  );
   for (let i = 0; i < payloadInputs.length; i++) {
     this.logger.debug("Importing payload envelope of known block", {
       slot: payloadInputs[i].slot,

package/src/chain/blocks/payloadEnvelopeProcessor.ts

@@ -13,11 +13,11 @@ const QUEUE_MAX_LENGTH = 16;
  *
  * Jobs are enqueued only on envelope arrival (gossip or API). The envelope may reach us before
  * the sampled data columns; importExecutionPayload awaits `verifyPayloadsDataAvailability`
- * internally, so a queued job can pend for up to `PAYLOAD_DATA_AVAILABILITY_TIMEOUT` while
- * waiting for columns. Duplicate triggers for the same payloadInput are deduped by sharing the
- * in-flight import promise: every caller observes the real import outcome. Resolving duplicates
- * early instead would let BlockInputSync mark a still-pending payload as processed, re-queue it,
- * and spin re-processing the same root until OOM while the import is blocked (e.g. EL syncing).
+ * internally, so a queued job can pend for up to one slot while waiting for columns. Duplicate
+ * triggers for the same payloadInput are deduped by sharing the in-flight import promise: every
+ * caller observes the real import outcome. Resolving duplicates early instead would let
+ * BlockInputSync mark a still-pending payload as processed, re-queue it, and spin re-processing
+ * the same root until OOM while the import is blocked (e.g. EL syncing).
  */
 export class PayloadEnvelopeProcessor {
   readonly jobQueue: JobItemQueue<[PayloadEnvelopeInput, ImportPayloadOpts], void>;

package/src/chain/blocks/verifyBlock.ts

@@ -143,6 +143,7 @@ export async function verifyBlocksInEpoch(
                 ? Array.from(payloadEnvelopes.values()).filter((payloadInput) => payloadInput.hasPayloadEnvelope())
                 : [];
             const {dataAvailabilityStatuses, availableTime} = await verifyPayloadsDataAvailability(
+              this.config,
               payloadInputsForDa,
               abortController.signal
             );

package/src/chain/blocks/verifyBlocksDataAvailability.ts

@@ -1,4 +1,6 @@
 import {DataAvailabilityStatus} from "@lodestar/state-transition";
+import {TimeoutError} from "@lodestar/utils";
+import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {DAData, DAType, IBlockInput} from "./blockInput/index.js";
 
 // we can now wait for full 12 seconds because unavailable block sync will try pulling
@@ -22,7 +24,14 @@ export async function verifyBlocksDataAvailability(
   for (const blockInput of blocks) {
     // block verification is triggered on a verified gossip block so we only need to wait for all data
     if (!blockInput.hasAllData()) {
-      promises.push(blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT, signal));
+      promises.push(
+        blockInput.waitForAllData(BLOB_AVAILABILITY_TIMEOUT, signal).catch((e) => {
+          if (e instanceof TimeoutError) {
+            throw new BlockError(blockInput.getBlock(), {code: BlockErrorCode.DATA_UNAVAILABLE});
+          }
+          throw e;
+        })
+      );
     }
   }
   await Promise.all(promises);

package/src/chain/blocks/verifyPayloadsDataAvailability.ts

@@ -1,18 +1,18 @@
+import {ChainForkConfig} from "@lodestar/config";
 import {DataAvailabilityStatus} from "@lodestar/state-transition";
 import {gloas} from "@lodestar/types";
+import {TimeoutError} from "@lodestar/utils";
 import {PayloadEnvelopeInput} from "../seenCache/seenPayloadEnvelopeInput.js";
-
-// we can now wait for full 12 seconds because sync and reconstruction will try pulling
-// the data columns from the network anyway while the envelope is being processed
-export const PAYLOAD_DATA_AVAILABILITY_TIMEOUT = 12_000;
+import {PayloadError, PayloadErrorCode} from "./importExecutionPayload.js";
 
 /**
  * Verifies that all payload envelope inputs have their data columns available.
- * - Waits a max of PAYLOAD_DATA_AVAILABILITY_TIMEOUT for all data to be available
+ * - Waits a max of one slot duration of the payload's fork for all data to be available
  * - Returns the time at which all data was available
  * - Returns the data availability status for each payload input
  */
 export async function verifyPayloadsDataAvailability(
+  config: ChainForkConfig,
   payloadInputs: PayloadEnvelopeInput[],
   signal: AbortSignal
 ): Promise<{
@@ -22,7 +22,17 @@ export async function verifyPayloadsDataAvailability(
   const promises: Promise<gloas.DataColumnSidecar[]>[] = [];
   for (const payloadInput of payloadInputs) {
     if (!payloadInput.hasAllData()) {
-      promises.push(payloadInput.waitForAllData(PAYLOAD_DATA_AVAILABILITY_TIMEOUT, signal));
+      // Sync and reconstruction keep pulling the data columns from the network while we wait. The wait is
+      // capped at one slot because the payload queue is serial: a longer wait lets consecutive unavailable
+      // payloads push every later payload import further behind with each slot.
+      promises.push(
+        payloadInput.waitForAllData(config.getSlotDurationMs(payloadInput.forkName), signal).catch((e) => {
+          if (e instanceof TimeoutError) {
+            throw new PayloadError(payloadInput, {code: PayloadErrorCode.DATA_UNAVAILABLE});
+          }
+          throw e;
+        })
+      );
     }
   }
   await Promise.all(promises);

package/src/chain/chain.ts

@@ -1486,10 +1486,10 @@ export class BeaconChain implements IBeaconChain {
     this.metrics?.forkChoice.requests.inc();
     const timer = this.metrics?.forkChoice.findHead.startTimer({caller: FindHeadFnName.predictProposerHead});
     const slot = this.clock.currentSlot;
-    const secFromSlot = this.clock.secFromSlot(slot);
+    const msFromSlot = this.clock.msFromSlot(slot);
 
     try {
-      return this.forkChoice.updateAndGetHead({mode: UpdateHeadOpt.GetPredictedProposerHead, secFromSlot, slot}).head;
+      return this.forkChoice.updateAndGetHead({mode: UpdateHeadOpt.GetPredictedProposerHead, msFromSlot, slot}).head;
     } catch (e) {
       this.metrics?.forkChoice.errors.inc({entrypoint: UpdateHeadOpt.GetPredictedProposerHead});
       throw e;
@@ -1501,13 +1501,13 @@ export class BeaconChain implements IBeaconChain {
   getProposerHead(slot: Slot): ProtoBlock {
     this.metrics?.forkChoice.requests.inc();
     const timer = this.metrics?.forkChoice.findHead.startTimer({caller: FindHeadFnName.getProposerHead});
-    const secFromSlot = this.clock.secFromSlot(slot);
+    const msFromSlot = this.clock.msFromSlot(slot);
 
     try {
       // Do not emit head event here, when proposing we rely on the one emitted when importing our own block
       const {head, isHeadTimely, notReorgedReason} = this.forkChoice.updateAndGetHead({
         mode: UpdateHeadOpt.GetProposerHead,
-        secFromSlot,
+        msFromSlot,
         slot,
       });
 

package/src/chain/errors/blockError.ts

@@ -72,7 +72,7 @@ export enum BlockErrorCode {
    * whoever served the block MAY be downscored. Mirrors PayloadErrorCode.EXECUTION_ENGINE_INVALID.
    */
   EXECUTION_ENGINE_INVALID = "BLOCK_ERROR_EXECUTION_INVALID",
-  /** The blobs are unavailable */
+  /** The blob or column data did not arrive within the availability wait */
   DATA_UNAVAILABLE = "BLOCK_ERROR_DATA_UNAVAILABLE",
   /** Block contains too many kzg commitments */
   TOO_MANY_KZG_COMMITMENTS = "BLOCK_ERROR_TOO_MANY_KZG_COMMITMENTS",

package/src/db/beacon.ts

@@ -30,6 +30,9 @@ import {
   VoluntaryExitRepository,
 } from "./repositories/index.js";
 
+/** Below this size, seeks over leftover legacy column data stay cheap */
+const LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES = 1024 ** 3;
+
 export type BeaconDbOpts = {
   dataColumnDir: string;
   logger: Logger;
@@ -67,6 +70,7 @@ export class BeaconDb implements IBeaconDb {
   readonly dataColumns: IDataColumnStore;
   lastLegacyArchiveSlot: Slot | null = null;
   private readonly flatFileStore: FlatFileStore;
+  private readonly logger: Logger;
 
   constructor(
     config: ChainForkConfig,
@@ -100,6 +104,7 @@ export class BeaconDb implements IBeaconDb {
     this.syncCommittee = new SyncCommitteeRepository(config, db);
     this.syncCommitteeWitness = new SyncCommitteeWitnessRepository(config, db);
 
+    this.logger = opts.logger;
     this.flatFileStore = new FlatFileStore(opts.dataColumnDir, config, opts.logger, opts.metrics);
     this.dataColumns = new LegacyDataColumnStore(
       this.flatFileStore,
@@ -125,11 +130,50 @@ export class BeaconDb implements IBeaconDb {
     this.flatFileStore.setMetrics(flatFileStoreMetrics);
   }
 
+  /**
+   * Deletes made while an iterator held a snapshot can stay in the bottom LevelDB level, which `compactRange` never
+   * rewrites, so every seek into the legacy column buckets walks them. Writing a key at each end of the pruned range
+   * makes the compaction overlap those files and rewrite them.
+   */
+  async compactLegacyDataColumns(
+    prunedBeforeSlot: Slot,
+    minSizeBytes = LEGACY_DATA_COLUMNS_COMPACTION_MIN_BYTES
+  ): Promise<void> {
+    const archiveBucket = Bucket.allForks_dataColumnSidecarsArchive;
+    const start = encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(0));
+    const end = encodeKey(archiveBucket, this.dataColumnSidecarArchive.getMinKeyRaw(prunedBeforeSlot));
+    const sizeBytes = await this.db.approximateSize(start, end);
+    if (sizeBytes < minSizeBytes) return;
+
+    // Column indices stay below 0xffff and hot column keys are shorter, so no sentinel matches a real key
+    const upperSentinel =
+      prunedBeforeSlot > 0
+        ? encodeKey(archiveBucket, this.dataColumnSidecarArchive.encodeKeyRaw(prunedBeforeSlot - 1, 0xffff))
+        : encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(35, 0xff));
+    const sentinels = [start, upperSentinel];
+
+    this.logger.info("Compacting legacy data columns, this may take a while", {sizeBytes});
+    const startTime = Date.now();
+    await this.db.batchPut(sentinels.map((key) => ({key, value: new Uint8Array()})));
+    await this.db.compactRange(start, end);
+    await this.db.batchDelete(sentinels);
+    await this.db.compactRange(start, end);
+    this.logger.verbose("Compacted legacy data columns", {
+      durationMs: Date.now() - startTime,
+      sizeBytes: await this.db.approximateSize(start, end),
+    });
+  }
+
   async pruneHotDb(): Promise<void> {
     // Prune all hot blobs
     await this.blobSidecars.batchDelete(await this.blobSidecars.keys());
     // New columns are persisted in flat files, so this bucket only contains legacy hot data.
     await this.deleteBucketData(Bucket.allForks_dataColumnSidecars);
+    // Compact even if empty to reclaim tombstones left by previous startups.
+    await this.db.compactRange(
+      encodeKey(Bucket.allForks_dataColumnSidecars, Buffer.alloc(0)),
+      encodeKey(Bucket.allForks_dataColumnSidecars + 1, Buffer.alloc(0))
+    );
     // Prune all hot blocks
     // TODO: Enable once it's deemed safe
     // await this.block.batchDelete(await this.block.keys());

package/src/db/dataColumnStore.ts

@@ -13,7 +13,7 @@ export type IndexedDataColumnBytes = {index: ColumnIndex; data: Uint8Array};
 type LegacyHotDataColumnStore = Pick<DataColumnSidecarRepository, "values" | "getManyBinary" | "deleteMany">;
 type LegacyArchiveDataColumnStore = Pick<
   DataColumnSidecarArchiveRepository,
-  "values" | "getManyBinary" | "keys" | "deleteMany"
+  "values" | "getManyBinary" | "pruneBefore"
 >;
 type BlockRootIndex = Pick<BlockArchiveRepository, "getSlotByRoot">;
 
@@ -86,9 +86,7 @@ export class LegacyDataColumnStore implements IDataColumnStore {
 
   async pruneBefore(slot: Slot): Promise<Slot[]> {
     const flatFileSlots = await this.flatFiles.pruneBefore(slot);
-    const prefixedKeys = await this.legacyArchive.keys({lt: {prefix: slot, id: 0}});
-    const slots = [...new Set(prefixedKeys.map(({prefix}) => prefix))];
-    if (slots.length > 0) await this.legacyArchive.deleteMany(slots);
+    const slots = await this.legacyArchive.pruneBefore(slot);
     return [...new Set([...flatFileSlots, ...slots])].sort((a, b) => a - b);
   }
 }

package/src/db/interface.ts

@@ -68,6 +68,8 @@ export interface IBeaconDb {
   readonly lastLegacyArchiveSlot: Slot | null;
   init(): Promise<void>;
 
+  compactLegacyDataColumns(prunedBeforeSlot: Slot, minSizeBytes?: number): Promise<void>;
+
   pruneHotDb(): Promise<void>;
 
   deleteDeprecatedEth1Data(): Promise<void>;

package/src/db/repositories/dataColumnSidecarArchive.ts

@@ -15,6 +15,8 @@ const SLOT_BYTE_SIZE = 8;
  * Indexed data by `slot` + `columnIndex`
  */
 export class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot, ColumnIndex, DataColumnSidecar> {
+  private prunedBeforeSlot: Slot | null = null;
+
   constructor(config: ChainForkConfig, db: Db) {
     const bucket = Bucket.allForks_dataColumnSidecarsArchive;
     // Type won't be used since we select it dynamically based on fork
@@ -70,4 +72,22 @@ export class DataColumnSidecarArchiveRepository extends PrefixedRepository<Slot,
       encodeNumberForDbKey(0, COLUMN_INDEX_BYTE_SIZE),
     ]);
   }
+
+  /**
+   * Deletes all entries before `slot`. Prunes only delete from the start of the bucket, so the first prune
+   * compacts the range to drop tombstones of earlier runs and later prunes resume where the previous one ended.
+   */
+  async pruneBefore(slot: Slot): Promise<Slot[]> {
+    if (this.prunedBeforeSlot === null) {
+      await this.db.compactRange(this.wrapKey(this.getMinKeyRaw(0)), this.wrapKey(this.getMinKeyRaw(slot)));
+    } else if (slot <= this.prunedBeforeSlot) {
+      return [];
+    }
+
+    const keys = await this.keys({gte: {prefix: this.prunedBeforeSlot ?? 0, id: 0}, lt: {prefix: slot, id: 0}});
+    const slots = [...new Set(keys.map(({prefix}) => prefix))];
+    if (slots.length > 0) await this.deleteMany(slots);
+    this.prunedBeforeSlot = slot;
+    return slots;
+  }
 }

package/src/index.ts

@@ -6,6 +6,8 @@ export {persistAnchorState} from "./chain/index.js";
 export {DbCPStateDatastore} from "./chain/stateCache/datastore/db.js";
 export {FileCPStateDatastore} from "./chain/stateCache/datastore/file.js";
 export {BeaconDb, type IBeaconDb} from "./db/index.js";
+// Export execution engine to make it usable by builder
+export {initializeExecutionEngine} from "./execution/index.js";
 // Export metrics utilities to de-duplicate validator metrics
 export {
   type HttpMetricsServer,

package/src/network/processor/gossipHandlers.ts

@@ -718,11 +718,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
         if (e instanceof BlockError) {
           switch (e.type.code) {
-            case BlockErrorCode.DATA_UNAVAILABLE: {
-              // Error is quite frequent and not critical
+            case BlockErrorCode.DATA_UNAVAILABLE:
+              // Data did not arrive in time, BlockInputSync keeps pulling it and retries the import
               logLevel = LogLevel.debug;
               break;
-            }
             // ALREADY_KNOWN should not happen with ignoreIfKnown=true above
             // PARENT_BLOCK_UNKNOWN should not happen, we handled this in validateBeaconBlock() function above
             case BlockErrorCode.ALREADY_KNOWN:
@@ -1358,6 +1357,10 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
             if (e instanceof PayloadError) {
               switch (e.type.code) {
+                case PayloadErrorCode.DATA_UNAVAILABLE:
+                  // Columns did not arrive in time, BlockInputSync keeps pulling them and retries the import
+                  logLevel = LogLevel.debug;
+                  break;
                 // BLOCK_NOT_IN_FORK_CHOICE should not happen, validateGossipExecutionPayloadEnvelope above
                 // already verified the block is in fork choice
                 case PayloadErrorCode.BLOCK_NOT_IN_FORK_CHOICE:

package/src/node/nodejs.ts

@@ -13,11 +13,17 @@ import {BeaconApiMethods} from "@lodestar/api/beacon/server";
 import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ZERO_HASH_HEX} from "@lodestar/params";
-import {IBeaconStateView, isStatePostBellatrix, isStatePostGloas} from "@lodestar/state-transition";
+import {
+  IBeaconStateView,
+  computeStartSlotAtEpoch,
+  isStatePostBellatrix,
+  isStatePostGloas,
+} from "@lodestar/state-transition";
 import {Slot} from "@lodestar/types";
 import {sleep, toRootHex} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
 import {BeaconRestApiServer, getApi} from "../api/index.js";
+import {getDataColumnSidecarsMinEpoch} from "../chain/archiveStore/utils/archiveBlocks.js";
 import {BeaconChain, IBeaconChain, initBeaconMetrics} from "../chain/index.js";
 import {ValidatorMonitor, createValidatorMonitor} from "../chain/validatorMonitor.js";
 import {IBeaconDb} from "../db/index.js";
@@ -212,6 +218,17 @@ export class BeaconNode {
 
     const clock = new Clock({config, genesisTime: anchorState.genesisTime, signal});
 
+    // Before any lookup into the legacy column buckets, which can be slow until compacted
+    const columnsMinEpoch =
+      opts.chain.archiveDataEpochs === Infinity
+        ? null
+        : getDataColumnSidecarsMinEpoch(config, clock.currentEpoch, opts.chain.archiveDataEpochs);
+    await db.compactLegacyDataColumns(
+      columnsMinEpoch !== null && columnsMinEpoch >= config.FULU_FORK_EPOCH
+        ? computeStartSlotAtEpoch(columnsMinEpoch)
+        : 0
+    );
+
     await db.init();
 
     // Prune hot db repos