@lodestar/beacon-node 1.50.0-dev.af3e3a6cff → 1.50.0-dev.b37917f96a

package/src/chain/opPools/aggregatedAttestationPool.ts

@@ -30,7 +30,7 @@ import {
   isStatePostGloas,
 } from "@lodestar/state-transition";
 import {Attestation, Epoch, RootHex, Slot, electra, isElectraAttestation, phase0, ssz} from "@lodestar/types";
-import {MapDef, assert, toRootHex} from "@lodestar/utils";
+import {Logger, MapDef, assert, toRootHex} from "@lodestar/utils";
 import {Metrics} from "../../metrics/metrics.js";
 import {IntersectResult, intersectUint8Arrays} from "../../util/bitArray.js";
 import {getShufflingDependentRoot} from "../../util/dependentRoot.js";
@@ -58,7 +58,7 @@ export type AttestationsConsolidation = {
 
 /**
  * This function returns not seen participation for a given epoch and slot and committee index.
- * Return null if all validators are seen or no info to check.
+ * Return an empty set if all validators are seen, or null if the committee cannot be computed (e.g. missing shuffling).
  */
 type GetNotSeenValidatorsFn = (epoch: Epoch, slot: Slot, committeeIndex: number) => Set<number> | null;
 
@@ -140,7 +140,8 @@ export class AggregatedAttestationPool {
 
   constructor(
     private readonly config: BeaconConfig,
-    private readonly metrics: Metrics | null = null
+    private readonly metrics: Metrics | null = null,
+    private readonly logger: Logger | null = null
   ) {
     metrics?.opPool.aggregatedAttestationPool.attDataPerSlot.addCollect(() => this.onScrapeMetrics(metrics));
   }
@@ -244,6 +245,7 @@ export class AggregatedAttestationPool {
     const consolidations = new Map<AttestationsConsolidation, number>();
     let scannedSlots = 0;
     let stopReason: ScannedSlotsTerminationReason | null = null;
+    const missingShufflingCommitteesByEpoch = new Map<Epoch, number>();
     slot: for (const slot of slots) {
       const attestationGroupByIndexByDataHash = this.attestationGroupByIndexByDataHexBySlot.get(slot);
       // should not happen
@@ -295,7 +297,12 @@ export class AggregatedAttestationPool {
 
         for (const [committeeIndex, attestationGroup] of attestationGroupByIndex.entries()) {
           const notSeenCommitteeMembers = notSeenValidatorsFn(epoch, slot, committeeIndex);
-          if (notSeenCommitteeMembers === null || notSeenCommitteeMembers.size === 0) {
+          if (notSeenCommitteeMembers === null) {
+            // skip this committee instead of failing the whole block production, other epochs may still have shuffling
+            missingShufflingCommitteesByEpoch.set(epoch, (missingShufflingCommitteesByEpoch.get(epoch) ?? 0) + 1);
+            continue;
+          }
+          if (notSeenCommitteeMembers.size === 0) {
             this.metrics?.opPool.aggregatedAttestationPool.packedAttestations.seenCommittees.inc();
             continue;
           }
@@ -386,6 +393,18 @@ export class AggregatedAttestationPool {
       scannedSlots++;
     }
 
+    for (const [epoch, missingCommittees] of missingShufflingCommitteesByEpoch) {
+      this.metrics?.opPool.aggregatedAttestationPool.packedAttestations.missingShufflingCommittees.inc(
+        missingCommittees
+      );
+      this.logger?.debug("Missing shuffling when packing attestations, skipped committees", {
+        slot: stateSlot,
+        epoch,
+        decisionRoot: state.getShufflingDecisionRoot(epoch),
+        missingCommittees,
+      });
+    }
+
     this.metrics?.opPool.aggregatedAttestationPool.packedAttestations.totalConsolidations.set(consolidations.size);
 
     const sortedConsolidationsByScore = Array.from(consolidations.entries())
@@ -770,13 +789,18 @@ export function getNotSeenValidatorsFn(
     }
     const cacheKey = slot + "_" + committeeIndex;
     let notSeenCommitteeMembers = cachedNotSeenValidators.get(cacheKey);
-    if (notSeenCommitteeMembers != null) {
-      // if all validators are seen then return null, we don't need to check for any attestations of same committee again
-      return notSeenCommitteeMembers.size === 0 ? null : notSeenCommitteeMembers;
+    if (notSeenCommitteeMembers !== undefined) {
+      return notSeenCommitteeMembers;
     }
 
     const decisionRoot = state.getShufflingDecisionRoot(computeEpochAtSlot(slot));
-    const committee = shufflingCache.getBeaconCommittee(epoch, decisionRoot, slot, committeeIndex);
+    const committee = shufflingCache.getBeaconCommitteeOrNull(epoch, decisionRoot, slot, committeeIndex);
+    if (committee === null) {
+      // attestations in the pool were validated against a cached shuffling of the previous or current epoch,
+      // so a miss is a ShufflingCache bug. Return null so the caller tracks it via metrics/logs without failing
+      // block production. Don't fall back to the state's shuffling, that would hide the bug
+      return null;
+    }
     notSeenCommitteeMembers = new Set<number>();
     for (const [i, validatorIndex] of committee.entries()) {
       // no need to check flagIsTimelySource as if validator is not seen, it's participation status is 0
@@ -786,8 +810,8 @@ export function getNotSeenValidatorsFn(
       }
     }
     cachedNotSeenValidators.set(cacheKey, notSeenCommitteeMembers);
-    // if all validators are seen then return null, we don't need to check for any attestations of same committee again
-    return notSeenCommitteeMembers.size === 0 ? null : notSeenCommitteeMembers;
+    // an empty set means all validators are seen, we don't need to check for any attestations of same committee again
+    return notSeenCommitteeMembers;
   };
 }
 

package/src/chain/opPools/proposerPreferencesPool.ts

@@ -6,8 +6,8 @@ import {toRootHex} from "@lodestar/utils";
  *
  * The primary consumer is `validateExecutionPayloadBid`, which looks up the matching
  * preferences via `get(bid.slot, dependent_root)` to enforce the IGNORE-existence and
- * REJECT-equality rules from the gloas spec. The beacon API `/pool/proposer_preferences`
- * GET endpoint reads from the same pool via `getAll`.
+ * REJECT-equality rules from the gloas spec. The beacon API `getProposerPreferences`
+ * endpoint reads from the same pool via `getAll`.
  *
  * `validator_index` is intentionally not part of the key: gossip validation enforces
  * `proposers[proposalSlot % SLOTS_PER_EPOCH] === validatorIndex` against the shuffling
@@ -37,15 +37,19 @@ export class ProposerPreferencesPool {
     byRoot.set(rootHex, signed);
   }
 
-  /** API read-out: flatten across branches, optionally filtered by slot. */
-  getAll(slot?: Slot): gloas.SignedProposerPreferences[] {
-    if (slot !== undefined) {
-      const byRoot = this.bySlot.get(slot);
-      return byRoot ? Array.from(byRoot.values()) : [];
-    }
+  /** API read-out: flatten across branches, optionally filtered by slot and dependent root. */
+  getAll(slot?: Slot, dependentRootHex?: RootHex): gloas.SignedProposerPreferences[] {
     const out: gloas.SignedProposerPreferences[] = [];
-    for (const byRoot of this.bySlot.values()) {
-      for (const v of byRoot.values()) out.push(v);
+    for (const [s, byRoot] of this.bySlot) {
+      if (slot !== undefined && s !== slot) {
+        continue;
+      }
+      for (const [root, signed] of byRoot) {
+        if (dependentRootHex !== undefined && root !== dependentRootHex) {
+          continue;
+        }
+        out.push(signed);
+      }
     }
     return out;
   }

package/src/chain/options.ts

@@ -27,8 +27,8 @@ export type IChainOptions = BlockProcessOpts &
     blsVerifyAllMainThread?: boolean;
     blsVerifyAllMultiThread?: boolean;
     blacklistedBlocks?: string[];
-    // TODO GLOAS: add similar option for execution payload envelopes?
     persistProducedBlocks?: boolean;
+    persistProducedPayloadEnvelopes?: boolean;
     /** Archive finalized Gloas payload envelopes as headers (bodies reconstructed from the EL on read) */
     dedupePayloads?: boolean;
     persistInvalidSszObjects?: boolean;

package/src/chain/seenCache/seenGossipBlockInput.ts

@@ -111,6 +111,9 @@ export class SeenBlockInput {
   // there should only 1 block root per slot but we need to always compare against rootHex
   // and the signature to ensure we only skip verification if both match
   private verifiedProposerSignatures = new Map<Slot, Map<RootHex, BLSSignature>>();
+  // Roots of gossip blocks under validation, counted because blocks with the same root but different signatures
+  // can be validated concurrently
+  private validatingBlockRoots = new Map<RootHex, number>();
 
   constructor({
     config,
@@ -153,6 +156,26 @@ export class SeenBlockInput {
     return this.blockInputs.get(rootHex)?.hasBlock() ?? false;
   }
 
+  /**
+   * Mark a gossip block root as under validation, every call MUST be paired with `unmarkValidatingBlock()`
+   */
+  markValidatingBlock(rootHex: RootHex): void {
+    this.validatingBlockRoots.set(rootHex, (this.validatingBlockRoots.get(rootHex) ?? 0) + 1);
+  }
+
+  unmarkValidatingBlock(rootHex: RootHex): void {
+    const count = this.validatingBlockRoots.get(rootHex) ?? 0;
+    if (count <= 1) {
+      this.validatingBlockRoots.delete(rootHex);
+    } else {
+      this.validatingBlockRoots.set(rootHex, count - 1);
+    }
+  }
+
+  isValidatingBlock(rootHex: RootHex): boolean {
+    return this.validatingBlockRoots.has(rootHex);
+  }
+
   get(rootHex: RootHex): IBlockInput | undefined {
     return this.blockInputs.get(rootHex);
   }

package/src/chain/shufflingCache.ts

@@ -7,14 +7,12 @@ import {
   getIndexedAttestation,
 } from "@lodestar/state-transition";
 import {Attestation, CommitteeIndex, Epoch, IndexedAttestation, RootHex, Slot} from "@lodestar/types";
-import {LodestarError, Logger, MapDef, pruneSetToMax} from "@lodestar/utils";
+import {LodestarError, Logger, MapDef} from "@lodestar/utils";
 import {Metrics} from "../metrics/metrics.js";
 
 /**
- * Same value to CheckpointBalancesCache, with the assumption that we don't have to use it for old epochs. In the worse case:
- * - when loading state bytes from disk, we need to compute shuffling for all epochs (~1s as of Sep 2023)
- * - don't have shuffling to verify attestations, need to do 1 epoch transition to add shuffling to this cache. This never happens
- * with default chain option of maxSkipSlots = 32
+ * Lookahead duties, gossip validation, and block production need epochs n-1, n, and n+1 for current epoch n.
+ * Keep the 4 highest epochs so that it works for epoch rotation.
  **/
 const MAX_EPOCHS = 4;
 
@@ -37,24 +35,29 @@ type ShufflingCacheItem = {
 type PromiseCacheItem = {
   type: CacheItemType.promise;
   timeInsertedMs: number;
-  promise: Promise<EpochShuffling>;
-  resolveFn: (shuffling: EpochShuffling) => void;
+  /** Resolves to null if the promise is cancelled */
+  promise: Promise<EpochShuffling | null>;
+  resolveFn: (shuffling: EpochShuffling | null) => void;
 };
 
 type CacheItem = ShufflingCacheItem | PromiseCacheItem;
 
+export enum ShufflingPromiseCancelReason {
+  regenError = "regen_error",
+  decisionRootMismatch = "decision_root_mismatch",
+  pruned = "pruned",
+}
+
 export type ShufflingCacheOpts = {
   maxShufflingCacheEpochs?: number;
 };
 
 /**
- * A shuffling cache to help:
- * - get committee quickly for attestation verification
- * - if a shuffling is not available (which does not happen with default chain option of maxSkipSlots = 32), track a promise to make sure we don't compute the same shuffling twice
- * - skip computing shuffling when loading state bytes from disk
+ * Cache recent epoch shufflings for lookahead duties, gossip validation, and block production.
+ * Pending promises deduplicate computation of missing shufflings.
  */
 export class ShufflingCache {
-  /** LRU cache implemented as a map, pruned every time we add an item */
+  /** Pruned to the `maxEpochs` highest epochs every time we add a shuffling */
   private readonly itemsByDecisionRootByEpoch: MapDef<Epoch, Map<RootHex, CacheItem>> = new MapDef(
     () => new Map<RootHex, CacheItem>()
   );
@@ -73,6 +76,9 @@ export class ShufflingCache {
           Array.from(this.itemsByDecisionRootByEpoch.values()).reduce((total, innerMap) => total + innerMap.size, 0)
         )
       );
+      metrics.shufflingCache.epochs.addCollect(() =>
+        metrics.shufflingCache.epochs.set(this.itemsByDecisionRootByEpoch.size)
+      );
     }
 
     this.maxEpochs = opts.maxShufflingCacheEpochs ?? MAX_EPOCHS;
@@ -97,8 +103,8 @@ export class ShufflingCache {
         `Too many shuffling promises: ${promiseCount}, shufflingEpoch: ${epoch}, decisionRootHex: ${decisionRoot}`
       );
     }
-    let resolveFn: ((shuffling: EpochShuffling) => void) | null = null;
-    const promise = new Promise<EpochShuffling>((resolve) => {
+    let resolveFn: ((shuffling: EpochShuffling | null) => void) | null = null;
+    const promise = new Promise<EpochShuffling | null>((resolve) => {
       resolveFn = resolve;
     });
     if (resolveFn === null) {
@@ -121,7 +127,7 @@ export class ShufflingCache {
    * Return null if we don't have a shuffling for this epoch and dependentRootHex.
    */
   async get(epoch: Epoch, decisionRoot: RootHex): Promise<EpochShuffling | null> {
-    const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
+    const cacheItem = this.itemsByDecisionRootByEpoch.get(epoch)?.get(decisionRoot);
     if (cacheItem === undefined) {
       this.metrics?.shufflingCache.miss.inc();
       return null;
@@ -136,12 +142,10 @@ export class ShufflingCache {
   }
 
   /**
-   * Get a shuffling synchronously, return null if not present.
-   * The only time we have a promise cache item is when we regen shuffling for attestation, which never happens
-   * with default chain option.
+   * Get a shuffling synchronously, return null if not present or if it's still being computed.
    */
   getSync(epoch: Epoch, decisionRoot: RootHex): EpochShuffling | null {
-    const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
+    const cacheItem = this.itemsByDecisionRootByEpoch.get(epoch)?.get(decisionRoot);
     if (cacheItem === undefined) {
       this.metrics?.shufflingCache.miss.inc();
       return null;
@@ -161,7 +165,7 @@ export class ShufflingCache {
    * `NativeBeaconStateView`.
    */
   has(epoch: Epoch, decisionRoot: RootHex): boolean {
-    const cacheItem = this.itemsByDecisionRootByEpoch.getOrDefault(epoch).get(decisionRoot);
+    const cacheItem = this.itemsByDecisionRootByEpoch.get(epoch)?.get(decisionRoot);
     return cacheItem !== undefined && isShufflingCacheItem(cacheItem);
   }
 
@@ -186,6 +190,25 @@ export class ShufflingCache {
     }
   }
 
+  /**
+   * Resolve a pending promise with null so that waiters don't hang, then remove it.
+   */
+  cancelPromise(epoch: Epoch, decisionRoot: RootHex, reason: ShufflingPromiseCancelReason): void {
+    const itemsByDecisionRoot = this.itemsByDecisionRootByEpoch.get(epoch);
+    const cacheItem = itemsByDecisionRoot?.get(decisionRoot);
+    if (itemsByDecisionRoot === undefined || cacheItem === undefined || !isPromiseCacheItem(cacheItem)) {
+      return;
+    }
+
+    cacheItem.resolveFn(null);
+    itemsByDecisionRoot.delete(decisionRoot);
+    if (itemsByDecisionRoot.size === 0) {
+      this.itemsByDecisionRootByEpoch.delete(epoch);
+    }
+    this.metrics?.shufflingCache.cancelledPromises.inc({reason});
+    this.logger?.debug("Cancelled shuffling promise", {epoch, decisionRoot, reason});
+  }
+
   getIndexedAttestation(
     epoch: number,
     decisionRoot: string,
@@ -205,6 +228,11 @@ export class ShufflingCache {
     return this.getBeaconCommittees(epoch, decisionRoot, slot, [index])[0];
   }
 
+  getBeaconCommitteeOrNull(epoch: number, decisionRoot: string, slot: Slot, index: CommitteeIndex): Uint32Array | null {
+    const shuffling = this.getSync(epoch, decisionRoot);
+    return shuffling === null ? null : getBeaconCommittees(shuffling, slot, [index])[0];
+  }
+
   getBeaconCommittees(epoch: number, decisionRoot: string, slot: Slot, indices: CommitteeIndex[]): Uint32Array[] {
     const shuffling = this.getShufflingOrThrow(epoch, decisionRoot);
     return getBeaconCommittees(shuffling, slot, indices);
@@ -243,8 +271,42 @@ export class ShufflingCache {
     }
     // set the shuffling
     shufflingAtEpoch.set(decisionRoot, {type: CacheItemType.shuffling, shuffling});
-    // prune the cache
-    pruneSetToMax(this.itemsByDecisionRootByEpoch, this.maxEpochs);
+    this.prune();
+  }
+
+  /**
+   * Keep the `maxEpochs` highest epochs, regardless of insertion order.
+   */
+  private prune(): void {
+    const toDelete = this.itemsByDecisionRootByEpoch.size - this.maxEpochs;
+    if (toDelete <= 0) {
+      return;
+    }
+
+    const prunedEpochs = Array.from(this.itemsByDecisionRootByEpoch.keys())
+      .sort((a, b) => a - b)
+      .slice(0, toDelete);
+    let prunedShufflings = 0;
+    for (const epoch of prunedEpochs) {
+      for (const cacheItem of this.itemsByDecisionRootByEpoch.get(epoch)?.values() ?? []) {
+        if (isPromiseCacheItem(cacheItem)) {
+          cacheItem.resolveFn(null);
+          this.metrics?.shufflingCache.cancelledPromises.inc({reason: ShufflingPromiseCancelReason.pruned});
+        } else {
+          prunedShufflings++;
+        }
+      }
+      this.itemsByDecisionRootByEpoch.delete(epoch);
+    }
+    this.metrics?.shufflingCache.prunedShufflings.inc(prunedShufflings);
+
+    this.logger?.verbose("Pruned shuffling cache", {
+      prunedEpochs: prunedEpochs.join(","),
+      prunedShufflings,
+      cachedEpochs: Array.from(this.itemsByDecisionRootByEpoch.keys())
+        .sort((a, b) => a - b)
+        .join(","),
+    });
   }
 }
 

package/src/db/beacon.ts

@@ -8,6 +8,7 @@ import {FlatFileStore} from "./flatFileStore/flatFileStore.js";
 import type {FlatFileStoreMetrics} from "./flatFileStore/metrics.js";
 import {IBeaconDb} from "./interface.js";
 import {CheckpointStateRepository} from "./repositories/checkpointState.js";
+import {EarliestAvailableSlotRepository} from "./repositories/earliestAvailableSlot.js";
 import {
   AttesterSlashingRepository,
   BLSToExecutionChangeRepository,
@@ -48,6 +49,7 @@ export class BeaconDb implements IBeaconDb {
 
   stateArchive: StateArchiveRepository;
   checkpointState: CheckpointStateRepository;
+  earliestAvailableSlot: EarliestAvailableSlotRepository;
 
   voluntaryExit: VoluntaryExitRepository;
   proposerSlashing: ProposerSlashingRepository;
@@ -83,6 +85,7 @@ export class BeaconDb implements IBeaconDb {
 
     this.stateArchive = new StateArchiveRepository(config, db);
     this.checkpointState = new CheckpointStateRepository(config, db);
+    this.earliestAvailableSlot = new EarliestAvailableSlotRepository(db);
     this.voluntaryExit = new VoluntaryExitRepository(config, db);
     this.blsToExecutionChange = new BLSToExecutionChangeRepository(config, db);
     this.proposerSlashing = new ProposerSlashingRepository(config, db);

package/src/db/buckets.ts

@@ -72,6 +72,7 @@ export enum Bucket {
 
   gloas_executionPayloadEnvelope = 59, // GLOAS BeaconBlockRoot -> SignedExecutionPayloadEnvelope
   gloas_executionPayloadEnvelopeArchive = 60, // GLOAS Slot -> SignedExecutionPayloadEnvelope, or 0x00 + SignedExecutionPayloadHeaderEnvelope
+  earliestAvailableSlot = 61, // singleton -> Slot
 }
 
 export function getBucketNameByValue<T extends Bucket>(enumValue: T): keyof typeof Bucket {

package/src/db/interface.ts

@@ -3,6 +3,7 @@ import {Slot} from "@lodestar/types";
 import type {IDataColumnStore} from "./dataColumnStore.js";
 import type {FlatFileStoreMetrics} from "./flatFileStore/metrics.js";
 import {CheckpointStateRepository} from "./repositories/checkpointState.js";
+import {EarliestAvailableSlotRepository} from "./repositories/earliestAvailableSlot.js";
 import {
   AttesterSlashingRepository,
   BLSToExecutionChangeRepository,
@@ -46,6 +47,7 @@ export interface IBeaconDb {
   stateArchive: StateArchiveRepository;
   // checkpoint states
   checkpointState: CheckpointStateRepository;
+  earliestAvailableSlot: EarliestAvailableSlotRepository;
 
   // op pool
   voluntaryExit: VoluntaryExitRepository;

package/src/db/repositories/earliestAvailableSlot.ts

@@ -0,0 +1,19 @@
+import {Db, encodeKey} from "@lodestar/db";
+import {Slot, ssz} from "@lodestar/types";
+import {Bucket, getBucketNameByValue} from "../buckets.js";
+
+export class EarliestAvailableSlotRepository {
+  private readonly key = encodeKey(Bucket.earliestAvailableSlot, 0);
+  private readonly opts = {bucketId: getBucketNameByValue(Bucket.earliestAvailableSlot)};
+
+  constructor(private readonly db: Db) {}
+
+  async get(): Promise<Slot | null> {
+    const bytes = await this.db.get(this.key, this.opts);
+    return bytes === null ? null : ssz.Slot.deserialize(bytes);
+  }
+
+  async set(slot: Slot): Promise<void> {
+    await this.db.put(this.key, ssz.Slot.serialize(slot), this.opts);
+  }
+}

package/src/metrics/metrics/lodestar.ts

@@ -20,6 +20,7 @@ import {InsertOutcome} from "../../chain/opPools/types.js";
 import {RegenCaller, RegenFnName} from "../../chain/regen/interface.js";
 import {ReprocessStatus} from "../../chain/reprocess.js";
 import {RejectReason} from "../../chain/seenCache/seenAttestationData.js";
+import type {ShufflingPromiseCancelReason} from "../../chain/shufflingCache.js";
 import {CacheItemType} from "../../chain/stateCache/types.js";
 import {OpSource} from "../../chain/validatorMonitor.js";
 import type {FlatFileStoreOperation} from "../../db/flatFileStore/metrics.js";
@@ -1281,6 +1282,10 @@ export function createLodestarMetrics(
             name: "lodestar_oppool_aggregated_attestation_pool_packed_attestations_seen_committees_total",
             help: "Total number of committees for which all members are seen when producing packed attestations",
           }),
+          missingShufflingCommittees: register.gauge({
+            name: "lodestar_oppool_aggregated_attestation_pool_packed_attestations_missing_shuffling_committees_total",
+            help: "Total number of committees skipped due to missing shuffling when producing packed attestations",
+          }),
         },
       },
       attestationPool: {
@@ -1614,6 +1619,19 @@ export function createLodestarMetrics(
         name: "lodestar_shuffling_cache_size",
         help: "Shuffling cache size",
       }),
+      epochs: register.gauge({
+        name: "lodestar_shuffling_cache_epochs",
+        help: "Number of epochs in shuffling cache",
+      }),
+      prunedShufflings: register.counter({
+        name: "lodestar_shuffling_cache_pruned_shufflings_total",
+        help: "Total number of shufflings pruned from shuffling cache",
+      }),
+      cancelledPromises: register.counter<{reason: ShufflingPromiseCancelReason}>({
+        name: "lodestar_shuffling_cache_cancelled_promises_total",
+        help: "Total number of shuffling promises resolved with null",
+        labelNames: ["reason"],
+      }),
       insertPromiseCount: register.gauge({
         name: "lodestar_shuffling_cache_insert_promise_count",
         help: "Total number of times insertPromise is called",