@lodestar/validator 1.51.0-dev.bf4ea87c7b → 1.51.0-dev.c6e566f915

package/src/services/inclusionListDuties.ts

@@ -0,0 +1,306 @@
+import {ApiClient, routes} from "@lodestar/api";
+import {ChainForkConfig} from "@lodestar/config";
+import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import {IClock, computeEpochAtSlot, isStartSlotOfEpoch} from "@lodestar/state-transition";
+import {Epoch, RootHex, Slot, ValidatorIndex} from "@lodestar/types";
+import {sleep, toPubkeyHex} from "@lodestar/utils";
+import {PubkeyHex} from "../types.js";
+import {LoggerVc} from "../util/index.js";
+import {ChainHeaderTracker, HeadEventData} from "./chainHeaderTracker.js";
+import {SyncingStatusTracker} from "./syncingStatusTracker.js";
+import {ValidatorStore} from "./validatorStore.js";
+
+/** Only retain `HISTORICAL_DUTIES_EPOCHS` duties prior to the current epoch. */
+const HISTORICAL_DUTIES_EPOCHS = 2;
+
+type InclusionListDuty = routes.validator.InclusionListDuty;
+// To assist with readability
+type InclusionListDutiesAtEpoch = {dependentRoot: RootHex; dutiesByIndex: Map<ValidatorIndex, InclusionListDuty>};
+
+/**
+ * Similar to AttestationDutiesService but
+ *   - No generating and caching selection proof
+ *   - No handling and maintaining subnet subscription
+ */
+export class InclusionListDutiesService {
+  /** Maps a validator index to its duty for each epoch */
+  private readonly dutiesByIndexByEpoch = new Map<Epoch, InclusionListDutiesAtEpoch>();
+  /**
+   * We may receive new dependentRoot of an epoch but it's not the last slot of epoch
+   * so we have to wait for getting close to the next epoch to redownload new inclusionListDuties.
+   */
+  private readonly pendingDependentRootByEpoch = new Map<Epoch, RootHex>();
+
+  constructor(
+    private readonly config: ChainForkConfig,
+    private readonly logger: LoggerVc,
+    private readonly api: ApiClient,
+    private clock: IClock,
+    private readonly validatorStore: ValidatorStore,
+    chainHeadTracker: ChainHeaderTracker,
+    syncingStatusTracker: SyncingStatusTracker
+  ) {
+    // Running this task every epoch is safe since a re-org of two epochs is very unlikely
+    clock.runEveryEpoch(this.runDutiesTasks);
+    clock.runEverySlot(this.prepareForNextEpoch);
+    chainHeadTracker.runOnNewHead(this.onNewHead);
+    syncingStatusTracker.runOnResynced(async (slot) => {
+      // Skip on first slot of epoch since tasks are already scheduled
+      if (!isStartSlotOfEpoch(slot)) {
+        return this.runDutiesTasks(computeEpochAtSlot(slot));
+      }
+    });
+  }
+
+  removeDutiesForKey(pubkey: PubkeyHex): void {
+    for (const [epoch, dutiesAtEpoch] of this.dutiesByIndexByEpoch) {
+      for (const [vIndex, duty] of dutiesAtEpoch.dutiesByIndex) {
+        if (toPubkeyHex(duty.pubkey) === pubkey) {
+          dutiesAtEpoch.dutiesByIndex.delete(vIndex);
+          if (dutiesAtEpoch.dutiesByIndex.size === 0) {
+            this.dutiesByIndexByEpoch.delete(epoch);
+          }
+        }
+      }
+    }
+  }
+
+  /** Returns all `ValidatorDuty` for the given `slot` */
+  getDutiesAtSlot(slot: Slot): InclusionListDuty[] {
+    const epoch = computeEpochAtSlot(slot);
+    const duties: InclusionListDuty[] = [];
+    const epochDuties = this.dutiesByIndexByEpoch.get(epoch);
+    if (epochDuties === undefined) {
+      return duties;
+    }
+
+    for (const validatorDuty of epochDuties.dutiesByIndex.values()) {
+      if (validatorDuty.slot === slot) {
+        duties.push(validatorDuty);
+      }
+    }
+
+    return duties;
+  }
+
+  /**
+   * If a reorg dependent root comes at a slot other than last slot of epoch
+   * just update this.pendingDependentRootByEpoch() and process here
+   */
+  private prepareForNextEpoch = async (slot: Slot, signal: AbortSignal): Promise<void> => {
+    // only interested in last slot of epoch
+    if ((slot + 1) % SLOTS_PER_EPOCH !== 0) {
+      return;
+    }
+
+    // during the 1 / 3 of epoch, last block of epoch may come
+    await sleep(this.clock.msToSlot(slot + 1 / 3), signal);
+
+    const nextEpoch = computeEpochAtSlot(slot) + 1;
+    const dependentRoot = this.dutiesByIndexByEpoch.get(nextEpoch)?.dependentRoot;
+    const pendingDependentRoot = this.pendingDependentRootByEpoch.get(nextEpoch);
+    if (dependentRoot && pendingDependentRoot && dependentRoot !== pendingDependentRoot) {
+      // this happens when pendingDependentRoot is not the last block of an epoch
+      this.logger.info("Redownload inclusion list duties when it's close to epoch boundary", {nextEpoch, slot});
+      await this.handleInclusionListDutiesReorg(nextEpoch, slot, dependentRoot, pendingDependentRoot);
+    }
+  };
+
+  private runDutiesTasks = async (epoch: Epoch): Promise<void> => {
+    // pollInclusionListCommittee also polls epoch + 1, so start one epoch before the fork
+    if (epoch + 1 < this.config.HEZE_FORK_EPOCH) {
+      return;
+    }
+
+    await Promise.all([
+      // Run pollInclusionListCommittee immediately for all known local indices
+      this.pollInclusionListCommittee(epoch, this.validatorStore.getAllLocalIndices()).catch((e: Error) => {
+        this.logger.error("Error on poll inclusion list committee", {epoch}, e);
+      }),
+
+      // At the same time fetch any remaining unknown validator indices, then poll duties for those newIndices only
+      this.validatorStore
+        .pollValidatorIndices()
+        .then((newIndices) => this.pollInclusionListCommittee(epoch, newIndices))
+        .catch((e: Error) => {
+          this.logger.error("Error on poll indices and inclusion list committee", {epoch}, e);
+        }),
+    ]);
+
+    // After both, prune
+    this.pruneOldDuties(epoch);
+  };
+
+  private async pollInclusionListCommittee(currentEpoch: Epoch, indexArr: ValidatorIndex[]): Promise<void> {
+    const nextEpoch = currentEpoch + 1;
+
+    // No need to bother the BN if we don't have any validators.
+    if (indexArr.length === 0) {
+      return;
+    }
+
+    for (const epoch of [currentEpoch, nextEpoch]) {
+      // Download the duties and update the duties for the current and next epoch.
+      await this.pollInclusionListCommitteeForEpoch(epoch, indexArr).catch((e: Error) => {
+        this.logger.error("Failed to download inclusion list duties", {epoch}, e);
+      });
+    }
+  }
+
+  /**
+   * For the given `indexArr`, download the duties for the given `epoch` and store them in duties.
+   */
+  private async pollInclusionListCommitteeForEpoch(epoch: Epoch, indexArr: ValidatorIndex[]): Promise<void> {
+    // Don't fetch duties for epochs before genesis. However, should fetch epoch 0 duties at epoch -1
+    if (epoch < 0) {
+      return;
+    }
+
+    // Inclusion list committees only exist post-heze; the beacon node rejects earlier epochs
+    if (epoch < this.config.HEZE_FORK_EPOCH) {
+      return;
+    }
+
+    const res = await this.api.validator.getInclusionListCommitteeDuties({epoch, indices: indexArr});
+    const inclusionListDuties = res.value();
+    const {dependentRoot} = res.meta();
+    const relevantDuties = inclusionListDuties.filter((duty) => {
+      const pubkeyHex = toPubkeyHex(duty.pubkey);
+      return this.validatorStore.hasVotingPubkey(pubkeyHex) && this.validatorStore.isDoppelgangerSafe(pubkeyHex);
+    });
+
+    this.logger.debug("Downloaded inclusion list duties", {epoch, dependentRoot, count: relevantDuties.length});
+
+    const dutiesAtEpoch = this.dutiesByIndexByEpoch.get(epoch);
+    const priorDependentRoot = dutiesAtEpoch?.dependentRoot;
+    const dependentRootChanged = priorDependentRoot !== undefined && priorDependentRoot !== dependentRoot;
+
+    if (!priorDependentRoot || dependentRootChanged) {
+      const dutiesByIndex = new Map<ValidatorIndex, InclusionListDuty>();
+      for (const duty of relevantDuties) {
+        dutiesByIndex.set(duty.validatorIndex, duty);
+      }
+      this.dutiesByIndexByEpoch.set(epoch, {dependentRoot, dutiesByIndex});
+
+      if (priorDependentRoot && dependentRootChanged) {
+        this.logger.warn("Inclusion list duties re-org. This may happen from time to time", {
+          priorDependentRoot: priorDependentRoot,
+          dependentRoot: dependentRoot,
+          epoch,
+        });
+      }
+    } else {
+      const existingDuties = dutiesAtEpoch.dutiesByIndex;
+      const existingDutiesCount = existingDuties.size;
+      for (const duty of relevantDuties) {
+        if (!existingDuties.has(duty.validatorIndex)) {
+          existingDuties.set(duty.validatorIndex, duty);
+        }
+      }
+
+      if (existingDuties.size > existingDutiesCount) {
+        this.logger.debug("Discovered new inclusion list duties", {
+          epoch,
+          dependentRoot,
+          count: existingDuties.size - existingDutiesCount,
+        });
+      }
+    }
+  }
+
+  private async handleInclusionListDutiesReorg(
+    dutyEpoch: Epoch,
+    slot: Slot,
+    oldDependentRoot: RootHex,
+    newDependentRoot: RootHex
+  ): Promise<void> {
+    const logContext = {dutyEpoch, slot, oldDependentRoot, newDependentRoot};
+    this.logger.debug("Redownload inclusion list duties", logContext);
+
+    await this.pollInclusionListCommitteeForEpoch(dutyEpoch, this.validatorStore.getAllLocalIndices())
+      .then(() => {
+        this.pendingDependentRootByEpoch.delete(dutyEpoch);
+      })
+      .catch((e: Error) => {
+        this.logger.error("Failed to redownload inclusion list duties when reorg happens", logContext, e);
+      });
+  }
+
+  /**
+   * inclusion list duties may be reorged due to 2 scenarios:
+   *   1. node is syncing (for nextEpoch duties)
+   *   2. node is reorged
+   * previousDutyDependentRoot = get_block_root_at_slot(state, compute_start_slot_at_epoch(epoch - 1) - 1)
+   *   => dependent root of current epoch
+   * currentDutyDependentRoot = get_block_root_at_slot(state, compute_start_slot_at_epoch(epoch) - 1)
+   *   => dependent root of next epoch
+   */
+  private onNewHead = async ({
+    slot,
+    head,
+    previousDutyDependentRoot,
+    currentDutyDependentRoot,
+  }: HeadEventData): Promise<void> => {
+    const currentEpoch = computeEpochAtSlot(slot);
+    const nextEpoch = currentEpoch + 1;
+    const nextTwoEpoch = currentEpoch + 2;
+    const nextTwoEpochDependentRoot = this.dutiesByIndexByEpoch.get(currentEpoch + 2)?.dependentRoot;
+
+    // this may happen ONLY when node is syncing
+    // it's safe to get inclusion list duties at epoch n + 1 thanks to nextEpochShuffling cache
+    // but it's an issue to request inclusion list duties for epoch n + 2 as dependent root keeps changing while node is syncing
+    // see https://github.com/ChainSafe/lodestar/issues/3211
+    if (nextTwoEpochDependentRoot && head !== nextTwoEpochDependentRoot) {
+      // last slot of epoch, we're sure it's the correct dependent root
+      if ((slot + 1) % SLOTS_PER_EPOCH === 0) {
+        this.logger.info("Next 2 epoch inclusion list duties reorg", {slot, dutyEpoch: nextTwoEpoch, head});
+        await this.handleInclusionListDutiesReorg(nextTwoEpoch, slot, nextTwoEpochDependentRoot, head);
+      } else {
+        this.logger.debug("Potential next 2 epoch inclusion list duties reorg", {slot, dutyEpoch: nextTwoEpoch, head});
+        // node may send adjacent onHead events while it's syncing
+        // wait for getting close to next epoch to make sure the dependRoot
+        this.pendingDependentRootByEpoch.set(nextTwoEpoch, head);
+      }
+    }
+
+    // dependent root for next epoch changed
+    const nextEpochDependentRoot = this.dutiesByIndexByEpoch.get(nextEpoch)?.dependentRoot;
+    if (nextEpochDependentRoot && currentDutyDependentRoot !== nextEpochDependentRoot) {
+      this.logger.warn("Potential next epoch inclusion list duties reorg", {
+        slot,
+        dutyEpoch: nextEpoch,
+        priorDependentRoot: nextEpochDependentRoot,
+        newDependentRoot: currentDutyDependentRoot,
+      });
+      await this.handleInclusionListDutiesReorg(nextEpoch, slot, nextEpochDependentRoot, currentDutyDependentRoot);
+    }
+
+    // dependent root for current epoch changed
+    const currentEpochDependentRoot = this.dutiesByIndexByEpoch.get(currentEpoch)?.dependentRoot;
+    if (currentEpochDependentRoot && currentEpochDependentRoot !== previousDutyDependentRoot) {
+      this.logger.warn("Potential current epoch inclusion list duties reorg", {
+        slot,
+        dutyEpoch: currentEpoch,
+        priorDependentRoot: currentEpochDependentRoot,
+        newDependentRoot: previousDutyDependentRoot,
+      });
+      await this.handleInclusionListDutiesReorg(
+        currentEpoch,
+        slot,
+        currentEpochDependentRoot,
+        previousDutyDependentRoot
+      );
+    }
+  };
+
+  /** Run once per epoch to prune duties map */
+  private pruneOldDuties(currentEpoch: Epoch): void {
+    for (const byEpochMap of [this.dutiesByIndexByEpoch, this.pendingDependentRootByEpoch]) {
+      for (const epoch of byEpochMap.keys()) {
+        if (epoch + HISTORICAL_DUTIES_EPOCHS < currentEpoch) {
+          byEpochMap.delete(epoch);
+        }
+      }
+    }
+  }
+}

package/src/services/validatorStore.ts

@@ -11,6 +11,7 @@ import {
   DOMAIN_BEACON_PROPOSER,
   DOMAIN_BUILDER_REQUEST_AUTH,
   DOMAIN_CONTRIBUTION_AND_PROOF,
+  DOMAIN_INCLUSION_LIST_COMMITTEE,
   DOMAIN_PROPOSER_PREFERENCES,
   DOMAIN_PTC_ATTESTER,
   DOMAIN_RANDAO,
@@ -46,6 +47,7 @@ import {
   altair,
   bellatrix,
   gloas,
+  heze,
   phase0,
   ssz,
 } from "@lodestar/types";
@@ -633,7 +635,8 @@ export class ValidatorStore {
   removeSigner(pubkeyHex: PubkeyHex): boolean {
     this.doppelgangerService?.unregisterValidator(pubkeyHex);
 
-    return this.indicesService.removeForKey(pubkeyHex) || this.validators.delete(pubkeyHex);
+    this.indicesService.removeForKey(pubkeyHex);
+    return this.validators.delete(pubkeyHex);
   }
 
   /** Return true if there is at least 1 pubkey registered */
@@ -809,6 +812,8 @@ export class ValidatorStore {
   ): Promise<SignedAggregateAndProof> {
     this.validateAttestationDuty(duty, aggregate.data);
 
+    this.assertDoppelgangerSafe(duty.pubkey);
+
     const aggregateAndProof: AggregateAndProof = {
       aggregate,
       aggregatorIndex: duty.validatorIndex,
@@ -929,8 +934,6 @@ export class ValidatorStore {
       throw Error(`Not signing proposer preferences for past slot ${duty.slot} (current ${currentSlot})`);
     }
 
-    this.assertDoppelgangerSafe(duty.pubkey);
-
     const message: gloas.ProposerPreferences = {
       dependentRoot,
       proposalSlot: duty.slot,
@@ -954,6 +957,38 @@ export class ValidatorStore {
     };
   }
 
+  async signInclusionList(
+    duty: routes.validator.InclusionListDuty,
+    inclusionList: heze.InclusionList
+  ): Promise<heze.SignedInclusionList> {
+    if (duty.slot !== inclusionList.slot) {
+      throw Error(
+        `Inconsistent duties during signing: duty.slot ${duty.slot} != inclusionList.slot ${inclusionList.slot}`
+      );
+    }
+    if (duty.validatorIndex !== inclusionList.validatorIndex) {
+      throw Error(
+        `Inconsistent duties during signing: duty.validatorIndex ${duty.validatorIndex} != inclusionList.validatorIndex ${inclusionList.validatorIndex}`
+      );
+    }
+
+    this.assertDoppelgangerSafe(duty.pubkey);
+
+    const signingSlot = inclusionList.slot;
+    const domain = this.config.getDomain(signingSlot, DOMAIN_INCLUSION_LIST_COMMITTEE);
+    const signingRoot = computeSigningRoot(ssz.heze.InclusionList, inclusionList, domain);
+
+    const signableMessage: SignableMessage = {
+      type: SignableMessageType.INCLUSION_LIST,
+      data: inclusionList,
+    };
+
+    return {
+      message: inclusionList,
+      signature: await this.getSignature(duty.pubkey, signingRoot, signingSlot, signableMessage),
+    };
+  }
+
   async signAttestationSelectionProof(pubkey: BLSPubkeyMaybeHex, slot: Slot): Promise<BLSSignature> {
     const signingSlot = slot;
     const domain = this.config.getDomain(slot, DOMAIN_SELECTION_PROOF);
@@ -1214,6 +1249,11 @@ export class ValidatorStore {
     }
   }
 
+  /**
+   * Required before signing a message that is slashable or that the beacon node counts as validator
+   * liveness (see `validatorSeenAtEpoch`), otherwise doppelganger detection would flag our own messages.
+   * Skip it for anything else.
+   */
   private assertDoppelgangerSafe(pubKey: PubkeyHex | BLSPubkey): void {
     const pubkeyHex = typeof pubKey === "string" ? pubKey : toPubkeyHex(pubKey);
     if (!this.isDoppelgangerSafe(pubkeyHex)) {

package/src/slashingProtection/attestation/index.ts

@@ -1,4 +1,5 @@
 import {BLSPubkey, Epoch} from "@lodestar/types";
+import {defer, toPubkeyHex} from "@lodestar/utils";
 import {MinMaxSurround, SurroundAttestationError, SurroundAttestationErrorCode} from "../minMaxSurround/index.js";
 import {SlashingProtectionAttestation} from "../types.js";
 import {ZERO_ROOT, isEqualNonZeroRoot, isEqualRoot, minEpoch} from "../utils.js";
@@ -21,6 +22,7 @@ export class SlashingProtectionAttestationService {
   private attestationByTarget: AttestationByTargetRepository;
   private attestationLowerBound: AttestationLowerBoundRepository;
   private minMaxSurround: MinMaxSurround;
+  private readonly pendingAttestations = new Map<string, Promise<void>>();
 
   constructor(
     signedAttestationDb: AttestationByTargetRepository,
@@ -37,10 +39,25 @@ export class SlashingProtectionAttestationService {
    * This is the safe, externally-callable interface for checking attestations
    */
   async checkAndInsertAttestation(pubKey: BLSPubkey, attestation: SlashingProtectionAttestation): Promise<void> {
-    const safeStatus = await this.checkAttestation(pubKey, attestation);
+    const pubkeyHex = toPubkeyHex(pubKey);
+    const previous = this.pendingAttestations.get(pubkeyHex);
+    const {promise, resolve} = defer<void>();
+    this.pendingAttestations.set(pubkeyHex, promise);
 
-    if (safeStatus !== SafeStatus.SAME_DATA) {
-      await this.insertAttestation(pubKey, attestation);
+    try {
+      if (previous !== undefined) {
+        await previous;
+      }
+      const safeStatus = await this.checkAttestation(pubKey, attestation);
+
+      if (safeStatus !== SafeStatus.SAME_DATA) {
+        await this.insertAttestation(pubKey, attestation);
+      }
+    } finally {
+      resolve();
+      if (this.pendingAttestations.get(pubkeyHex) === promise) {
+        this.pendingAttestations.delete(pubkeyHex);
+      }
     }
 
     // TODO: Implement safe clean-up of stored attestations
@@ -164,55 +181,73 @@ export class SlashingProtectionAttestationService {
    * Interchange import / export functionality
    */
   async importAttestations(pubkey: BLSPubkey, attestations: SlashingProtectionAttestation[]): Promise<void> {
-    // Min-max surround misses a surround vote beyond its lookback, require the highest target attestation to
-    // have the highest source epoch as `checkAttestation` relies on it
-    let latestAtt = await this.attestationByTarget.getLatest(pubkey);
-    let maxSourceAtt = latestAtt;
-    for (const attestation of attestations) {
-      if (latestAtt === null || attestation.targetEpoch >= latestAtt.targetEpoch) {
-        latestAtt = attestation;
+    const pubkeyHex = toPubkeyHex(pubkey);
+    const previous = this.pendingAttestations.get(pubkeyHex);
+    const {promise, resolve} = defer<void>();
+    this.pendingAttestations.set(pubkeyHex, promise);
+
+    try {
+      if (previous !== undefined) {
+        await previous;
       }
-      if (maxSourceAtt === null || attestation.sourceEpoch > maxSourceAtt.sourceEpoch) {
-        maxSourceAtt = attestation;
+      // Min-max surround misses a surround vote beyond its lookback, require the highest target attestation to
+      // have the highest source epoch as `checkAttestation` relies on it
+      let latestAtt = await this.attestationByTarget.getLatest(pubkey);
+      let maxSourceAtt = latestAtt;
+      for (const attestation of attestations) {
+        if (latestAtt === null || attestation.targetEpoch >= latestAtt.targetEpoch) {
+          latestAtt = attestation;
+        }
+        if (maxSourceAtt === null || attestation.sourceEpoch > maxSourceAtt.sourceEpoch) {
+          maxSourceAtt = attestation;
+        }
       }
-    }
-    if (latestAtt && maxSourceAtt && latestAtt.sourceEpoch < maxSourceAtt.sourceEpoch) {
-      throw new InvalidAttestationError({
-        code: InvalidAttestationErrorCode.NEW_SURROUNDS_PREV,
-        attestation: latestAtt,
-        prev: maxSourceAtt,
-      });
-    }
-
-    // Never replace a recorded attestation with a different source or signing root, a zero root refuses any attestation
-    // with that target. The highest source epoch is kept as the latest attestation must have it, see above.
-    const attestationsByTarget = new Map<Epoch, SlashingProtectionAttestation>();
-    for (const attestation of attestations) {
-      const {sourceEpoch, targetEpoch, signingRoot} = attestation;
-      const prevAtt =
-        attestationsByTarget.get(targetEpoch) ?? (await this.attestationByTarget.get(pubkey, targetEpoch));
-      if (prevAtt === null || (prevAtt.sourceEpoch === sourceEpoch && isEqualRoot(prevAtt.signingRoot, signingRoot))) {
-        attestationsByTarget.set(targetEpoch, attestation);
-      } else {
-        attestationsByTarget.set(targetEpoch, {
-          sourceEpoch: Math.max(prevAtt.sourceEpoch, sourceEpoch),
-          targetEpoch,
-          signingRoot: ZERO_ROOT,
+      if (latestAtt && maxSourceAtt && latestAtt.sourceEpoch < maxSourceAtt.sourceEpoch) {
+        throw new InvalidAttestationError({
+          code: InvalidAttestationErrorCode.NEW_SURROUNDS_PREV,
+          attestation: latestAtt,
+          prev: maxSourceAtt,
         });
       }
-    }
-    await this.attestationByTarget.set(pubkey, Array.from(attestationsByTarget.values()));
 
-    // Pre-compute spans for all attestations
-    for (const attestation of attestations) {
-      await this.minMaxSurround.insertAttestation(pubkey, attestation);
-    }
+      // Never replace a recorded attestation with a different source or signing root, a zero root refuses any attestation
+      // with that target. The highest source epoch is kept as the latest attestation must have it, see above.
+      const attestationsByTarget = new Map<Epoch, SlashingProtectionAttestation>();
+      for (const attestation of attestations) {
+        const {sourceEpoch, targetEpoch, signingRoot} = attestation;
+        const prevAtt =
+          attestationsByTarget.get(targetEpoch) ?? (await this.attestationByTarget.get(pubkey, targetEpoch));
+        if (
+          prevAtt === null ||
+          (prevAtt.sourceEpoch === sourceEpoch && isEqualRoot(prevAtt.signingRoot, signingRoot))
+        ) {
+          attestationsByTarget.set(targetEpoch, attestation);
+        } else {
+          attestationsByTarget.set(targetEpoch, {
+            sourceEpoch: Math.max(prevAtt.sourceEpoch, sourceEpoch),
+            targetEpoch,
+            signingRoot: ZERO_ROOT,
+          });
+        }
+      }
+      await this.attestationByTarget.set(pubkey, Array.from(attestationsByTarget.values()));
+
+      // Pre-compute spans for all attestations
+      for (const attestation of attestations) {
+        await this.minMaxSurround.insertAttestation(pubkey, attestation);
+      }
 
-    // Pre-compute and store lower-bound
-    const minSourceEpoch = minEpoch(attestations.map((attestation) => attestation.sourceEpoch));
-    const minTargetEpoch = minEpoch(attestations.map((attestation) => attestation.targetEpoch));
-    if (minSourceEpoch != null && minTargetEpoch != null) {
-      await this.attestationLowerBound.set(pubkey, {minSourceEpoch, minTargetEpoch});
+      // Pre-compute and store lower-bound
+      const minSourceEpoch = minEpoch(attestations.map((attestation) => attestation.sourceEpoch));
+      const minTargetEpoch = minEpoch(attestations.map((attestation) => attestation.targetEpoch));
+      if (minSourceEpoch != null && minTargetEpoch != null) {
+        await this.attestationLowerBound.set(pubkey, {minSourceEpoch, minTargetEpoch});
+      }
+    } finally {
+      resolve();
+      if (this.pendingAttestations.get(pubkeyHex) === promise) {
+        this.pendingAttestations.delete(pubkeyHex);
+      }
     }
   }
 

package/src/slashingProtection/index.ts

@@ -1,8 +1,8 @@
 import {GENESIS_EPOCH} from "@lodestar/params";
 import {BLSPubkey, Epoch, Root} from "@lodestar/types";
-import {Logger, toPubkeyHex} from "@lodestar/utils";
+import {Logger, defer, toPubkeyHex} from "@lodestar/utils";
 import {uniqueVectorArr} from "../slashingProtection/utils.js";
-import {LodestarValidatorDatabaseController} from "../types.js";
+import {LodestarValidatorDatabaseController, PubkeyHex} from "../types.js";
 import {
   AttestationByTargetRepository,
   AttestationLowerBoundRepository,
@@ -34,6 +34,8 @@ export type {ISlashingProtection, InterchangeFormatVersion, SlashingProtectionBl
 export class SlashingProtection implements ISlashingProtection {
   private blockService: SlashingProtectionBlockService;
   private attestationService: SlashingProtectionAttestationService;
+  private readonly importing = new Map<PubkeyHex, Promise<void>>();
+  private lastImport: Promise<void> | null = null;
 
   constructor(protected db: LodestarValidatorDatabaseController) {
     const blockBySlotRepository = new BlockBySlotRepository(db);
@@ -51,10 +53,26 @@ export class SlashingProtection implements ISlashingProtection {
   }
 
   async checkAndInsertBlockProposal(pubKey: BLSPubkey, block: SlashingProtectionBlock): Promise<void> {
+    if (this.importing.size !== 0) {
+      const pubkeyHex = toPubkeyHex(pubKey);
+      let importing = this.importing.get(pubkeyHex);
+      while (importing !== undefined) {
+        await importing;
+        importing = this.importing.get(pubkeyHex);
+      }
+    }
     await this.blockService.checkAndInsertBlockProposal(pubKey, block);
   }
 
   async checkAndInsertAttestation(pubKey: BLSPubkey, attestation: SlashingProtectionAttestation): Promise<void> {
+    if (this.importing.size !== 0) {
+      const pubkeyHex = toPubkeyHex(pubKey);
+      let importing = this.importing.get(pubkeyHex);
+      while (importing !== undefined) {
+        await importing;
+        importing = this.importing.get(pubkeyHex);
+      }
+    }
     await this.attestationService.checkAndInsertAttestation(pubKey, attestation);
   }
 
@@ -84,10 +102,36 @@ export class SlashingProtection implements ISlashingProtection {
         }
       }
     }
-    for (const validator of data) {
-      logger?.info("Importing slashing protection", {pubkey: toPubkeyHex(validator.pubkey)});
-      await this.blockService.importBlocks(validator.pubkey, validator.signedBlocks);
-      await this.attestationService.importAttestations(validator.pubkey, validator.signedAttestations);
+
+    // Checks for keys in the file wait for the whole import, a key that is already signing must not sign against
+    // a history the import is still extending. Imports run one after another so a later one cannot release a key early.
+    const pubkeyHexes = data.map((validator) => toPubkeyHex(validator.pubkey));
+    const previous = this.lastImport;
+    const {promise, resolve} = defer<void>();
+    this.lastImport = promise;
+    for (const pubkeyHex of pubkeyHexes) {
+      this.importing.set(pubkeyHex, promise);
+    }
+
+    try {
+      if (previous !== null) {
+        await previous;
+      }
+      for (const validator of data) {
+        logger?.info("Importing slashing protection", {pubkey: toPubkeyHex(validator.pubkey)});
+        await this.blockService.importBlocks(validator.pubkey, validator.signedBlocks);
+        await this.attestationService.importAttestations(validator.pubkey, validator.signedAttestations);
+      }
+    } finally {
+      resolve();
+      if (this.lastImport === promise) {
+        this.lastImport = null;
+      }
+      for (const pubkeyHex of pubkeyHexes) {
+        if (this.importing.get(pubkeyHex) === promise) {
+          this.importing.delete(pubkeyHex);
+        }
+      }
     }
   }