@lodestar/validator 1.43.0-dev.bc569affb9 → 1.43.0-dev.c98da75ec7

package/src/services/blockDuties.ts

@@ -20,12 +20,12 @@ const HISTORICAL_DUTIES_EPOCHS = 2;
 const GENESIS_EPOCH = 0;
 export const GENESIS_SLOT = 0;
 
-type BlockDutyAtEpoch = {dependentRoot: RootHex; data: routes.validator.ProposerDuty[]};
+export type BlockDutyAtEpoch = {dependentRoot: RootHex; data: routes.validator.ProposerDuty[]};
 type NotifyBlockProductionFn = (slot: Slot, proposers: BLSPubkey[]) => void;
 
 export class BlockDutiesService {
   /** Notify the block service if it should produce a block. */
-  private readonly notifyBlockProductionFn: NotifyBlockProductionFn;
+  private notifyBlockProductionFn: NotifyBlockProductionFn = () => {};
   /** Maps an epoch to all *local* proposers in this epoch. Notably, this does not contain
       proposals for any validators which are not registered locally. */
   private readonly proposers = new Map<Epoch, BlockDutyAtEpoch>();
@@ -36,11 +36,8 @@ export class BlockDutiesService {
     private readonly api: ApiClient,
     private readonly clock: IClock,
     private readonly validatorStore: ValidatorStore,
-    private readonly metrics: Metrics | null,
-    notifyBlockProductionFn: NotifyBlockProductionFn
+    private readonly metrics: Metrics | null
   ) {
-    this.notifyBlockProductionFn = notifyBlockProductionFn;
-
     // TODO: Instead of polling every CLOCK_SLOT, poll every CLOCK_EPOCH and track re-org events
     //       only then re-fetch the block duties. Make sure most clients (including Lodestar)
     //       properly emit the re-org event
@@ -53,6 +50,14 @@ export class BlockDutiesService {
     }
   }
 
+  /**
+   * Late-bind the production callback. Allows the duties service to be constructed
+   * before the consumer that handles proposal production.
+   */
+  setNotifyBlockProductionFn(notifyBlockProductionFn: NotifyBlockProductionFn): void {
+    this.notifyBlockProductionFn = notifyBlockProductionFn;
+  }
+
   /**
    * Returns the pubkeys of the validators which are assigned to propose in the given slot.
    *
@@ -75,6 +80,16 @@ export class BlockDutiesService {
     return Array.from(publicKeys.values());
   }
 
+  /**
+   * Returns the cached `{dependentRoot, data}` entry for `epoch`, or `undefined` if duties
+   * for that epoch are not yet known. Consumers can detect a proposer-shuffling change
+   * (e.g. after a reorg) by observing a different `dependentRoot` than the one they last
+   * read for the same epoch.
+   */
+  getProposersAtEpoch(epoch: Epoch): BlockDutyAtEpoch | undefined {
+    return this.proposers.get(epoch);
+  }
+
   removeDutiesForKey(pubkey: PubkeyHex): void {
     for (const blockDutyAtEpoch of this.proposers.values()) {
       blockDutyAtEpoch.data = blockDutyAtEpoch.data.filter((proposer) => {

package/src/services/chainHeaderTracker.ts

@@ -1,4 +1,5 @@
 import {ApiClient, routes} from "@lodestar/api";
+import {BeaconConfig} from "@lodestar/config";
 import {GENESIS_SLOT} from "@lodestar/params";
 import {Root, RootHex, Slot} from "@lodestar/types";
 import {Logger, fromHex} from "@lodestar/utils";
@@ -13,6 +14,11 @@ export type HeadEventData = {
   currentDutyDependentRoot: RootHex;
 };
 
+export type ExecutionPayloadAvailableEventData = {
+  slot: Slot;
+  blockRoot: RootHex;
+};
+
 type RunEveryFn = (event: HeadEventData) => Promise<void>;
 
 /**
@@ -24,26 +30,35 @@ export class ChainHeaderTracker {
   private readonly fns: RunEveryFn[] = [];
 
   constructor(
+    private readonly config: BeaconConfig,
     private readonly logger: Logger,
     private readonly api: ApiClient,
     private readonly emitter: ValidatorEventEmitter
   ) {}
 
   start(signal: AbortSignal): void {
-    this.logger.verbose("Subscribing to head event");
+    this.logger.verbose("Subscribing to validator events");
+
+    const topics = [EventType.head];
+    // We wait until the gloas fork is configured to avoid breaking
+    // connections with pre-gloas beacon nodes
+    if (this.config.GLOAS_FORK_EPOCH !== Infinity) {
+      topics.push(EventType.executionPayloadAvailable);
+    }
+
     this.api.events
       .eventstream({
-        topics: [EventType.head],
+        topics,
         signal,
-        onEvent: this.onHeadUpdate,
+        onEvent: this.onEvent,
         onError: (e) => {
-          this.logger.error("Failed to receive head event", {}, e);
+          this.logger.error("Failed to receive validator event", {}, e);
         },
         onClose: () => {
-          this.logger.verbose("Closed stream for head event", {});
+          this.logger.verbose("Closed stream for validator events", {});
         },
       })
-      .catch((e) => this.logger.error("Failed to subscribe to head event", {}, e));
+      .catch((e) => this.logger.error("Failed to subscribe to validator events", {}, e));
   }
 
   getCurrentChainHead(slot: Slot): Root | null {
@@ -58,7 +73,7 @@ export class ChainHeaderTracker {
     this.fns.push(fn);
   }
 
-  private onHeadUpdate = (event: routes.events.BeaconEvent): void => {
+  private onEvent = (event: routes.events.BeaconEvent): void => {
     if (event.type === EventType.head) {
       const {message} = event;
       const {slot, block, previousDutyDependentRoot, currentDutyDependentRoot} = message;
@@ -85,5 +100,14 @@ export class ChainHeaderTracker {
         currentDuty: currentDutyDependentRoot,
       });
     }
+
+    if (event.type === EventType.executionPayloadAvailable) {
+      this.emitter.emit(ValidatorEvent.executionPayloadAvailable, event.message);
+
+      this.logger.verbose("Found execution payload available", {
+        slot: event.message.slot,
+        blockRoot: event.message.blockRoot,
+      });
+    }
   };
 }

package/src/services/emitter.ts

@@ -8,10 +8,20 @@ export enum ValidatorEvent {
    * This event signals that the node chain has a new head.
    */
   chainHead = "chainHead",
+  /**
+   * This event signals that an execution payload and blobs are available for payload attestation.
+   */
+  executionPayloadAvailable = "executionPayloadAvailable",
 }
 
+export type ExecutionPayloadAvailableEventData = {
+  slot: Slot;
+  blockRoot: string;
+};
+
 export type ValidatorEvents = {
   [ValidatorEvent.chainHead]: (head: HeadEventData) => void;
+  [ValidatorEvent.executionPayloadAvailable]: (payload: ExecutionPayloadAvailableEventData) => void;
 };
 
 /**
@@ -40,4 +50,25 @@ export class ValidatorEventEmitter extends (EventEmitter as {
       this.on(ValidatorEvent.chainHead, headListener);
     });
   }
+
+  /**
+   * Wait for the first execution payload availability event to come with slot >= provided slot.
+   */
+  async waitForExecutionPayloadAvailableSlot(slot: Slot): Promise<void> {
+    let payloadListener: (payload: ExecutionPayloadAvailableEventData) => void;
+
+    const onDone = (): void => {
+      this.off(ValidatorEvent.executionPayloadAvailable, payloadListener);
+    };
+
+    return new Promise((resolve) => {
+      payloadListener = (payload): void => {
+        if (payload.slot >= slot) {
+          onDone();
+          resolve();
+        }
+      };
+      this.on(ValidatorEvent.executionPayloadAvailable, payloadListener);
+    });
+  }
 }

package/src/services/proposerPreferences.ts

@@ -0,0 +1,124 @@
+import {ApiClient} from "@lodestar/api";
+import {ChainForkConfig} from "@lodestar/config";
+import {SLOTS_PER_EPOCH, isForkPostGloas} from "@lodestar/params";
+import {computeEpochAtSlot} from "@lodestar/state-transition";
+import {Epoch, RootHex, Slot, gloas} from "@lodestar/types";
+import {fromHex, toPubkeyHex} from "@lodestar/utils";
+import {Metrics} from "../metrics.js";
+import {IClock, LoggerVc} from "../util/index.js";
+import {BlockDutiesService} from "./blockDuties.js";
+import {ValidatorStore} from "./validatorStore.js";
+
+/**
+ * Submit a proposer's `SignedProposerPreferences` this many slots before the proposal slot.
+ *
+ * Earlier submission means more reorg-triggered resubmits (and gossip flood); later
+ * submission risks missing the bid-auction window for this proposal slot. The bid for
+ * slot S typically arrives at slot S-1, so we want preferences propagated to the network
+ * and consumed by builders before then. SLOTS_PER_EPOCH / 4 (8 slots @ 32 SPE, ~96s @ 12s
+ * slots) gives ample margin while bounding redundant resubmits.
+ */
+const SUBMIT_BEFORE_PROPOSAL_SLOTS = Math.floor(SLOTS_PER_EPOCH / 4);
+
+/** Per-epoch tracking of preferences already submitted under the current dependent_root. */
+type SubmittedAtEpoch = {dependentRoot: RootHex; slots: Set<Slot>};
+
+/**
+ * Signs and submits `SignedProposerPreferences` for any local validator that will propose
+ * within the next `SUBMIT_BEFORE_PROPOSAL_SLOTS`. Re-submits automatically when the proposer
+ * dependent root for an epoch shifts (e.g. after a reorg) — detected by comparing the cached
+ * `dependentRoot` reported by `BlockDutiesService` against the one we last submitted under.
+ *
+ * No-op pre-gloas.
+ */
+export class ProposerPreferencesService {
+  private readonly submitted = new Map<Epoch, SubmittedAtEpoch>();
+
+  constructor(
+    private readonly config: ChainForkConfig,
+    private readonly logger: LoggerVc,
+    private readonly api: ApiClient,
+    clock: IClock,
+    private readonly validatorStore: ValidatorStore,
+    private readonly blockDutiesService: BlockDutiesService,
+    _metrics: Metrics | null
+  ) {
+    clock.runEverySlot(this.runProposerPreferencesTask);
+  }
+
+  private runProposerPreferencesTask = async (slot: Slot): Promise<void> => {
+    if (!isForkPostGloas(this.config.getForkName(slot))) {
+      return;
+    }
+
+    const currentEpoch = computeEpochAtSlot(slot);
+    const batch: gloas.SignedProposerPreferences[] = [];
+    // Track which `(submission, slot)` pairs are pending an API submission so we can mark
+    // them only after the network call succeeds. Marking before would silently drop a
+    // preference on transient API failure (no retry until dependent_root shifts).
+    const pending: {submission: SubmittedAtEpoch; slot: Slot}[] = [];
+
+    for (const epoch of [currentEpoch, currentEpoch + 1]) {
+      const dutiesAtEpoch = this.blockDutiesService.getProposersAtEpoch(epoch);
+      if (!dutiesAtEpoch) continue;
+
+      // Reset submission tracking if the dependent root for this epoch has shifted
+      // (e.g. due to a reorg). Any previously-submitted preferences are now stale.
+      let submission = this.submitted.get(epoch);
+      if (submission === undefined || submission.dependentRoot !== dutiesAtEpoch.dependentRoot) {
+        if (submission !== undefined) {
+          this.logger.info("Proposer-shuffling dependent root shifted; resubmitting preferences", {
+            epoch,
+            priorDependentRoot: submission.dependentRoot,
+            dependentRoot: dutiesAtEpoch.dependentRoot,
+          });
+        }
+        submission = {dependentRoot: dutiesAtEpoch.dependentRoot, slots: new Set()};
+        this.submitted.set(epoch, submission);
+      }
+
+      const dependentRootBytes = fromHex(dutiesAtEpoch.dependentRoot);
+
+      for (const duty of dutiesAtEpoch.data) {
+        if (duty.slot <= slot) continue;
+        if (duty.slot > slot + SUBMIT_BEFORE_PROPOSAL_SLOTS) continue;
+        if (submission.slots.has(duty.slot)) continue;
+
+        try {
+          const pubkeyHex = toPubkeyHex(duty.pubkey);
+          const signed = await this.validatorStore.signProposerPreferences(
+            duty,
+            dependentRootBytes,
+            this.validatorStore.getFeeRecipient(pubkeyHex),
+            this.validatorStore.getGasLimit(pubkeyHex),
+            slot
+          );
+          batch.push(signed);
+          pending.push({submission, slot: duty.slot});
+        } catch (e) {
+          this.logger.error(
+            "Error signing proposer preferences",
+            {slot: duty.slot, validatorIndex: duty.validatorIndex},
+            e as Error
+          );
+        }
+      }
+    }
+
+    if (batch.length === 0) {
+      return;
+    }
+
+    try {
+      await this.api.beacon.submitSignedProposerPreferences({signedProposerPreferences: batch});
+      // Only mark as submitted after the API call succeeds; a thrown error leaves the
+      // slot eligible for retry on the next tick.
+      for (const {submission, slot: submittedSlot} of pending) {
+        submission.slots.add(submittedSlot);
+      }
+      this.logger.debug("Submitted signed proposer preferences", {count: batch.length});
+    } catch (e) {
+      this.logger.error("Error submitting signed proposer preferences", {count: batch.length}, e as Error);
+    }
+  };
+}

package/src/services/ptc.ts

@@ -0,0 +1,131 @@
+import {ApiClient, routes} from "@lodestar/api";
+import {ChainForkConfig} from "@lodestar/config";
+import {isForkPostGloas} from "@lodestar/params";
+import {Slot, gloas} from "@lodestar/types";
+import {prettyBytes, sleep, toRootHex} from "@lodestar/utils";
+import {Metrics} from "../metrics.js";
+import {PubkeyHex} from "../types.js";
+import {IClock, LoggerVc} from "../util/index.js";
+import {ChainHeaderTracker} from "./chainHeaderTracker.js";
+import {ValidatorEventEmitter} from "./emitter.js";
+import {PtcDutiesService} from "./ptcDuties.js";
+import {SyncingStatusTracker} from "./syncingStatusTracker.js";
+import {ValidatorStore} from "./validatorStore.js";
+
+/**
+ * Service that sets up and handles validator Payload Timeliness Committee duties.
+ */
+export class PtcService {
+  private readonly dutiesService: PtcDutiesService;
+
+  constructor(
+    private readonly config: ChainForkConfig,
+    private readonly logger: LoggerVc,
+    private readonly api: ApiClient,
+    private readonly clock: IClock,
+    private readonly validatorStore: ValidatorStore,
+    private readonly emitter: ValidatorEventEmitter,
+    chainHeadTracker: ChainHeaderTracker,
+    syncingStatusTracker: SyncingStatusTracker,
+    private readonly metrics: Metrics | null
+  ) {
+    this.dutiesService = new PtcDutiesService(
+      config,
+      logger,
+      api,
+      clock,
+      validatorStore,
+      chainHeadTracker,
+      syncingStatusTracker,
+      metrics
+    );
+
+    clock.runEverySlot(this.runPtcTasks);
+  }
+
+  removeDutiesForKey(pubkey: PubkeyHex): void {
+    this.dutiesService.removeDutiesForKey(pubkey);
+  }
+
+  private runPtcTasks = async (slot: Slot, signal: AbortSignal): Promise<void> => {
+    const fork = this.config.getForkName(slot);
+    if (!isForkPostGloas(fork)) {
+      return;
+    }
+
+    const duties = this.dutiesService.getDutiesAtSlot(slot);
+    if (duties.length === 0) {
+      return;
+    }
+
+    const payloadAttestationDueMs = this.config.getSlotComponentDurationMs(this.config.PAYLOAD_ATTESTATION_DUE_BPS);
+    await Promise.race([
+      sleep(payloadAttestationDueMs - this.clock.msFromSlot(slot), signal),
+      this.emitter.waitForExecutionPayloadAvailableSlot(slot),
+    ]);
+
+    this.metrics?.ptcStepCallProducePayloadAttestation.observe(
+      this.clock.secFromSlot(slot) - payloadAttestationDueMs / 1000
+    );
+
+    try {
+      const payloadAttestationData = await this.producePayloadAttestationData(slot);
+      await this.signAndPublishPayloadAttestations(slot, payloadAttestationData, duties);
+    } catch (e) {
+      this.logger.error("Error on PTC routine", {slot}, e as Error);
+    }
+  };
+
+  private async producePayloadAttestationData(slot: Slot): Promise<gloas.PayloadAttestationData> {
+    return (await this.api.validator.producePayloadAttestationData({slot})).value();
+  }
+
+  private async signAndPublishPayloadAttestations(
+    slot: Slot,
+    payloadAttestationData: gloas.PayloadAttestationData,
+    duties: routes.validator.PtcDuty[]
+  ): Promise<void> {
+    const payloadAttestationMessages: gloas.PayloadAttestationMessage[] = [];
+    const beaconBlockRootHex = toRootHex(payloadAttestationData.beaconBlockRoot);
+
+    await Promise.all(
+      duties.map(async (duty) => {
+        const logCtxValidator = {slot, validatorIndex: duty.validatorIndex, beaconBlockRoot: beaconBlockRootHex};
+        try {
+          payloadAttestationMessages.push(
+            await this.validatorStore.signPayloadAttestation(
+              duty,
+              payloadAttestationData,
+              this.clock.getCurrentSlot(),
+              this.logger
+            )
+          );
+          this.logger.debug("Signed payload attestation message", logCtxValidator);
+        } catch (e) {
+          this.metrics?.ptcError.inc({error: "sign"});
+          this.logger.error("Error signing payload attestation message", logCtxValidator, e as Error);
+        }
+      })
+    );
+
+    this.metrics?.ptcStepCallPublishPayloadAttestation.observe(
+      this.clock.secFromSlot(slot) -
+        this.config.getSlotComponentDurationMs(this.config.PAYLOAD_ATTESTATION_DUE_BPS) / 1000
+    );
+
+    if (payloadAttestationMessages.length > 0) {
+      try {
+        (await this.api.beacon.submitPayloadAttestationMessages({payloadAttestationMessages})).assertOk();
+        this.logger.info("Published payload attestation messages", {
+          slot,
+          beaconBlockRoot: prettyBytes(beaconBlockRootHex),
+          count: payloadAttestationMessages.length,
+        });
+        this.metrics?.publishedPayloadAttestations.inc(payloadAttestationMessages.length);
+      } catch (e) {
+        this.metrics?.ptcError.inc({error: "publish"});
+        this.logger.error("Error publishing payload attestation messages", {slot}, e as Error);
+      }
+    }
+  }
+}

package/src/services/ptcDuties.ts

@@ -0,0 +1,246 @@
+import {ApiClient, routes} from "@lodestar/api";
+import {ChainForkConfig} from "@lodestar/config";
+import {SLOTS_PER_EPOCH, isForkPostGloas} from "@lodestar/params";
+import {computeEpochAtSlot, isStartSlotOfEpoch} from "@lodestar/state-transition";
+import {Epoch, RootHex, Slot, ValidatorIndex} from "@lodestar/types";
+import {toPubkeyHex} from "@lodestar/utils";
+import {Metrics} from "../metrics.js";
+import {PubkeyHex} from "../types.js";
+import {IClock, LoggerVc} from "../util/index.js";
+import {ChainHeaderTracker, HeadEventData} from "./chainHeaderTracker.js";
+import {SyncingStatusTracker} from "./syncingStatusTracker.js";
+import {ValidatorStore} from "./validatorStore.js";
+
+/** Only retain `HISTORICAL_DUTIES_EPOCHS` duties prior to the current epoch. */
+const HISTORICAL_DUTIES_EPOCHS = 2;
+
+type PtcDutiesAtEpoch = {dependentRoot: RootHex; dutiesByIndex: Map<ValidatorIndex, routes.validator.PtcDuty>};
+
+export class PtcDutiesService {
+  /** Maps a validator index to its PTC duty for each epoch. */
+  private readonly dutiesByIndexByEpoch = new Map<Epoch, PtcDutiesAtEpoch>();
+
+  constructor(
+    private readonly config: ChainForkConfig,
+    private readonly logger: LoggerVc,
+    private readonly api: ApiClient,
+    private readonly clock: IClock,
+    private readonly validatorStore: ValidatorStore,
+    chainHeadTracker: ChainHeaderTracker,
+    syncingStatusTracker: SyncingStatusTracker,
+    private readonly metrics: Metrics | null
+  ) {
+    clock.runEveryEpoch(this.runDutiesTasks);
+    chainHeadTracker.runOnNewHead(this.onNewHead);
+    syncingStatusTracker.runOnResynced(async (slot) => {
+      // Skip on first slot of epoch since tasks are already scheduled.
+      if (!isStartSlotOfEpoch(slot)) {
+        return this.runDutiesTasks(computeEpochAtSlot(slot));
+      }
+    });
+
+    if (metrics) {
+      metrics.ptcDutiesCount.addCollect(() => {
+        const currentSlot = this.clock.getCurrentSlot();
+        let duties = 0;
+        let nextDutySlot = null;
+        for (const [epoch, ptcDutiesAtEpoch] of this.dutiesByIndexByEpoch) {
+          duties += ptcDutiesAtEpoch.dutiesByIndex.size;
+
+          // Epochs are sorted, stop searching once a next duty slot is found.
+          if (epoch < this.clock.currentEpoch || nextDutySlot !== null) continue;
+
+          for (const duty of ptcDutiesAtEpoch.dutiesByIndex.values()) {
+            if (duty.slot > currentSlot && (nextDutySlot === null || duty.slot < nextDutySlot)) {
+              nextDutySlot = duty.slot;
+            }
+          }
+        }
+        metrics.ptcDutiesCount.set(duties);
+        metrics.ptcDutiesEpochCount.set(this.dutiesByIndexByEpoch.size);
+        if (nextDutySlot !== null) metrics.ptcDutiesNextSlot.set(nextDutySlot);
+      });
+    }
+  }
+
+  removeDutiesForKey(pubkey: PubkeyHex): void {
+    for (const [epoch, ptcDutiesAtEpoch] of this.dutiesByIndexByEpoch) {
+      for (const [validatorIndex, duty] of ptcDutiesAtEpoch.dutiesByIndex) {
+        if (toPubkeyHex(duty.pubkey) === pubkey) {
+          ptcDutiesAtEpoch.dutiesByIndex.delete(validatorIndex);
+          if (ptcDutiesAtEpoch.dutiesByIndex.size === 0) {
+            this.dutiesByIndexByEpoch.delete(epoch);
+          }
+        }
+      }
+    }
+  }
+
+  /** Returns all PTC duties for the given slot. */
+  getDutiesAtSlot(slot: Slot): routes.validator.PtcDuty[] {
+    const epoch = computeEpochAtSlot(slot);
+    const duties: routes.validator.PtcDuty[] = [];
+    const epochDuties = this.dutiesByIndexByEpoch.get(epoch);
+    if (epochDuties === undefined) {
+      return duties;
+    }
+
+    for (const duty of epochDuties.dutiesByIndex.values()) {
+      if (duty.slot === slot) {
+        duties.push(duty);
+      }
+    }
+
+    return duties;
+  }
+
+  private runDutiesTasks = async (epoch: Epoch): Promise<void> => {
+    const nextEpoch = epoch + 1;
+    if (!isForkPostGloas(this.config.getForkName(nextEpoch * SLOTS_PER_EPOCH))) {
+      return;
+    }
+
+    await Promise.all([
+      this.pollPtcDuties(epoch, this.validatorStore.getAllLocalIndices()).catch((e: Error) => {
+        this.logger.error("Error on poll PTC duties", {epoch}, e);
+      }),
+
+      this.validatorStore
+        .pollValidatorIndices()
+        .then((newIndices) => this.pollPtcDuties(epoch, newIndices))
+        .catch((e: Error) => {
+          this.logger.error("Error on poll indices and PTC duties", {epoch}, e);
+        }),
+    ]);
+
+    this.pruneOldDuties(epoch);
+  };
+
+  private async pollPtcDuties(currentEpoch: Epoch, indexArr: ValidatorIndex[]): Promise<void> {
+    const nextEpoch = currentEpoch + 1;
+
+    if (indexArr.length === 0) {
+      return;
+    }
+
+    for (const epoch of [currentEpoch, nextEpoch]) {
+      await this.pollPtcDutiesForEpoch(epoch, indexArr).catch((e: Error) => {
+        this.logger.error("Failed to download PTC duties", {epoch}, e);
+      });
+    }
+  }
+
+  private async pollPtcDutiesForEpoch(epoch: Epoch, indexArr: ValidatorIndex[]): Promise<void> {
+    if (epoch < 0) {
+      return;
+    }
+
+    if (!isForkPostGloas(this.config.getForkName(epoch * SLOTS_PER_EPOCH))) {
+      return;
+    }
+
+    const res = await this.api.validator.getPtcDuties({epoch, indices: indexArr});
+    const ptcDuties = res.value();
+    const {dependentRoot} = res.meta();
+    const relevantDuties = ptcDuties.filter((duty) => {
+      const pubkeyHex = toPubkeyHex(duty.pubkey);
+      return this.validatorStore.hasVotingPubkey(pubkeyHex) && this.validatorStore.isDoppelgangerSafe(pubkeyHex);
+    });
+
+    this.logger.debug("Downloaded PTC duties", {epoch, dependentRoot, count: relevantDuties.length});
+
+    const dutiesAtEpoch = this.dutiesByIndexByEpoch.get(epoch);
+    const priorDependentRoot = dutiesAtEpoch?.dependentRoot;
+    const dependentRootChanged = priorDependentRoot !== undefined && priorDependentRoot !== dependentRoot;
+
+    if (!priorDependentRoot || dependentRootChanged) {
+      const dutiesByIndex = new Map<ValidatorIndex, routes.validator.PtcDuty>();
+      for (const duty of relevantDuties) {
+        dutiesByIndex.set(duty.validatorIndex, duty);
+      }
+      this.dutiesByIndexByEpoch.set(epoch, {dependentRoot, dutiesByIndex});
+
+      if (priorDependentRoot && dependentRootChanged) {
+        this.metrics?.ptcDutiesReorg.inc();
+        this.logger.warn("PTC duties re-org. This may happen from time to time", {
+          priorDependentRoot,
+          dependentRoot,
+          epoch,
+        });
+      }
+    } else {
+      const existingDuties = dutiesAtEpoch.dutiesByIndex;
+      const existingDutiesCount = existingDuties.size;
+      const discoveredNewDuties = relevantDuties.length > existingDutiesCount;
+
+      if (discoveredNewDuties) {
+        for (const duty of relevantDuties) {
+          if (!existingDuties.has(duty.validatorIndex)) {
+            existingDuties.set(duty.validatorIndex, duty);
+          }
+        }
+
+        this.logger.debug("Discovered new PTC duties", {
+          epoch,
+          dependentRoot,
+          count: relevantDuties.length - existingDutiesCount,
+        });
+      }
+    }
+  }
+
+  private onNewHead = async ({
+    slot,
+    previousDutyDependentRoot,
+    currentDutyDependentRoot,
+  }: HeadEventData): Promise<void> => {
+    const currentEpoch = computeEpochAtSlot(slot);
+    const nextEpoch = currentEpoch + 1;
+
+    const nextEpochDependentRoot = this.dutiesByIndexByEpoch.get(nextEpoch)?.dependentRoot;
+    if (nextEpochDependentRoot && currentDutyDependentRoot !== nextEpochDependentRoot) {
+      this.logger.warn("Potential next epoch PTC duties reorg", {
+        slot,
+        dutyEpoch: nextEpoch,
+        priorDependentRoot: nextEpochDependentRoot,
+        newDependentRoot: currentDutyDependentRoot,
+      });
+      await this.handlePtcDutiesReorg(nextEpoch, slot, nextEpochDependentRoot, currentDutyDependentRoot);
+    }
+
+    const currentEpochDependentRoot = this.dutiesByIndexByEpoch.get(currentEpoch)?.dependentRoot;
+    if (currentEpochDependentRoot && currentEpochDependentRoot !== previousDutyDependentRoot) {
+      this.logger.warn("Potential current epoch PTC duties reorg", {
+        slot,
+        dutyEpoch: currentEpoch,
+        priorDependentRoot: currentEpochDependentRoot,
+        newDependentRoot: previousDutyDependentRoot,
+      });
+      await this.handlePtcDutiesReorg(currentEpoch, slot, currentEpochDependentRoot, previousDutyDependentRoot);
+    }
+  };
+
+  private async handlePtcDutiesReorg(
+    dutyEpoch: Epoch,
+    slot: Slot,
+    oldDependentRoot: RootHex,
+    newDependentRoot: RootHex
+  ): Promise<void> {
+    this.metrics?.ptcDutiesReorg.inc();
+    const logContext = {dutyEpoch, slot, oldDependentRoot, newDependentRoot};
+    this.logger.debug("Redownload PTC duties", logContext);
+
+    await this.pollPtcDutiesForEpoch(dutyEpoch, this.validatorStore.getAllLocalIndices()).catch((e: Error) => {
+      this.logger.error("Failed to redownload PTC duties when reorg happens", logContext, e);
+    });
+  }
+
+  /** Run once per epoch to prune duties map. */
+  private pruneOldDuties(currentEpoch: Epoch): void {
+    for (const epoch of this.dutiesByIndexByEpoch.keys()) {
+      if (epoch + HISTORICAL_DUTIES_EPOCHS < currentEpoch) {
+        this.dutiesByIndexByEpoch.delete(epoch);
+      }
+    }
+  }
+}