@lodestar/beacon-node 1.22.0 → 1.23.0-dev.06b4c2d324

package/lib/api/impl/validator/index.js

@@ -1,24 +1,26 @@
+import { PubkeyIndexMap } from "@chainsafe/pubkey-index-map";
 import { routes } from "@lodestar/api";
-import { computeStartSlotAtEpoch, proposerShufflingDecisionRoot, attesterShufflingDecisionRoot, getBlockRootAtSlot, computeEpochAtSlot, getCurrentSlot, beaconBlockToBlinded, } from "@lodestar/state-transition";
-import { GENESIS_SLOT, SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT, SYNC_COMMITTEE_SUBNET_SIZE, isForkBlobs, isForkExecution, ForkSeq, isForkPostElectra, } from "@lodestar/params";
+import { DataAvailabilityStatus, ExecutionStatus } from "@lodestar/fork-choice";
+import { ForkSeq, GENESIS_SLOT, SLOTS_PER_EPOCH, SLOTS_PER_HISTORICAL_ROOT, SYNC_COMMITTEE_SUBNET_SIZE, isForkBlobs, isForkExecution, isForkPostElectra, } from "@lodestar/params";
+import { attesterShufflingDecisionRoot, beaconBlockToBlinded, calculateCommitteeAssignments, computeEpochAtSlot, computeStartSlotAtEpoch, createCachedBeaconState, getBlockRootAtSlot, getCurrentSlot, loadState, proposerShufflingDecisionRoot, } from "@lodestar/state-transition";
+import { ProducedBlockSource, getValidatorStatus, isBlindedBeaconBlock, isBlockContents, ssz, } from "@lodestar/types";
+import { TimeoutError, formatWeiToEth, fromHex, prettyWeiToEth, resolveOrRacePromises, toHex, toRootHex, } from "@lodestar/utils";
 import { MAX_BUILDER_BOOST_FACTOR } from "@lodestar/validator";
-import { ssz, ProducedBlockSource, isBlindedBeaconBlock, isBlockContents, } from "@lodestar/types";
-import { ExecutionStatus, DataAvailabilityStatus } from "@lodestar/fork-choice";
-import { fromHex, toHex, resolveOrRacePromises, prettyWeiToEth, toRootHex } from "@lodestar/utils";
 import { AttestationError, AttestationErrorCode, GossipAction, SyncCommitteeError, SyncCommitteeErrorCode, } from "../../../chain/errors/index.js";
+import { ChainEvent } from "../../../chain/index.js";
+import { SCHEDULER_LOOKAHEAD_FACTOR } from "../../../chain/prepareNextSlot.js";
+import { RegenCaller } from "../../../chain/regen/index.js";
 import { validateApiAggregateAndProof } from "../../../chain/validation/index.js";
+import { validateSyncCommitteeGossipContributionAndProof } from "../../../chain/validation/syncCommitteeContributionAndProof.js";
 import { ZERO_HASH } from "../../../constants/index.js";
+import { NoBidReceived } from "../../../execution/builder/http.js";
+import { validateGossipFnRetryUnknownRoot } from "../../../network/processor/gossipHandlers.js";
 import { SyncState } from "../../../sync/index.js";
 import { isOptimisticBlock } from "../../../util/forkChoice.js";
 import { getDefaultGraffiti, toGraffitiBuffer } from "../../../util/graffiti.js";
-import { ApiError, NodeIsSyncing, OnlySupportedByDVT } from "../errors.js";
-import { validateSyncCommitteeGossipContributionAndProof } from "../../../chain/validation/syncCommitteeContributionAndProof.js";
-import { RegenCaller } from "../../../chain/regen/index.js";
-import { getValidatorStatus } from "../beacon/state/utils.js";
-import { validateGossipFnRetryUnknownRoot } from "../../../network/processor/gossipHandlers.js";
-import { SCHEDULER_LOOKAHEAD_FACTOR } from "../../../chain/prepareNextSlot.js";
-import { ChainEvent } from "../../../chain/index.js";
 import { getLodestarClientVersion } from "../../../util/metadata.js";
+import { getStateResponseWithRegen } from "../beacon/state/utils.js";
+import { ApiError, NodeIsSyncing, OnlySupportedByDVT } from "../errors.js";
 import { computeSubnetForCommitteesAtSlot, getPubkeysForIndices, selectBlockProductionSource } from "./utils.js";
 /**
  * If the node is within this many epochs from the head, we declare it to be synced regardless of
@@ -35,7 +37,7 @@ import { computeSubnetForCommitteesAtSlot, getPubkeysForIndices, selectBlockProd
  */
 export const SYNC_TOLERANCE_EPOCHS = 1;
 /**
- * Cutoff time to wait for execution and builder block production apis to resolve
+ * Cutoff time to wait from start of the slot for execution and builder block production apis to resolve
  * Post this time, race execution and builder to pick whatever resolves first
  *
  * Empirically the builder block resolves in ~1.5+ seconds, and execution should resolve <1 sec.
@@ -45,6 +47,31 @@ export const SYNC_TOLERANCE_EPOCHS = 1;
 const BLOCK_PRODUCTION_RACE_CUTOFF_MS = 2_000;
 /** Overall timeout for execution and block production apis */
 const BLOCK_PRODUCTION_RACE_TIMEOUT_MS = 12_000;
+/**
+ * Engine block selection reasons tracked in metrics
+ */
+export var EngineBlockSelectionReason;
+(function (EngineBlockSelectionReason) {
+    EngineBlockSelectionReason["BuilderDisabled"] = "builder_disabled";
+    EngineBlockSelectionReason["BuilderError"] = "builder_error";
+    EngineBlockSelectionReason["BuilderTimeout"] = "builder_timeout";
+    EngineBlockSelectionReason["BuilderPending"] = "builder_pending";
+    EngineBlockSelectionReason["BuilderNoBid"] = "builder_no_bid";
+    EngineBlockSelectionReason["BuilderCensorship"] = "builder_censorship";
+    EngineBlockSelectionReason["BlockValue"] = "block_value";
+    EngineBlockSelectionReason["EnginePreferred"] = "engine_preferred";
+})(EngineBlockSelectionReason || (EngineBlockSelectionReason = {}));
+/**
+ * Builder block selection reasons tracked in metrics
+ */
+export var BuilderBlockSelectionReason;
+(function (BuilderBlockSelectionReason) {
+    BuilderBlockSelectionReason["EngineDisabled"] = "engine_disabled";
+    BuilderBlockSelectionReason["EngineError"] = "engine_error";
+    BuilderBlockSelectionReason["EnginePending"] = "engine_pending";
+    BuilderBlockSelectionReason["BlockValue"] = "block_value";
+    BuilderBlockSelectionReason["BuilderPreferred"] = "builder_preferred";
+})(BuilderBlockSelectionReason || (BuilderBlockSelectionReason = {}));
 /**
  * Server implementation for handling validator duties.
  * See `@lodestar/validator/src/api` for the client implementation).
@@ -90,7 +117,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
         if (msToSlot > MAX_API_CLOCK_DISPARITY_MS) {
             throw Error(`Requested slot ${slot} is in the future`);
         }
-        else if (msToSlot > 0) {
+        if (msToSlot > 0) {
             await chain.clock.waitForSlot(slot);
         }
         // else, clock already in slot or slot is in the past
@@ -129,20 +156,18 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                 blockTotalValue: prettyWeiToEth(totalValue),
             };
         }
-        else if (source === ProducedBlockSource.builder) {
+        if (source === ProducedBlockSource.builder) {
             return {
                 builderExecutionPayloadValue: prettyWeiToEth(executionValue),
                 builderConsensusBlockValue: prettyWeiToEth(consensusValue),
                 builderBlockTotalValue: prettyWeiToEth(totalValue),
             };
         }
-        else {
-            return {
-                engineExecutionPayloadValue: prettyWeiToEth(executionValue),
-                engineConsensusBlockValue: prettyWeiToEth(consensusValue),
-                engineBlockTotalValue: prettyWeiToEth(totalValue),
-            };
-        }
+        return {
+            engineExecutionPayloadValue: prettyWeiToEth(executionValue),
+            engineConsensusBlockValue: prettyWeiToEth(consensusValue),
+            engineBlockTotalValue: prettyWeiToEth(totalValue),
+        };
     }
     /**
      * This function is called 1s before next epoch, usually at that time PrepareNextSlotScheduler finishes
@@ -204,9 +229,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                 if (currentSlot - headSlot > SYNC_TOLERANCE_EPOCHS * SLOTS_PER_EPOCH) {
                     throw new NodeIsSyncing(`headSlot ${headSlot} currentSlot ${currentSlot}`);
                 }
-                else {
-                    return;
-                }
+                return;
             }
             case SyncState.Synced:
                 return;
@@ -269,10 +292,6 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
         if (skipHeadChecksAndUpdate !== true) {
             notWhileSyncing();
             await waitForSlot(slot); // Must never request for a future slot > currentSlot
-            // Process the queued attestations in the forkchoice for correct head estimation
-            // forkChoice.updateTime() might have already been called by the onSlot clock
-            // handler, in which case this should just return.
-            chain.forkChoice.updateTime(slot);
             parentBlockRoot = fromHex(chain.getProposerHead(slot).blockRoot);
         }
         else {
@@ -291,6 +310,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             });
             metrics?.blockProductionSuccess.inc({ source });
             metrics?.blockProductionNumAggregated.observe({ source }, block.body.attestations.length);
+            metrics?.blockProductionExecutionPayloadValue.observe({ source }, Number(formatWeiToEth(executionPayloadValue)));
             logger.verbose("Produced blinded block", {
                 slot,
                 executionPayloadValue,
@@ -314,10 +334,6 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
         if (skipHeadChecksAndUpdate !== true) {
             notWhileSyncing();
             await waitForSlot(slot); // Must never request for a future slot > currentSlot
-            // Process the queued attestations in the forkchoice for correct head estimation
-            // forkChoice.updateTime() might have already been called by the onSlot clock
-            // handler, in which case this should just return.
-            chain.forkChoice.updateTime(slot);
             parentBlockRoot = fromHex(chain.getProposerHead(slot).blockRoot);
         }
         else {
@@ -344,6 +360,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             }
             metrics?.blockProductionSuccess.inc({ source });
             metrics?.blockProductionNumAggregated.observe({ source }, block.body.attestations.length);
+            metrics?.blockProductionExecutionPayloadValue.observe({ source }, Number(formatWeiToEth(executionPayloadValue)));
             logger.verbose("Produced execution block", {
                 slot,
                 executionPayloadValue,
@@ -367,9 +384,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                     shouldOverrideBuilder,
                 };
             }
-            else {
-                return { data: block, version, executionPayloadValue, consensusBlockValue, shouldOverrideBuilder };
-            }
+            return { data: block, version, executionPayloadValue, consensusBlockValue, shouldOverrideBuilder };
         }
         finally {
             if (timer)
@@ -381,10 +396,6 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
     _skipRandaoVerification, builderBoostFactor, { feeRecipient, builderSelection, strictFeeRecipientCheck } = {}) {
         notWhileSyncing();
         await waitForSlot(slot); // Must never request for a future slot > currentSlot
-        // Process the queued attestations in the forkchoice for correct head estimation
-        // forkChoice.updateTime() might have already been called by the onSlot clock
-        // handler, in which case this should just return.
-        chain.forkChoice.updateTime(slot);
         const parentBlockRoot = fromHex(chain.getProposerHead(slot).blockRoot);
         notOnOutOfRangeData(parentBlockRoot);
         const fork = config.getForkName(slot);
@@ -429,9 +440,11 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             graffiti: toGraffitiBuffer(graffiti ?? getDefaultGraffiti(getLodestarClientVersion(opts), chain.executionEngine.clientVersion, opts)),
         });
         logger.debug("Produced common block body", loggerContext);
+        // Calculate cutoff time based on start of the slot
+        const cutoffMs = Math.max(0, BLOCK_PRODUCTION_RACE_CUTOFF_MS - Math.round(chain.clock.secFromSlot(slot) * 1000));
         logger.verbose("Block production race (builder vs execution) starting", {
             ...loggerContext,
-            cutoffMs: BLOCK_PRODUCTION_RACE_CUTOFF_MS,
+            cutoffMs,
             timeoutMs: BLOCK_PRODUCTION_RACE_TIMEOUT_MS,
         });
         // use abort controller to stop waiting for both block sources
@@ -470,7 +483,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             })
             : Promise.reject(new Error("Engine disabled"));
         const [builder, engine] = await resolveOrRacePromises([builderPromise, enginePromise], {
-            resolveTimeoutMs: BLOCK_PRODUCTION_RACE_CUTOFF_MS,
+            resolveTimeoutMs: cutoffMs,
             raceTimeoutMs: BLOCK_PRODUCTION_RACE_TIMEOUT_MS,
             signal: controller.signal,
         });
@@ -484,10 +497,18 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             }, engine.reason);
         }
         if (builder.status === "rejected" && isBuilderEnabled) {
-            logger.warn("Builder failed to produce the block", {
-                ...loggerContext,
-                durationMs: builder.durationMs,
-            }, builder.reason);
+            if (builder.reason instanceof NoBidReceived) {
+                logger.info("Builder did not provide a bid", {
+                    ...loggerContext,
+                    durationMs: builder.durationMs,
+                });
+            }
+            else {
+                logger.warn("Builder failed to produce the block", {
+                    ...loggerContext,
+                    durationMs: builder.durationMs,
+                }, builder.reason);
+            }
         }
         if (builder.status === "rejected" && engine.status === "rejected") {
             throw Error(`${isBuilderEnabled && isEngineEnabled ? "Builder and engine both" : isBuilderEnabled ? "Builder" : "Engine"} failed to produce the block`);
@@ -500,6 +521,10 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                 shouldOverrideBuilder: engine.value.shouldOverrideBuilder,
                 ...getBlockValueLogInfo(engine.value),
             });
+            metrics?.blockProductionSelectionResults.inc({
+                source: ProducedBlockSource.engine,
+                reason: EngineBlockSelectionReason.BuilderCensorship,
+            });
             return { ...engine.value, executionPayloadBlinded: false, executionPayloadSource: ProducedBlockSource.engine };
         }
         if (builder.status === "fulfilled" && engine.status !== "fulfilled") {
@@ -508,6 +533,14 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                 durationMs: builder.durationMs,
                 ...getBlockValueLogInfo(builder.value),
             });
+            metrics?.blockProductionSelectionResults.inc({
+                source: ProducedBlockSource.builder,
+                reason: isEngineEnabled === false
+                    ? BuilderBlockSelectionReason.EngineDisabled
+                    : engine.status === "pending"
+                        ? BuilderBlockSelectionReason.EnginePending
+                        : BuilderBlockSelectionReason.EngineError,
+            });
             return { ...builder.value, executionPayloadBlinded: true, executionPayloadSource: ProducedBlockSource.builder };
         }
         if (engine.status === "fulfilled" && builder.status !== "fulfilled") {
@@ -516,15 +549,29 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                 durationMs: engine.durationMs,
                 ...getBlockValueLogInfo(engine.value),
             });
+            metrics?.blockProductionSelectionResults.inc({
+                source: ProducedBlockSource.engine,
+                reason: isBuilderEnabled === false
+                    ? EngineBlockSelectionReason.BuilderDisabled
+                    : builder.status === "pending"
+                        ? EngineBlockSelectionReason.BuilderPending
+                        : builder.reason instanceof NoBidReceived
+                            ? EngineBlockSelectionReason.BuilderNoBid
+                            : builder.reason instanceof TimeoutError
+                                ? EngineBlockSelectionReason.BuilderTimeout
+                                : EngineBlockSelectionReason.BuilderError,
+            });
             return { ...engine.value, executionPayloadBlinded: false, executionPayloadSource: ProducedBlockSource.engine };
         }
         if (engine.status === "fulfilled" && builder.status === "fulfilled") {
-            const executionPayloadSource = selectBlockProductionSource({
+            const result = selectBlockProductionSource({
                 builderBlockValue: builder.value.executionPayloadValue + builder.value.consensusBlockValue,
                 engineBlockValue: engine.value.executionPayloadValue + engine.value.consensusBlockValue,
                 builderBoostFactor,
                 builderSelection,
             });
+            const executionPayloadSource = result.source;
+            metrics?.blockProductionSelectionResults.inc(result);
             logger.info(`Selected ${executionPayloadSource} block`, {
                 ...loggerContext,
                 engineDurationMs: engine.durationMs,
@@ -539,13 +586,11 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                     executionPayloadSource,
                 };
             }
-            else {
-                return {
-                    ...builder.value,
-                    executionPayloadBlinded: true,
-                    executionPayloadSource,
-                };
-            }
+            return {
+                ...builder.value,
+                executionPayloadBlinded: true,
+                executionPayloadSource,
+            };
         }
         throw Error("Unreachable error occurred during the builder and execution block production");
     }
@@ -560,27 +605,21 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                 if (meta.executionPayloadBlinded) {
                     return { data, meta };
                 }
-                else {
-                    if (isBlockContents(data)) {
-                        const { block } = data;
-                        const blindedBlock = beaconBlockToBlinded(config, block);
-                        return {
-                            data: blindedBlock,
-                            meta: { ...meta, executionPayloadBlinded: true },
-                        };
-                    }
-                    else {
-                        const blindedBlock = beaconBlockToBlinded(config, data);
-                        return {
-                            data: blindedBlock,
-                            meta: { ...meta, executionPayloadBlinded: true },
-                        };
-                    }
+                if (isBlockContents(data)) {
+                    const { block } = data;
+                    const blindedBlock = beaconBlockToBlinded(config, block);
+                    return {
+                        data: blindedBlock,
+                        meta: { ...meta, executionPayloadBlinded: true },
+                    };
                 }
+                const blindedBlock = beaconBlockToBlinded(config, data);
+                return {
+                    data: blindedBlock,
+                    meta: { ...meta, executionPayloadBlinded: true },
+                };
             }
-            else {
-                return { data, meta };
-            }
+            return { data, meta };
         },
         async produceBlindedBlock({ slot, randaoReveal, graffiti }) {
             const { data, version } = await produceEngineOrBuilderBlock(slot, randaoReveal, graffiti);
@@ -592,13 +631,11 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                 const blindedBlock = beaconBlockToBlinded(config, block);
                 return { data: blindedBlock, meta: { version } };
             }
-            else if (isBlindedBeaconBlock(data)) {
+            if (isBlindedBeaconBlock(data)) {
                 return { data, meta: { version } };
             }
-            else {
-                const blindedBlock = beaconBlockToBlinded(config, data);
-                return { data: blindedBlock, meta: { version } };
-            }
+            const blindedBlock = beaconBlockToBlinded(config, data);
+            return { data: blindedBlock, meta: { version } };
         },
         async produceAttestationData({ committeeIndex, slot }) {
             notWhileSyncing();
@@ -674,14 +711,15 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
         },
         async getProposerDuties({ epoch }) {
             notWhileSyncing();
-            // Early check that epoch is within [current_epoch, current_epoch + 1], or allow for pre-genesis
+            // Early check that epoch is no more than current_epoch + 1, or allow for pre-genesis
             const currentEpoch = currentEpochWithDisparity();
             const nextEpoch = currentEpoch + 1;
-            if (currentEpoch >= 0 && epoch !== currentEpoch && epoch !== nextEpoch) {
-                throw Error(`Requested epoch ${epoch} must equal current ${currentEpoch} or next epoch ${nextEpoch}`);
+            if (currentEpoch >= 0 && epoch > nextEpoch) {
+                throw new ApiError(400, `Requested epoch ${epoch} must not be more than one epoch in the future`);
             }
             const head = chain.forkChoice.getHead();
             let state = undefined;
+            const startSlot = computeStartSlotAtEpoch(epoch);
             const slotMs = config.SECONDS_PER_SLOT * 1000;
             const prepareNextSlotLookAheadMs = slotMs / SCHEDULER_LOOKAHEAD_FACTOR;
             const toNextEpochMs = msToNextEpoch();
@@ -699,28 +737,59 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
                 }
             }
             if (!state) {
-                state = await chain.getHeadStateAtCurrentEpoch(RegenCaller.getDuties);
+                if (epoch >= currentEpoch - 1) {
+                    // Cached beacon state stores proposers for previous, current and next epoch. The
+                    // requested epoch is within that range, we can use the head state at current epoch
+                    state = await chain.getHeadStateAtCurrentEpoch(RegenCaller.getDuties);
+                }
+                else {
+                    const res = await getStateResponseWithRegen(chain, startSlot);
+                    const stateViewDU = res.state instanceof Uint8Array
+                        ? loadState(config, chain.getHeadState(), res.state).state
+                        : res.state.clone();
+                    state = createCachedBeaconState(stateViewDU, {
+                        config: chain.config,
+                        // Not required to compute proposers
+                        pubkey2index: new PubkeyIndexMap(),
+                        index2pubkey: [],
+                    }, { skipSyncPubkeys: true, skipSyncCommitteeCache: true });
+                    if (state.epochCtx.epoch !== epoch) {
+                        throw Error(`Loaded state epoch ${state.epochCtx.epoch} does not match requested epoch ${epoch}`);
+                    }
+                }
             }
             const stateEpoch = state.epochCtx.epoch;
             let indexes = [];
-            if (epoch === stateEpoch) {
-                indexes = state.epochCtx.getBeaconProposers();
-            }
-            else if (epoch === stateEpoch + 1) {
-                // Requesting duties for next epoch is allow since they can be predicted with high probabilities.
-                // @see `epochCtx.getBeaconProposersNextEpoch` JSDocs for rationale.
-                indexes = state.epochCtx.getBeaconProposersNextEpoch();
-            }
-            else {
-                // Should never happen, epoch is checked to be in bounds above
-                throw Error(`Proposer duties for epoch ${epoch} not supported, current epoch ${stateEpoch}`);
+            switch (epoch) {
+                case stateEpoch:
+                    indexes = state.epochCtx.getBeaconProposers();
+                    break;
+                case stateEpoch + 1:
+                    // make sure shuffling is calculated and ready for next call to calculate nextProposers
+                    await chain.shufflingCache.get(state.epochCtx.nextEpoch, state.epochCtx.nextDecisionRoot);
+                    // Requesting duties for next epoch is allowed since they can be predicted with high probabilities.
+                    // @see `epochCtx.getBeaconProposersNextEpoch` JSDocs for rationale.
+                    indexes = state.epochCtx.getBeaconProposersNextEpoch();
+                    break;
+                case stateEpoch - 1: {
+                    const indexesPrevEpoch = state.epochCtx.getBeaconProposersPrevEpoch();
+                    if (indexesPrevEpoch === null) {
+                        // Should not happen as previous proposer duties should be initialized for head state
+                        // and if we load state from `Uint8Array` it will always be the state of requested epoch
+                        throw Error(`Proposer duties for previous epoch ${epoch} not yet initialized`);
+                    }
+                    indexes = indexesPrevEpoch;
+                    break;
+                }
+                default:
+                    // Should never happen, epoch is checked to be in bounds above
+                    throw Error(`Proposer duties for epoch ${epoch} not supported, current epoch ${stateEpoch}`);
             }
             // NOTE: this is the fastest way of getting compressed pubkeys.
             //       See benchmark -> packages/lodestar/test/perf/api/impl/validator/attester.test.ts
             // After dropping the flat caches attached to the CachedBeaconState it's no longer available.
             // TODO: Add a flag to just send 0x00 as pubkeys since the Lodestar validator does not need them.
             const pubkeys = getPubkeysForIndices(state.validators, indexes);
-            const startSlot = computeStartSlotAtEpoch(epoch);
             const duties = [];
             for (let i = 0; i < SLOTS_PER_EPOCH; i++) {
                 duties.push({ slot: startSlot + i, validatorIndex: indexes[i], pubkey: pubkeys[i] });
@@ -758,7 +827,12 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             // will equal `currentEpoch + 1`
             // Check that all validatorIndex belong to the state before calling getCommitteeAssignments()
             const pubkeys = getPubkeysForIndices(state.validators, indices);
-            const committeeAssignments = state.epochCtx.getCommitteeAssignments(epoch, indices);
+            const decisionRoot = state.epochCtx.getShufflingDecisionRoot(epoch);
+            const shuffling = await chain.shufflingCache.get(epoch, decisionRoot);
+            if (!shuffling) {
+                throw new ApiError(500, `No shuffling found to calculate committee assignments for epoch: ${epoch} and decisionRoot: ${decisionRoot}`);
+            }
+            const committeeAssignments = calculateCommitteeAssignments(shuffling, indices);
             const duties = [];
             for (let i = 0, len = indices.length; i < len; i++) {
                 const validatorIndex = indices[i];
@@ -829,7 +903,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
         async getAggregatedAttestation({ attestationDataRoot, slot }) {
             notWhileSyncing();
             await waitForSlot(slot); // Must never request for a future slot > currentSlot
-            const dataRootHex = toHex(attestationDataRoot);
+            const dataRootHex = toRootHex(attestationDataRoot);
             const aggregate = chain.attestationPool.getAggregate(slot, null, dataRootHex);
             const fork = chain.config.getForkName(slot);
             if (isForkPostElectra(fork)) {
@@ -868,7 +942,6 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             await Promise.all(signedAggregateAndProofs.map(async (signedAggregateAndProof, i) => {
                 try {
                     // TODO: Validate in batch
-                    // eslint-disable-next-line @typescript-eslint/explicit-function-return-type
                     const validateFn = () => validateApiAggregateAndProof(fork, chain, signedAggregateAndProof);
                     const { slot, beaconBlockRoot } = signedAggregateAndProof.message.aggregate.data;
                     // when a validator is configured with multiple beacon node urls, this attestation may come from another beacon node
@@ -898,7 +971,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             if (errors.length > 1) {
                 throw Error("Multiple errors on publishAggregateAndProofs\n" + errors.map((e) => e.message).join("\n"));
             }
-            else if (errors.length === 1) {
+            if (errors.length === 1) {
                 throw errors[0];
             }
         },
@@ -940,7 +1013,7 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             if (errors.length > 1) {
                 throw Error("Multiple errors on publishContributionAndProofs\n" + errors.map((e) => e.message).join("\n"));
             }
-            else if (errors.length === 1) {
+            if (errors.length === 1) {
                 throw errors[0];
             }
         },
@@ -1031,12 +1104,11 @@ export function getValidatorApi(opts, { chain, config, logger, metrics, network,
             const filteredRegistrations = registrations.filter((registration) => {
                 const { pubkey } = registration.message;
                 const validatorIndex = headState.epochCtx.pubkey2index.get(pubkey);
-                if (validatorIndex === undefined)
+                if (validatorIndex === null)
                     return false;
                 const validator = headState.validators.getReadonly(validatorIndex);
                 const status = getValidatorStatus(validator, currentEpoch);
-                return (status === "active" ||
-                    status === "active_exiting" ||
+                return (status === "active_exiting" ||
                     status === "active_ongoing" ||
                     status === "active_slashed" ||
                     status === "pending_initialized" ||