@lodestar/beacon-node 1.35.0-dev.f80d2d52da → 1.35.0-dev.fd1dac853d

package/src/api/impl/validator/index.ts

@@ -0,0 +1,1533 @@
+import {PubkeyIndexMap} from "@chainsafe/pubkey-index-map";
+import {routes} from "@lodestar/api";
+import {ApplicationMethods} from "@lodestar/api/server";
+import {ExecutionStatus} from "@lodestar/fork-choice";
+import {
+  ForkName,
+  ForkPostBellatrix,
+  ForkSeq,
+  GENESIS_SLOT,
+  SLOTS_PER_EPOCH,
+  SLOTS_PER_HISTORICAL_ROOT,
+  SYNC_COMMITTEE_SUBNET_SIZE,
+  isForkPostBellatrix,
+  isForkPostDeneb,
+  isForkPostElectra,
+} from "@lodestar/params";
+import {
+  CachedBeaconStateAllForks,
+  DataAvailabilityStatus,
+  attesterShufflingDecisionRoot,
+  beaconBlockToBlinded,
+  calculateCommitteeAssignments,
+  computeEpochAtSlot,
+  computeStartSlotAtEpoch,
+  computeTimeAtSlot,
+  createCachedBeaconState,
+  getBlockRootAtSlot,
+  getCurrentSlot,
+  loadState,
+  proposerShufflingDecisionRoot,
+} from "@lodestar/state-transition";
+import {
+  BLSSignature,
+  BeaconBlock,
+  BlindedBeaconBlock,
+  BlockContents,
+  Bytes32,
+  CommitteeIndex,
+  Epoch,
+  ProducedBlockSource,
+  Root,
+  Slot,
+  ValidatorIndex,
+  Wei,
+  bellatrix,
+  getValidatorStatus,
+  phase0,
+  ssz,
+} from "@lodestar/types";
+import {
+  TimeoutError,
+  defer,
+  formatWeiToEth,
+  fromHex,
+  prettyWeiToEth,
+  resolveOrRacePromises,
+  toHex,
+  toRootHex,
+} from "@lodestar/utils";
+import {MAX_BUILDER_BOOST_FACTOR} from "@lodestar/validator";
+import {BlockInputSource} from "../../../chain/blocks/blockInput/types.js";
+import {
+  AttestationError,
+  AttestationErrorCode,
+  GossipAction,
+  SyncCommitteeError,
+  SyncCommitteeErrorCode,
+} from "../../../chain/errors/index.js";
+import {ChainEvent, CheckpointHex, CommonBlockBody} from "../../../chain/index.js";
+import {PREPARE_NEXT_SLOT_BPS} from "../../../chain/prepareNextSlot.js";
+import {BlockType, ProduceFullDeneb} from "../../../chain/produceBlock/index.js";
+import {RegenCaller} from "../../../chain/regen/index.js";
+import {validateApiAggregateAndProof} from "../../../chain/validation/index.js";
+import {validateSyncCommitteeGossipContributionAndProof} from "../../../chain/validation/syncCommitteeContributionAndProof.js";
+import {ZERO_HASH} from "../../../constants/index.js";
+import {BuilderStatus, NoBidReceived} from "../../../execution/builder/http.js";
+import {validateGossipFnRetryUnknownRoot} from "../../../network/processor/gossipHandlers.js";
+import {CommitteeSubscription} from "../../../network/subnets/index.js";
+import {SyncState} from "../../../sync/index.js";
+import {callInNextEventLoop} from "../../../util/eventLoop.js";
+import {isOptimisticBlock} from "../../../util/forkChoice.js";
+import {getDefaultGraffiti, toGraffitiBytes} from "../../../util/graffiti.js";
+import {getLodestarClientVersion} from "../../../util/metadata.js";
+import {ApiOptions} from "../../options.js";
+import {getStateResponseWithRegen} from "../beacon/state/utils.js";
+import {ApiError, FailureList, IndexedError, NodeIsSyncing, OnlySupportedByDVT} from "../errors.js";
+import {ApiModules} from "../types.js";
+import {computeSubnetForCommitteesAtSlot, getPubkeysForIndices, selectBlockProductionSource} from "./utils.js";
+
+/**
+ * If the node is within this many epochs from the head, we declare it to be synced regardless of
+ * the network sync state.
+ *
+ * This helps prevent attacks where nodes can convince us that we're syncing some non-existent
+ * finalized head.
+ *
+ * TODO: Lighthouse uses 8 for the attack described above. However, 8 kills Lodestar since validators
+ * can trigger regen to fast-forward head state 8 epochs to be immediately invalidated as sync sets
+ * a new head. Then the checkpoint state cache grows unbounded with very different states (because
+ * they are 8 epochs apart) and causes an OOM. Research a proper solution once regen and the state
+ * caches are better.
+ */
+export const SYNC_TOLERANCE_EPOCHS = 1;
+
+/**
+ * Cutoff time to wait from start of the slot for execution and builder block production apis to resolve.
+ * Post this time, race execution and builder to pick whatever resolves first.
+ *
+ * Empirically the builder block resolves in ~1 second, and execution block resolves in <500 ms.
+ * A cutoff of 2 seconds gives enough time and if there are unexpected delays it ensures we publish
+ * in time as proposals post 4 seconds into the slot will likely be orphaned due to proposer boost reorg.
+ *
+ * TODO GLOAS: re-evaluate cutoff timing
+ */
+const BLOCK_PRODUCTION_RACE_CUTOFF_MS = 2_000;
+/** Overall timeout for execution and block production apis */
+const BLOCK_PRODUCTION_RACE_TIMEOUT_MS = 12_000;
+
+type ProduceBlockContentsRes = {executionPayloadValue: Wei; consensusBlockValue: Wei} & {
+  data: BlockContents;
+  version: ForkName;
+};
+type ProduceBlindedBlockRes = {executionPayloadValue: Wei; consensusBlockValue: Wei} & {
+  data: BlindedBeaconBlock;
+  version: ForkPostBellatrix;
+};
+
+type ProduceBlindedBlockOrBlockContentsRes = {executionPayloadSource: ProducedBlockSource} & (
+  | (ProduceBlockContentsRes & {executionPayloadBlinded: false})
+  | (ProduceBlindedBlockRes & {executionPayloadBlinded: true})
+);
+
+/**
+ * Engine block selection reasons tracked in metrics / logs
+ */
+export enum EngineBlockSelectionReason {
+  BuilderDisabled = "builder_disabled",
+  BuilderError = "builder_error",
+  BuilderTimeout = "builder_timeout",
+  BuilderPending = "builder_pending",
+  BuilderNoBid = "builder_no_bid",
+  BuilderCensorship = "builder_censorship",
+  BlockValue = "block_value",
+  EnginePreferred = "engine_preferred",
+}
+
+/**
+ * Builder block selection reasons tracked in metrics / logs
+ */
+export enum BuilderBlockSelectionReason {
+  EngineDisabled = "engine_disabled",
+  EngineError = "engine_error",
+  EnginePending = "engine_pending",
+  BlockValue = "block_value",
+  BuilderPreferred = "builder_preferred",
+}
+
+export type BlockSelectionResult =
+  | {
+      source: ProducedBlockSource.engine;
+      reason: EngineBlockSelectionReason;
+    }
+  | {
+      source: ProducedBlockSource.builder;
+      reason: BuilderBlockSelectionReason;
+    };
+
+/**
+ * Server implementation for handling validator duties.
+ * See `@lodestar/validator/src/api` for the client implementation).
+ */
+export function getValidatorApi(
+  opts: ApiOptions,
+  {chain, config, logger, metrics, network, sync}: ApiModules
+): ApplicationMethods<routes.validator.Endpoints> {
+  let genesisBlockRoot: Root | null = null;
+
+  /**
+   * Validator clock may be advanced from beacon's clock. If the validator requests a resource in a
+   * future slot, wait some time instead of rejecting the request because it's in the future.
+   * This value is the same to MAXIMUM_GOSSIP_CLOCK_DISPARITY_SEC.
+   * For very fast networks, reduce clock disparity to half a slot.
+   */
+  const MAX_API_CLOCK_DISPARITY_SEC = Math.min(0.5, config.SLOT_DURATION_MS / 2000);
+  const MAX_API_CLOCK_DISPARITY_MS = MAX_API_CLOCK_DISPARITY_SEC * 1000;
+
+  /** Compute and cache the genesis block root */
+  async function getGenesisBlockRoot(state: CachedBeaconStateAllForks): Promise<Root> {
+    if (!genesisBlockRoot) {
+      // Close to genesis the genesis block may not be available in the DB
+      if (state.slot < SLOTS_PER_HISTORICAL_ROOT) {
+        genesisBlockRoot = state.blockRoots.get(0);
+      }
+
+      const blockRes = await chain.getCanonicalBlockAtSlot(GENESIS_SLOT);
+      if (blockRes) {
+        genesisBlockRoot = config
+          .getForkTypes(blockRes.block.message.slot)
+          .SignedBeaconBlock.hashTreeRoot(blockRes.block);
+      }
+    }
+
+    // If for some reason the genesisBlockRoot is not able don't prevent validators from
+    // proposing or attesting. If the genesisBlockRoot is wrong, at worst it may trigger a re-fetch of the duties
+    return genesisBlockRoot || ZERO_HASH;
+  }
+
+  /**
+   * If advancing the local clock `MAX_API_CLOCK_DISPARITY_MS` ticks to the requested slot, wait for its start
+   * Prevents the validator from getting errors from the API if the clock is a bit advanced
+   */
+  async function waitForSlot(slot: Slot): Promise<void> {
+    if (slot <= 0) {
+      return;
+    }
+
+    const slotStartSec = computeTimeAtSlot(config, slot, chain.genesisTime);
+    const msToSlot = slotStartSec * 1000 - Date.now();
+
+    if (msToSlot > MAX_API_CLOCK_DISPARITY_MS) {
+      throw Error(`Requested slot ${slot} is in the future`);
+    }
+
+    if (msToSlot > 0) {
+      await chain.clock.waitForSlot(slot);
+    }
+
+    // else, clock already in slot or slot is in the past
+  }
+
+  /**
+   * If advancing the local clock `MAX_API_CLOCK_DISPARITY_MS` ticks to the next epoch, wait for slot 0 of the next epoch.
+   * Prevents a validator from not being able to get the attestater duties correctly if the beacon and validator clocks are off
+   */
+  async function waitForNextClosestEpoch(): Promise<void> {
+    const toNextEpochMs = msToNextEpoch();
+    if (toNextEpochMs > 0 && toNextEpochMs < MAX_API_CLOCK_DISPARITY_MS) {
+      const nextEpoch = chain.clock.currentEpoch + 1;
+      await chain.clock.waitForSlot(computeStartSlotAtEpoch(nextEpoch));
+    }
+  }
+
+  /**
+   * Compute ms to the next epoch.
+   */
+  function msToNextEpoch(): number {
+    const nextEpoch = chain.clock.currentEpoch + 1;
+    const secPerEpoch = (SLOTS_PER_EPOCH * config.SLOT_DURATION_MS) / 1000;
+    const nextEpochStartSec = chain.genesisTime + nextEpoch * secPerEpoch;
+    return nextEpochStartSec * 1000 - Date.now();
+  }
+
+  function currentEpochWithDisparity(): Epoch {
+    return computeEpochAtSlot(getCurrentSlot(config, chain.genesisTime - MAX_API_CLOCK_DISPARITY_SEC));
+  }
+
+  function getBlockValueLogInfo(
+    block: {executionPayloadValue: bigint; consensusBlockValue: bigint},
+    source?: ProducedBlockSource
+  ): Record<string, string> {
+    const executionValue = block.executionPayloadValue;
+    const consensusValue = block.consensusBlockValue;
+    const totalValue = executionValue + consensusValue;
+
+    if (source == null) {
+      return {
+        executionPayloadValue: prettyWeiToEth(executionValue),
+        consensusBlockValue: prettyWeiToEth(consensusValue),
+        blockTotalValue: prettyWeiToEth(totalValue),
+      };
+    }
+    if (source === ProducedBlockSource.builder) {
+      return {
+        builderExecutionPayloadValue: prettyWeiToEth(executionValue),
+        builderConsensusBlockValue: prettyWeiToEth(consensusValue),
+        builderBlockTotalValue: prettyWeiToEth(totalValue),
+      };
+    }
+    return {
+      engineExecutionPayloadValue: prettyWeiToEth(executionValue),
+      engineConsensusBlockValue: prettyWeiToEth(consensusValue),
+      engineBlockTotalValue: prettyWeiToEth(totalValue),
+    };
+  }
+
+  /**
+   * This function is called 1s before next epoch, usually at that time PrepareNextSlotScheduler finishes
+   * so we should have checkpoint state, otherwise wait for up to the slot 1 of epoch.
+   *      slot epoch        0            1
+   *           |------------|------------|
+   *                    ^  ^
+   *                    |  |
+   *                    |  |
+   *                    | waitForCheckpointState (1s before slot 0 of epoch, wait until slot 1 of epoch)
+   *                    |
+   *              prepareNextSlot (4s before next slot)
+   */
+  async function waitForCheckpointState(cpHex: CheckpointHex): Promise<CachedBeaconStateAllForks | null> {
+    const cpState = chain.regen.getCheckpointStateSync(cpHex);
+    if (cpState) {
+      return cpState;
+    }
+    const cp = {
+      epoch: cpHex.epoch,
+      root: fromHex(cpHex.rootHex),
+    };
+    const slot0 = computeStartSlotAtEpoch(cp.epoch);
+    // if not, wait for ChainEvent.checkpoint event until slot 1 of epoch
+    let listener: ((eventCp: phase0.Checkpoint) => void) | null = null;
+    const foundCPState = await Promise.race([
+      new Promise((resolve) => {
+        listener = (eventCp) => {
+          resolve(ssz.phase0.Checkpoint.equals(eventCp, cp));
+        };
+        chain.emitter.once(ChainEvent.checkpoint, listener);
+      }),
+      // in rare case, checkpoint state cache may happen up to 6s of slot 0 of epoch
+      // so we wait for it until the slot 1 of epoch
+      chain.clock.waitForSlot(slot0 + 1),
+    ]);
+
+    if (listener != null) {
+      chain.emitter.off(ChainEvent.checkpoint, listener);
+    }
+
+    if (foundCPState === true) {
+      return chain.regen.getCheckpointStateSync(cpHex);
+    }
+
+    return null;
+  }
+
+  /**
+   * Reject any request while the node is syncing
+   */
+  function notWhileSyncing(): void {
+    // Consider node synced before or close to genesis
+    if (chain.clock.currentSlot < SLOTS_PER_EPOCH) {
+      return;
+    }
+
+    const syncState = sync.state;
+    switch (syncState) {
+      case SyncState.SyncingFinalized:
+      case SyncState.SyncingHead: {
+        const currentSlot = chain.clock.currentSlot;
+        const headSlot = chain.forkChoice.getHead().slot;
+        if (currentSlot - headSlot > SYNC_TOLERANCE_EPOCHS * SLOTS_PER_EPOCH) {
+          throw new NodeIsSyncing(`headSlot ${headSlot} currentSlot ${currentSlot}`);
+        }
+
+        return;
+      }
+
+      case SyncState.Synced:
+        return;
+
+      case SyncState.Stalled:
+        throw new NodeIsSyncing("waiting for peers");
+    }
+  }
+
+  /**
+   * Post merge, the CL and EL could be out of step in the sync, and could result in
+   * Syncing status of the chain head. To be precise:
+   * 1. CL could be ahead of the EL, with the validity of head payload not yet verified
+   * 2. CL could be on an invalid chain of execution blocks with a non-existent
+   *    or non-available parent that never syncs up
+   *
+   * Both the above scenarios could be problematic and hence validator shouldn't participate
+   * or weigh its vote on a head till it resolves to a Valid execution status.
+   * Following activities should be skipped on an Optimistic head (with Syncing status):
+   * 1. Attestation if targetRoot is optimistic
+   * 2. SyncCommitteeContribution if if the root for which to produce contribution is Optimistic.
+   * 3. ProduceBlock if the parentRoot (chain's current head is optimistic). However this doesn't
+   *    need to be checked/aborted here as assembleBody would call EL's api for the latest
+   *    executionStatus of the parentRoot. If still not validated, produceBlock will throw error.
+   *
+   * TODO/PENDING: SyncCommitteeSignatures should also be aborted, the best way to address this
+   *   is still in flux and will be updated as and when other CL's figure this out.
+   */
+
+  function notOnOptimisticBlockRoot(beaconBlockRoot: Root): void {
+    const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot);
+    if (!protoBeaconBlock) {
+      throw new ApiError(404, `Block not in forkChoice, beaconBlockRoot=${toRootHex(beaconBlockRoot)}`);
+    }
+
+    if (protoBeaconBlock.executionStatus === ExecutionStatus.Syncing)
+      throw new NodeIsSyncing(
+        `Block's execution payload not yet validated, executionPayloadBlockHash=${protoBeaconBlock.executionPayloadBlockHash} number=${protoBeaconBlock.executionPayloadNumber}`
+      );
+  }
+
+  function notOnOutOfRangeData(beaconBlockRoot: Root): void {
+    const protoBeaconBlock = chain.forkChoice.getBlock(beaconBlockRoot);
+    if (!protoBeaconBlock) {
+      throw new ApiError(404, `Block not in forkChoice, beaconBlockRoot=${toRootHex(beaconBlockRoot)}`);
+    }
+
+    if (protoBeaconBlock.dataAvailabilityStatus === DataAvailabilityStatus.OutOfRange)
+      throw new NodeIsSyncing("Block's data is out of range and not validated");
+  }
+
+  async function produceBuilderBlindedBlock(
+    slot: Slot,
+    randaoReveal: BLSSignature,
+    graffiti: Bytes32,
+    // as of now fee recipient checks can not be performed because builder does not return bid recipient
+    {
+      commonBlockBodyPromise,
+      parentBlockRoot,
+      parentSlot,
+    }: Omit<routes.validator.ExtraProduceBlockOpts, "builderSelection"> & {
+      commonBlockBodyPromise: Promise<CommonBlockBody>;
+      parentBlockRoot: Root;
+      parentSlot: Slot;
+    }
+  ): Promise<ProduceBlindedBlockRes> {
+    const version = config.getForkName(slot);
+    if (!isForkPostBellatrix(version)) {
+      throw Error(`Invalid fork=${version} for produceBuilderBlindedBlock`);
+    }
+
+    const source = ProducedBlockSource.builder;
+    metrics?.blockProductionRequests.inc({source});
+
+    // Error early for builder if builder flow not active
+    if (!chain.executionBuilder) {
+      throw Error("External builder not configured");
+    }
+
+    switch (chain.executionBuilder.status) {
+      case BuilderStatus.disabled:
+        throw Error("External builder disabled due to failed status check");
+      case BuilderStatus.circuitBreaker:
+        throw Error("External builder circuit breaker is activated");
+      case BuilderStatus.enabled:
+        // continue
+        break;
+    }
+
+    let timer: undefined | ((opts: {source: ProducedBlockSource}) => number);
+    try {
+      timer = metrics?.blockProductionTime.startTimer();
+      const {block, executionPayloadValue, consensusBlockValue} = await chain.produceBlindedBlock({
+        slot,
+        parentBlockRoot,
+        parentSlot,
+        randaoReveal,
+        graffiti,
+        commonBlockBodyPromise,
+      });
+
+      metrics?.blockProductionSuccess.inc({source});
+      metrics?.blockProductionNumAggregated.observe({source}, block.body.attestations.length);
+      metrics?.blockProductionConsensusBlockValue.observe({source}, Number(formatWeiToEth(consensusBlockValue)));
+      metrics?.blockProductionExecutionPayloadValue.observe({source}, Number(formatWeiToEth(executionPayloadValue)));
+      logger.verbose("Produced blinded block", {
+        slot,
+        executionPayloadValue,
+        consensusBlockValue,
+        root: toRootHex(config.getPostBellatrixForkTypes(slot).BlindedBeaconBlock.hashTreeRoot(block)),
+      });
+
+      if (chain.opts.persistProducedBlocks) {
+        void chain.persistBlock(block, "produced_builder_block");
+      }
+
+      return {data: block, version, executionPayloadValue, consensusBlockValue};
+    } finally {
+      if (timer) timer({source});
+    }
+  }
+
+  async function produceEngineBlockContents(
+    slot: Slot,
+    randaoReveal: BLSSignature,
+    graffiti: Bytes32,
+    {
+      feeRecipient,
+      strictFeeRecipientCheck,
+      commonBlockBodyPromise,
+      parentBlockRoot,
+      parentSlot,
+    }: Omit<routes.validator.ExtraProduceBlockOpts, "builderSelection"> & {
+      commonBlockBodyPromise: Promise<CommonBlockBody>;
+      parentBlockRoot: Root;
+      parentSlot: Slot;
+    }
+  ): Promise<ProduceBlockContentsRes & {shouldOverrideBuilder?: boolean}> {
+    const source = ProducedBlockSource.engine;
+    metrics?.blockProductionRequests.inc({source});
+
+    let timer: undefined | ((opts: {source: ProducedBlockSource}) => number);
+    try {
+      timer = metrics?.blockProductionTime.startTimer();
+      const {block, executionPayloadValue, consensusBlockValue, shouldOverrideBuilder} = await chain.produceBlock({
+        slot,
+        parentBlockRoot,
+        parentSlot,
+        randaoReveal,
+        graffiti,
+        feeRecipient,
+        commonBlockBodyPromise,
+      });
+      const version = config.getForkName(block.slot);
+      if (strictFeeRecipientCheck && feeRecipient && isForkPostBellatrix(version)) {
+        const blockFeeRecipient = toHex((block as bellatrix.BeaconBlock).body.executionPayload.feeRecipient);
+        if (blockFeeRecipient !== feeRecipient) {
+          throw Error(`Invalid feeRecipient set in engine block expected=${feeRecipient} actual=${blockFeeRecipient}`);
+        }
+      }
+
+      metrics?.blockProductionSuccess.inc({source});
+      metrics?.blockProductionNumAggregated.observe({source}, block.body.attestations.length);
+      metrics?.blockProductionConsensusBlockValue.observe({source}, Number(formatWeiToEth(consensusBlockValue)));
+      metrics?.blockProductionExecutionPayloadValue.observe({source}, Number(formatWeiToEth(executionPayloadValue)));
+
+      const blockRoot = toRootHex(config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block));
+      logger.verbose("Produced execution block", {
+        slot,
+        executionPayloadValue,
+        consensusBlockValue,
+        root: blockRoot,
+      });
+      if (chain.opts.persistProducedBlocks) {
+        void chain.persistBlock(block, "produced_engine_block");
+      }
+      if (isForkPostDeneb(version)) {
+        const produceResult = chain.blockProductionCache.get(blockRoot);
+        if (produceResult === undefined) {
+          throw Error("production result missing in cache");
+        }
+        if (!isForkPostDeneb(produceResult.fork)) {
+          throw Error("production result is for pre-deneb fork");
+        }
+        if (produceResult.type !== BlockType.Full) {
+          throw Error("production result is not full block");
+        }
+        const blobsBundle = (produceResult as ProduceFullDeneb).blobsBundle;
+
+        return {
+          data: {
+            block,
+            blobs: blobsBundle.blobs,
+            kzgProofs: blobsBundle.proofs,
+          },
+          version,
+          executionPayloadValue,
+          consensusBlockValue,
+          shouldOverrideBuilder,
+        };
+      }
+
+      return {data: {block}, version, executionPayloadValue, consensusBlockValue, shouldOverrideBuilder};
+    } finally {
+      if (timer) timer({source});
+    }
+  }
+
+  async function produceEngineOrBuilderBlock(
+    slot: Slot,
+    randaoReveal: BLSSignature,
+    graffiti?: string,
+    // TODO deneb: skip randao verification
+    _skipRandaoVerification?: boolean,
+    builderBoostFactor?: bigint,
+    {feeRecipient, builderSelection, strictFeeRecipientCheck}: routes.validator.ExtraProduceBlockOpts = {}
+  ): Promise<ProduceBlindedBlockOrBlockContentsRes> {
+    notWhileSyncing();
+    await waitForSlot(slot); // Must never request for a future slot > currentSlot
+
+    const {blockRoot: parentBlockRootHex, slot: parentSlot} = chain.getProposerHead(slot);
+    const parentBlockRoot = fromHex(parentBlockRootHex);
+    notOnOutOfRangeData(parentBlockRoot);
+    metrics?.blockProductionSlotDelta.set(slot - parentSlot);
+
+    const fork = config.getForkName(slot);
+    // set some sensible opts
+    // builderSelection will be deprecated and will run in mode MaxProfit if builder is enabled
+    // and the actual selection will be determined using builderBoostFactor passed by the validator
+    builderSelection = builderSelection ?? routes.validator.BuilderSelection.MaxProfit;
+    builderBoostFactor = builderBoostFactor ?? BigInt(100);
+    if (builderBoostFactor > MAX_BUILDER_BOOST_FACTOR) {
+      throw new ApiError(400, `Invalid builderBoostFactor=${builderBoostFactor} > MAX_BUILDER_BOOST_FACTOR`);
+    }
+
+    const isBuilderEnabled =
+      ForkSeq[fork] >= ForkSeq.bellatrix &&
+      chain.executionBuilder !== undefined &&
+      builderSelection !== routes.validator.BuilderSelection.ExecutionOnly;
+
+    // At any point either the builder or execution or both flows should be active.
+    //
+    // Ideally such a scenario should be prevented on startup, but proposerSettingsFile or keymanager
+    // configurations could cause a validator pubkey to have builder disabled with builder selection builder only
+    // (TODO: independently make sure such an options update is not successful for a validator pubkey)
+    //
+    // So if builder is disabled ignore builder selection of builder only if caused by user mistake
+    // https://github.com/ChainSafe/lodestar/issues/6338
+    const isEngineEnabled = !isBuilderEnabled || builderSelection !== routes.validator.BuilderSelection.BuilderOnly;
+
+    if (!isEngineEnabled && !isBuilderEnabled) {
+      throw Error(
+        `Internal Error: Neither builder nor execution proposal flow activated isBuilderEnabled=${isBuilderEnabled} builderSelection=${builderSelection}`
+      );
+    }
+
+    const graffitiBytes = toGraffitiBytes(
+      graffiti ?? getDefaultGraffiti(getLodestarClientVersion(opts), chain.executionEngine.clientVersion, opts)
+    );
+
+    const loggerContext = {
+      slot,
+      parentSlot,
+      parentBlockRoot: parentBlockRootHex,
+      fork,
+      builderSelection,
+      isBuilderEnabled,
+      isEngineEnabled,
+      strictFeeRecipientCheck,
+      // winston logger doesn't like bigint
+      builderBoostFactor: `${builderBoostFactor}`,
+    };
+
+    logger.verbose("Assembling block with produceEngineOrBuilderBlock", loggerContext);
+
+    // Defer common block body production to make sure we sent async builder and engine requests before
+    const deferredCommonBlockBody = defer<CommonBlockBody>();
+    const commonBlockBodyPromise = deferredCommonBlockBody.promise;
+
+    // use abort controller to stop waiting for both block sources
+    const controller = new AbortController();
+
+    // Start calls for building execution and builder blocks
+    const builderPromise = isBuilderEnabled
+      ? produceBuilderBlindedBlock(slot, randaoReveal, graffitiBytes, {
+          feeRecipient,
+          // can't do fee recipient checks as builder bid doesn't return feeRecipient as of now
+          strictFeeRecipientCheck: false,
+          commonBlockBodyPromise,
+          parentBlockRoot,
+          parentSlot,
+        })
+      : Promise.reject(new Error("Builder disabled"));
+
+    const enginePromise = isEngineEnabled
+      ? produceEngineBlockContents(slot, randaoReveal, graffitiBytes, {
+          feeRecipient,
+          strictFeeRecipientCheck,
+          commonBlockBodyPromise,
+          parentBlockRoot,
+          parentSlot,
+        }).then((engineBlock) => {
+          // Once the engine returns a block, in the event of either:
+          // - suspected builder censorship
+          // - builder boost factor set to 0 or builder selection `executionalways`
+          // we don't need to wait for builder block as engine block will always be selected
+          if (
+            engineBlock.shouldOverrideBuilder ||
+            builderBoostFactor === BigInt(0) ||
+            builderSelection === routes.validator.BuilderSelection.ExecutionAlways
+          ) {
+            controller.abort();
+          }
+          return engineBlock;
+        })
+      : Promise.reject(new Error("Engine disabled"));
+
+    // Calculate cutoff time based on start of the slot
+    const cutoffMs = Math.max(0, BLOCK_PRODUCTION_RACE_CUTOFF_MS - chain.clock.msFromSlot(slot));
+
+    logger.verbose("Block production race (builder vs execution) starting", {
+      ...loggerContext,
+      cutoffMs,
+      timeoutMs: BLOCK_PRODUCTION_RACE_TIMEOUT_MS,
+    });
+
+    const blockProductionRacePromise = resolveOrRacePromises([builderPromise, enginePromise], {
+      resolveTimeoutMs: cutoffMs,
+      raceTimeoutMs: BLOCK_PRODUCTION_RACE_TIMEOUT_MS,
+      signal: controller.signal,
+    });
+
+    // Ensure builder and engine HTTP requests are sent before starting common block body production
+    // by deferring the call to next event loop iteration, allowing pending I/O operations like
+    // HTTP requests to be processed first and sent out early in slot.
+    callInNextEventLoop(() => {
+      logger.verbose("Producing common block body", loggerContext);
+      const commonBlockBodyStartedAt = Date.now();
+
+      chain
+        .produceCommonBlockBody({
+          slot,
+          parentBlockRoot,
+          parentSlot,
+          randaoReveal,
+          graffiti: graffitiBytes,
+        })
+        .then((commonBlockBody) => {
+          deferredCommonBlockBody.resolve(commonBlockBody);
+          logger.verbose("Produced common block body", {
+            ...loggerContext,
+            durationMs: Date.now() - commonBlockBodyStartedAt,
+          });
+        })
+        .catch(deferredCommonBlockBody.reject);
+    });
+
+    const [builder, engine] = await blockProductionRacePromise;
+
+    if (builder.status === "pending" && engine.status === "pending") {
+      throw Error("Builder and engine both failed to produce the block within timeout");
+    }
+
+    if (builder.status === "pending" && !isEngineEnabled) {
+      throw Error("Builder failed to produce the block within timeout");
+    }
+
+    if (engine.status === "pending" && !isBuilderEnabled) {
+      throw Error("Engine failed to produce the block within timeout");
+    }
+
+    if (isEngineEnabled) {
+      if (engine.status === "rejected") {
+        logger.warn(
+          "Engine failed to produce the block",
+          {
+            ...loggerContext,
+            durationMs: engine.durationMs,
+          },
+          engine.reason
+        );
+      } else if (engine.status === "pending") {
+        logger.warn("Engine failed to produce the block within cutoff time", {
+          ...loggerContext,
+          cutoffMs,
+        });
+      }
+    }
+
+    if (isBuilderEnabled) {
+      if (builder.status === "rejected") {
+        if (builder.reason instanceof NoBidReceived) {
+          logger.info("Builder did not provide a bid", {
+            ...loggerContext,
+            durationMs: builder.durationMs,
+          });
+        } else {
+          logger.warn(
+            "Builder failed to produce the block",
+            {
+              ...loggerContext,
+              durationMs: builder.durationMs,
+            },
+            builder.reason
+          );
+        }
+      } else if (builder.status === "pending") {
+        logger.warn("Builder failed to produce the block within cutoff time", {
+          ...loggerContext,
+          cutoffMs,
+        });
+      }
+    }
+
+    if (builder.status === "rejected" && engine.status === "rejected") {
+      throw Error(
+        `${isBuilderEnabled && isEngineEnabled ? "Builder and engine both" : isBuilderEnabled ? "Builder" : "Engine"} failed to produce the block`
+      );
+    }
+
+    // handle shouldOverrideBuilder separately
+    if (engine.status === "fulfilled" && engine.value.shouldOverrideBuilder) {
+      logger.info("Selected engine block: censorship suspected in builder blocks", {
+        ...loggerContext,
+        durationMs: engine.durationMs,
+        shouldOverrideBuilder: engine.value.shouldOverrideBuilder,
+        ...getBlockValueLogInfo(engine.value),
+      });
+
+      metrics?.blockProductionSelectionResults.inc({
+        source: ProducedBlockSource.engine,
+        reason: EngineBlockSelectionReason.BuilderCensorship,
+      });
+
+      return {...engine.value, executionPayloadBlinded: false, executionPayloadSource: ProducedBlockSource.engine};
+    }
+
+    if (builder.status === "fulfilled" && engine.status !== "fulfilled") {
+      const reason =
+        isEngineEnabled === false
+          ? BuilderBlockSelectionReason.EngineDisabled
+          : engine.status === "pending"
+            ? BuilderBlockSelectionReason.EnginePending
+            : BuilderBlockSelectionReason.EngineError;
+
+      logger.info("Selected builder block: no engine block produced", {
+        reason,
+        ...loggerContext,
+        durationMs: builder.durationMs,
+        ...getBlockValueLogInfo(builder.value),
+      });
+
+      metrics?.blockProductionSelectionResults.inc({
+        source: ProducedBlockSource.builder,
+        reason,
+      });
+
+      return {...builder.value, executionPayloadBlinded: true, executionPayloadSource: ProducedBlockSource.builder};
+    }
+
+    if (engine.status === "fulfilled" && builder.status !== "fulfilled") {
+      const reason =
+        isBuilderEnabled === false
+          ? EngineBlockSelectionReason.BuilderDisabled
+          : builder.status === "pending"
+            ? EngineBlockSelectionReason.BuilderPending
+            : builder.reason instanceof NoBidReceived
+              ? EngineBlockSelectionReason.BuilderNoBid
+              : builder.reason instanceof TimeoutError
+                ? EngineBlockSelectionReason.BuilderTimeout
+                : EngineBlockSelectionReason.BuilderError;
+
+      logger.info("Selected engine block: no builder block produced", {
+        reason,
+        ...loggerContext,
+        durationMs: engine.durationMs,
+        ...getBlockValueLogInfo(engine.value),
+      });
+
+      metrics?.blockProductionSelectionResults.inc({
+        source: ProducedBlockSource.engine,
+        reason,
+      });
+
+      return {...engine.value, executionPayloadBlinded: false, executionPayloadSource: ProducedBlockSource.engine};
+    }
+
+    if (engine.status === "fulfilled" && builder.status === "fulfilled") {
+      const result = selectBlockProductionSource({
+        builderExecutionPayloadValue: builder.value.executionPayloadValue,
+        engineExecutionPayloadValue: engine.value.executionPayloadValue,
+        builderBoostFactor,
+        builderSelection,
+      });
+      const executionPayloadSource = result.source;
+
+      metrics?.blockProductionSelectionResults.inc(result);
+
+      logger.info(`Selected ${executionPayloadSource} block`, {
+        reason: result.reason,
+        ...loggerContext,
+        engineDurationMs: engine.durationMs,
+        ...getBlockValueLogInfo(engine.value, ProducedBlockSource.engine),
+        builderDurationMs: builder.durationMs,
+        ...getBlockValueLogInfo(builder.value, ProducedBlockSource.builder),
+      });
+
+      if (executionPayloadSource === ProducedBlockSource.engine) {
+        return {
+          ...engine.value,
+          executionPayloadBlinded: false,
+          executionPayloadSource,
+        };
+      }
+
+      return {
+        ...builder.value,
+        executionPayloadBlinded: true,
+        executionPayloadSource,
+      };
+    }
+
+    throw Error("Unreachable error occurred during the builder and execution block production");
+  }
+
+  return {
+    async produceBlockV3({slot, randaoReveal, graffiti, skipRandaoVerification, builderBoostFactor, ...opts}) {
+      const {data, ...meta} = await produceEngineOrBuilderBlock(
+        slot,
+        randaoReveal,
+        graffiti,
+        skipRandaoVerification,
+        builderBoostFactor,
+        opts
+      );
+
+      if (opts.blindedLocal === true && ForkSeq[meta.version] >= ForkSeq.bellatrix) {
+        if (meta.executionPayloadBlinded) {
+          return {data, meta};
+        }
+
+        const {block} = data as BlockContents;
+        const blindedBlock = beaconBlockToBlinded(config, block as BeaconBlock<ForkPostBellatrix>);
+        return {
+          data: blindedBlock,
+          meta: {...meta, executionPayloadBlinded: true},
+        };
+      }
+
+      return {data, meta};
+    },
+
+    async produceAttestationData({committeeIndex, slot}) {
+      notWhileSyncing();
+
+      await waitForSlot(slot); // Must never request for a future slot > currentSlot
+
+      // This needs a state in the same epoch as `slot` such that state.currentJustifiedCheckpoint is correct.
+      // Note: This may trigger an epoch transition if there skipped slots at the beginning of the epoch.
+      const headState = chain.getHeadState();
+      const headSlot = headState.slot;
+      const attEpoch = computeEpochAtSlot(slot);
+      const headBlockRootHex = chain.forkChoice.getHead().blockRoot;
+      const headBlockRoot = fromHex(headBlockRootHex);
+      const fork = config.getForkName(slot);
+
+      let index: CommitteeIndex;
+      if (isForkPostElectra(fork)) {
+        index = 0;
+      } else {
+        if (committeeIndex === undefined) {
+          throw new ApiError(400, `Committee index must be provided for pre-electra fork=${fork}`);
+        }
+        index = committeeIndex;
+      }
+
+      const beaconBlockRoot =
+        slot >= headSlot
+          ? // When attesting to the head slot or later, always use the head of the chain.
+            headBlockRoot
+          : // Permit attesting to slots *prior* to the current head. This is desirable when
+            // the VC and BN are out-of-sync due to time issues or overloading.
+            getBlockRootAtSlot(headState, slot);
+
+      const targetSlot = computeStartSlotAtEpoch(attEpoch);
+      const targetRoot =
+        targetSlot >= headSlot
+          ? // If the state is earlier than the target slot then the target *must* be the head block root.
+            headBlockRoot
+          : getBlockRootAtSlot(headState, targetSlot);
+
+      // Check the execution status as validator shouldn't vote on an optimistic head
+      // Check on target is sufficient as a valid target would imply a valid source
+      notOnOptimisticBlockRoot(targetRoot);
+      notOnOutOfRangeData(targetRoot);
+
+      // To get the correct source we must get a state in the same epoch as the attestation's epoch.
+      // An epoch transition may change state.currentJustifiedCheckpoint
+      const attEpochState = await chain.getHeadStateAtEpoch(attEpoch, RegenCaller.produceAttestationData);
+
+      // TODO confirm if the below is correct assertion
+      // notOnOutOfRangeData(attEpochState.currentJustifiedCheckpoint.root);
+
+      return {
+        data: {
+          slot,
+          index,
+          beaconBlockRoot,
+          source: attEpochState.currentJustifiedCheckpoint,
+          target: {epoch: attEpoch, root: targetRoot},
+        },
+      };
+    },
+
+    /**
+     * GET `/eth/v1/validator/sync_committee_contribution`
+     *
+     * Requests that the beacon node produce a sync committee contribution.
+     *
+     * https://github.com/ethereum/beacon-APIs/pull/138
+     *
+     * @param slot The slot for which a sync committee contribution should be created.
+     * @param subcommitteeIndex The subcommittee index for which to produce the contribution.
+     * @param beaconBlockRoot The block root for which to produce the contribution.
+     */
+    async produceSyncCommitteeContribution({slot, subcommitteeIndex, beaconBlockRoot}) {
+      // when a validator is configured with multiple beacon node urls, this beaconBlockRoot may come from another beacon node
+      // and it hasn't been in our forkchoice since we haven't seen / processing that block
+      // see https://github.com/ChainSafe/lodestar/issues/5063
+      if (!chain.forkChoice.hasBlock(beaconBlockRoot)) {
+        const rootHex = toRootHex(beaconBlockRoot);
+        network.searchUnknownSlotRoot({slot, root: rootHex}, BlockInputSource.api);
+        // if result of this call is false, i.e. block hasn't seen after 1 slot then the below notOnOptimisticBlockRoot call will throw error
+        await chain.waitForBlock(slot, rootHex);
+      }
+
+      // Check the execution status as validator shouldn't contribute on an optimistic head
+      notOnOptimisticBlockRoot(beaconBlockRoot);
+      notOnOutOfRangeData(beaconBlockRoot);
+
+      const contribution = chain.syncCommitteeMessagePool.getContribution(subcommitteeIndex, slot, beaconBlockRoot);
+      if (!contribution) {
+        throw new ApiError(
+          404,
+          `No sync committee contribution for slot=${slot}, subnet=${subcommitteeIndex}, beaconBlockRoot=${toRootHex(beaconBlockRoot)}`
+        );
+      }
+
+      metrics?.production.producedSyncContributionParticipants.observe(
+        contribution.aggregationBits.getTrueBitIndexes().length
+      );
+
+      return {data: contribution};
+    },
+
+    async getProposerDuties({epoch}) {
+      notWhileSyncing();
+
+      // Early check that epoch is no more than current_epoch + 1, or allow for pre-genesis
+      const currentEpoch = currentEpochWithDisparity();
+      const nextEpoch = currentEpoch + 1;
+      if (currentEpoch >= 0 && epoch > nextEpoch) {
+        throw new ApiError(400, `Requested epoch ${epoch} must not be more than one epoch in the future`);
+      }
+
+      const head = chain.forkChoice.getHead();
+      let state: CachedBeaconStateAllForks | undefined = undefined;
+      const startSlot = computeStartSlotAtEpoch(epoch);
+      const prepareNextSlotLookAheadMs =
+        config.SLOT_DURATION_MS - config.getSlotComponentDurationMs(PREPARE_NEXT_SLOT_BPS);
+      const toNextEpochMs = msToNextEpoch();
+      // validators may request next epoch's duties when it's close to next epoch
+      // this is to avoid missed block proposal due to 0 epoch look ahead
+      if (epoch === nextEpoch && toNextEpochMs < prepareNextSlotLookAheadMs) {
+        // wait for maximum 1 slot for cp state which is the timeout of validator api
+        const cpState = await waitForCheckpointState({rootHex: head.blockRoot, epoch});
+        if (cpState) {
+          state = cpState;
+          metrics?.duties.requestNextEpochProposalDutiesHit.inc();
+        } else {
+          metrics?.duties.requestNextEpochProposalDutiesMiss.inc();
+        }
+      }
+
+      if (!state) {
+        if (epoch >= currentEpoch - 1) {
+          // Cached beacon state stores proposers for previous, current and next epoch. The
+          // requested epoch is within that range, we can use the head state at current epoch
+          state = await chain.getHeadStateAtCurrentEpoch(RegenCaller.getDuties);
+        } else {
+          const res = await getStateResponseWithRegen(chain, startSlot);
+
+          const stateViewDU =
+            res.state instanceof Uint8Array
+              ? loadState(config, chain.getHeadState(), res.state).state
+              : res.state.clone();
+
+          state = createCachedBeaconState(
+            stateViewDU,
+            {
+              config: chain.config,
+              // Not required to compute proposers
+              pubkey2index: new PubkeyIndexMap(),
+              index2pubkey: [],
+            },
+            {skipSyncPubkeys: true, skipSyncCommitteeCache: true}
+          );
+
+          if (state.epochCtx.epoch !== epoch) {
+            throw Error(`Loaded state epoch ${state.epochCtx.epoch} does not match requested epoch ${epoch}`);
+          }
+        }
+      }
+
+      const stateEpoch = state.epochCtx.epoch;
+      let indexes: ValidatorIndex[] = [];
+
+      switch (epoch) {
+        case stateEpoch:
+          indexes = state.epochCtx.getBeaconProposers();
+          break;
+
+        case stateEpoch + 1:
+          // make sure shuffling is calculated and ready for next call to calculate nextProposers
+          await chain.shufflingCache.get(state.epochCtx.nextEpoch, state.epochCtx.nextDecisionRoot);
+          // Requesting duties for next epoch is allowed since they can be predicted with high probabilities.
+          // @see `epochCtx.getBeaconProposersNextEpoch` JSDocs for rationale.
+          indexes = state.epochCtx.getBeaconProposersNextEpoch();
+          break;
+
+        case stateEpoch - 1: {
+          const indexesPrevEpoch = state.epochCtx.getBeaconProposersPrevEpoch();
+          if (indexesPrevEpoch === null) {
+            // Should not happen as previous proposer duties should be initialized for head state
+            // and if we load state from `Uint8Array` it will always be the state of requested epoch
+            throw Error(`Proposer duties for previous epoch ${epoch} not yet initialized`);
+          }
+          indexes = indexesPrevEpoch;
+          break;
+        }
+
+        default:
+          // Should never happen, epoch is checked to be in bounds above
+          throw Error(`Proposer duties for epoch ${epoch} not supported, current epoch ${stateEpoch}`);
+      }
+
+      // NOTE: this is the fastest way of getting compressed pubkeys.
+      //       See benchmark -> packages/lodestar/test/perf/api/impl/validator/attester.test.ts
+      // After dropping the flat caches attached to the CachedBeaconState it's no longer available.
+      // TODO: Add a flag to just send 0x00 as pubkeys since the Lodestar validator does not need them.
+      const pubkeys = getPubkeysForIndices(state.validators, indexes);
+
+      const duties: routes.validator.ProposerDuty[] = [];
+      for (let i = 0; i < SLOTS_PER_EPOCH; i++) {
+        duties.push({slot: startSlot + i, validatorIndex: indexes[i], pubkey: pubkeys[i]});
+      }
+
+      // Returns `null` on the one-off scenario where the genesis block decides its own shuffling.
+      // It should be set to the latest block applied to `self` or the genesis block root.
+      const dependentRoot = proposerShufflingDecisionRoot(state) || (await getGenesisBlockRoot(state));
+
+      return {
+        data: duties,
+        meta: {
+          dependentRoot: toRootHex(dependentRoot),
+          executionOptimistic: isOptimisticBlock(head),
+        },
+      };
+    },
+
+    async getAttesterDuties({epoch, indices}) {
+      notWhileSyncing();
+
+      if (indices.length === 0) {
+        throw new ApiError(400, "No validator to get attester duties");
+      }
+
+      // May request for an epoch that's in the future
+      await waitForNextClosestEpoch();
+
+      // should not compare to headEpoch in order to handle skipped slots
+      // Check if the epoch is in the future after waiting for requested slot
+      if (epoch > chain.clock.currentEpoch + 1) {
+        throw new ApiError(400, "Cannot get duties for epoch more than one ahead");
+      }
+
+      const head = chain.forkChoice.getHead();
+      const state = await chain.getHeadStateAtCurrentEpoch(RegenCaller.getDuties);
+
+      // TODO: Determine what the current epoch would be if we fast-forward our system clock by
+      // `MAXIMUM_GOSSIP_CLOCK_DISPARITY`.
+      //
+      // Most of the time, `tolerantCurrentEpoch` will be equal to `currentEpoch`. However, during
+      // the first `MAXIMUM_GOSSIP_CLOCK_DISPARITY` duration of the epoch `tolerantCurrentEpoch`
+      // will equal `currentEpoch + 1`
+
+      // Check that all validatorIndex belong to the state before calling getCommitteeAssignments()
+      const pubkeys = getPubkeysForIndices(state.validators, indices);
+      const decisionRoot = state.epochCtx.getShufflingDecisionRoot(epoch);
+      const shuffling = await chain.shufflingCache.get(epoch, decisionRoot);
+      if (!shuffling) {
+        throw new ApiError(
+          500,
+          `No shuffling found to calculate committee assignments for epoch: ${epoch} and decisionRoot: ${decisionRoot}`
+        );
+      }
+      const committeeAssignments = calculateCommitteeAssignments(shuffling, indices);
+      const duties: routes.validator.AttesterDuty[] = [];
+      for (let i = 0, len = indices.length; i < len; i++) {
+        const validatorIndex = indices[i];
+        const duty = committeeAssignments.get(validatorIndex) as routes.validator.AttesterDuty | undefined;
+        if (duty) {
+          // Mutate existing object instead of re-creating another new object with spread operator
+          // Should be faster and require less memory
+          duty.pubkey = pubkeys[i];
+          duties.push(duty);
+        }
+      }
+
+      const dependentRoot = attesterShufflingDecisionRoot(state, epoch) || (await getGenesisBlockRoot(state));
+
+      return {
+        data: duties,
+        meta: {
+          dependentRoot: toRootHex(dependentRoot),
+          executionOptimistic: isOptimisticBlock(head),
+        },
+      };
+    },
+
+    /**
+     * `POST /eth/v1/validator/duties/sync/{epoch}`
+     *
+     * Requests the beacon node to provide a set of sync committee duties for a particular epoch.
+     * - Although pubkey can be inferred from the index we return it to keep this call analogous with the one that
+     *   fetches attester duties.
+     * - `sync_committee_index` is the index of the validator in the sync committee. This can be used to infer the
+     *   subnet to which the contribution should be broadcast. Note, there can be multiple per validator.
+     *
+     * https://github.com/ethereum/beacon-APIs/pull/134
+     *
+     * @param validatorIndices an array of the validator indices for which to obtain the duties.
+     */
+    async getSyncCommitteeDuties({epoch, indices}) {
+      notWhileSyncing();
+
+      if (indices.length === 0) {
+        throw new ApiError(400, "No validator to get attester duties");
+      }
+
+      // May request for an epoch that's in the future
+      await waitForNextClosestEpoch();
+
+      // sync committee duties have a lookahead of 1 day. Assuming the validator only requests duties for upcoming
+      // epochs, the head state will very likely have the duties available for the requested epoch.
+      // Note: does not support requesting past duties
+      const head = chain.forkChoice.getHead();
+      const state = chain.getHeadState();
+
+      // Check that all validatorIndex belong to the state before calling getCommitteeAssignments()
+      const pubkeys = getPubkeysForIndices(state.validators, indices);
+      // Ensures `epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD <= current_epoch // EPOCHS_PER_SYNC_COMMITTEE_PERIOD + 1`
+      const syncCommitteeCache = state.epochCtx.getIndexedSyncCommitteeAtEpoch(epoch);
+      const validatorSyncCommitteeIndexMap = syncCommitteeCache.validatorIndexMap;
+
+      const duties: routes.validator.SyncDuty[] = [];
+      for (let i = 0, len = indices.length; i < len; i++) {
+        const validatorIndex = indices[i];
+        const validatorSyncCommitteeIndices = validatorSyncCommitteeIndexMap.get(validatorIndex);
+        if (validatorSyncCommitteeIndices) {
+          duties.push({
+            pubkey: pubkeys[i],
+            validatorIndex,
+            validatorSyncCommitteeIndices,
+          });
+        }
+      }
+
+      return {
+        data: duties,
+        meta: {executionOptimistic: isOptimisticBlock(head)},
+      };
+    },
+
+    async getAggregatedAttestation({attestationDataRoot, slot}) {
+      notWhileSyncing();
+
+      await waitForSlot(slot); // Must never request for a future slot > currentSlot
+
+      const dataRootHex = toRootHex(attestationDataRoot);
+      const aggregate = chain.attestationPool.getAggregate(slot, dataRootHex, null);
+      const fork = chain.config.getForkName(slot);
+
+      if (isForkPostElectra(fork)) {
+        throw new ApiError(
+          400,
+          `Use getAggregatedAttestationV2 to retrieve aggregated attestations for post-electra fork=${fork}`
+        );
+      }
+
+      if (!aggregate) {
+        throw new ApiError(404, `No aggregated attestation for slot=${slot}, dataRoot=${dataRootHex}`);
+      }
+
+      metrics?.production.producedAggregateParticipants.observe(aggregate.aggregationBits.getTrueBitIndexes().length);
+
+      return {
+        data: aggregate,
+      };
+    },
+
+    async getAggregatedAttestationV2({attestationDataRoot, slot, committeeIndex}) {
+      notWhileSyncing();
+
+      await waitForSlot(slot); // Must never request for a future slot > currentSlot
+
+      const dataRootHex = toRootHex(attestationDataRoot);
+      const aggregate = chain.attestationPool.getAggregate(slot, dataRootHex, committeeIndex);
+
+      if (!aggregate) {
+        throw new ApiError(
+          404,
+          `No aggregated attestation for slot=${slot}, dataRoot=${dataRootHex}, committeeIndex=${committeeIndex}`
+        );
+      }
+
+      metrics?.production.producedAggregateParticipants.observe(aggregate.aggregationBits.getTrueBitIndexes().length);
+
+      return {
+        data: aggregate,
+        meta: {version: config.getForkName(slot)},
+      };
+    },
+
+    async publishAggregateAndProofs({signedAggregateAndProofs}) {
+      await this.publishAggregateAndProofsV2({signedAggregateAndProofs});
+    },
+
+    async publishAggregateAndProofsV2({signedAggregateAndProofs}) {
+      notWhileSyncing();
+
+      const seenTimestampSec = Date.now() / 1000;
+      const failures: FailureList = [];
+      const fork = chain.config.getForkName(chain.clock.currentSlot);
+
+      await Promise.all(
+        signedAggregateAndProofs.map(async (signedAggregateAndProof, i) => {
+          try {
+            // TODO: Validate in batch
+            const validateFn = () => validateApiAggregateAndProof(fork, chain, signedAggregateAndProof);
+            const {slot, beaconBlockRoot} = signedAggregateAndProof.message.aggregate.data;
+            // when a validator is configured with multiple beacon node urls, this attestation may come from another beacon node
+            // and the block hasn't been in our forkchoice since we haven't seen / processing that block
+            // see https://github.com/ChainSafe/lodestar/issues/5098
+            const {indexedAttestation, committeeValidatorIndices, attDataRootHex} =
+              await validateGossipFnRetryUnknownRoot(validateFn, network, chain, slot, beaconBlockRoot);
+
+            const insertOutcome = chain.aggregatedAttestationPool.add(
+              signedAggregateAndProof.message.aggregate,
+              attDataRootHex,
+              indexedAttestation.attestingIndices.length,
+              committeeValidatorIndices
+            );
+            metrics?.opPool.aggregatedAttestationPool.apiInsertOutcome.inc({insertOutcome});
+
+            const sentPeers = await network.publishBeaconAggregateAndProof(signedAggregateAndProof);
+            chain.validatorMonitor?.onPoolSubmitAggregatedAttestation(seenTimestampSec, indexedAttestation, sentPeers);
+          } catch (e) {
+            const logCtx = {
+              slot: signedAggregateAndProof.message.aggregate.data.slot,
+              index: signedAggregateAndProof.message.aggregate.data.index,
+            };
+
+            if (e instanceof AttestationError && e.type.code === AttestationErrorCode.AGGREGATOR_ALREADY_KNOWN) {
+              logger.debug("Ignoring known signedAggregateAndProof", logCtx);
+              return; // Ok to submit the same aggregate twice
+            }
+
+            failures.push({index: i, message: (e as Error).message});
+            logger.verbose(`Error on publishAggregateAndProofs [${i}]`, logCtx, e as Error);
+            if (e instanceof AttestationError && e.action === GossipAction.REJECT) {
+              chain.persistInvalidSszValue(ssz.phase0.SignedAggregateAndProof, signedAggregateAndProof, "api_reject");
+            }
+          }
+        })
+      );
+
+      if (failures.length > 0) {
+        throw new IndexedError("Error processing aggregate and proofs", failures);
+      }
+    },
+
+    /**
+     * POST `/eth/v1/validator/contribution_and_proofs`
+     *
+     * Publish multiple signed sync committee contribution and proofs
+     *
+     * https://github.com/ethereum/beacon-APIs/pull/137
+     */
+    async publishContributionAndProofs({contributionAndProofs}) {
+      notWhileSyncing();
+
+      const failures: FailureList = [];
+
+      await Promise.all(
+        contributionAndProofs.map(async (contributionAndProof, i) => {
+          try {
+            // TODO: Validate in batch
+            const {syncCommitteeParticipantIndices} = await validateSyncCommitteeGossipContributionAndProof(
+              chain,
+              contributionAndProof,
+              true // skip known participants check
+            );
+            const insertOutcome = chain.syncContributionAndProofPool.add(
+              contributionAndProof.message,
+              syncCommitteeParticipantIndices.length,
+              true
+            );
+            metrics?.opPool.syncContributionAndProofPool.apiInsertOutcome.inc({insertOutcome});
+            await network.publishContributionAndProof(contributionAndProof);
+          } catch (e) {
+            const logCtx = {
+              slot: contributionAndProof.message.contribution.slot,
+              subcommitteeIndex: contributionAndProof.message.contribution.subcommitteeIndex,
+            };
+
+            if (
+              e instanceof SyncCommitteeError &&
+              e.type.code === SyncCommitteeErrorCode.SYNC_COMMITTEE_AGGREGATOR_ALREADY_KNOWN
+            ) {
+              logger.debug("Ignoring known contributionAndProof", logCtx);
+              return; // Ok to submit the same aggregate twice
+            }
+
+            failures.push({index: i, message: (e as Error).message});
+            logger.verbose(`Error on publishContributionAndProofs [${i}]`, logCtx, e as Error);
+            if (e instanceof SyncCommitteeError && e.action === GossipAction.REJECT) {
+              chain.persistInvalidSszValue(ssz.altair.SignedContributionAndProof, contributionAndProof, "api_reject");
+            }
+          }
+        })
+      );
+
+      if (failures.length > 0) {
+        throw new IndexedError("Error processing contribution and proofs", failures);
+      }
+    },
+
+    async prepareBeaconCommitteeSubnet({subscriptions}) {
+      notWhileSyncing();
+
+      await network.prepareBeaconCommitteeSubnets(
+        subscriptions.map(({validatorIndex, slot, isAggregator, committeesAtSlot, committeeIndex}) => ({
+          validatorIndex: validatorIndex,
+          subnet: computeSubnetForCommitteesAtSlot(slot, committeesAtSlot, committeeIndex),
+          slot: slot,
+          isAggregator: isAggregator,
+        }))
+      );
+
+      // TODO:
+      // If the discovery mechanism isn't disabled, attempt to set up a peer discovery for the
+      // required subnets.
+
+      if (metrics) {
+        for (const subscription of subscriptions) {
+          chain.validatorMonitor?.registerLocalValidator(subscription.validatorIndex);
+        }
+      }
+    },
+
+    /**
+     * POST `/eth/v1/validator/sync_committee_subscriptions`
+     *
+     * Subscribe to a number of sync committee subnets.
+     * Sync committees are not present in phase0, but are required for Altair networks.
+     * Subscribing to sync committee subnets is an action performed by VC to enable network participation in Altair networks,
+     * and only required if the VC has an active validator in an active sync committee.
+     *
+     * https://github.com/ethereum/beacon-APIs/pull/136
+     */
+    async prepareSyncCommitteeSubnets({subscriptions}) {
+      notWhileSyncing();
+
+      // A `validatorIndex` can be in multiple subnets, so compute the CommitteeSubscription with double for loop
+      const subs: CommitteeSubscription[] = [];
+      for (const sub of subscriptions) {
+        for (const committeeIndex of sub.syncCommitteeIndices) {
+          const subnet = Math.floor(committeeIndex / SYNC_COMMITTEE_SUBNET_SIZE);
+          subs.push({
+            validatorIndex: sub.validatorIndex,
+            subnet: subnet,
+            // Subscribe until the end of `untilEpoch`: https://github.com/ethereum/beacon-APIs/pull/136#issuecomment-840315097
+            slot: computeStartSlotAtEpoch(sub.untilEpoch + 1),
+            isAggregator: true,
+          });
+        }
+      }
+
+      await network.prepareSyncCommitteeSubnets(subs);
+
+      if (metrics) {
+        for (const subscription of subscriptions) {
+          chain.validatorMonitor?.registerLocalValidatorInSyncCommittee(
+            subscription.validatorIndex,
+            subscription.untilEpoch
+          );
+        }
+      }
+    },
+
+    async prepareBeaconProposer({proposers}) {
+      await chain.updateBeaconProposerData(chain.clock.currentEpoch, proposers);
+    },
+
+    async submitBeaconCommitteeSelections() {
+      throw new OnlySupportedByDVT();
+    },
+
+    async submitSyncCommitteeSelections() {
+      throw new OnlySupportedByDVT();
+    },
+
+    async getLiveness({epoch, indices}) {
+      if (indices.length === 0) {
+        return {
+          data: [],
+        };
+      }
+      const currentEpoch = chain.clock.currentEpoch;
+      if (epoch < currentEpoch - 1 || epoch > currentEpoch + 1) {
+        throw new ApiError(
+          400,
+          `Request epoch ${epoch} is more than one epoch before or after the current epoch ${currentEpoch}`
+        );
+      }
+
+      return {
+        data: indices.map((index) => ({
+          index,
+          isLive: chain.validatorSeenAtEpoch(index, epoch),
+        })),
+      };
+    },
+
+    async registerValidator({registrations}) {
+      if (!chain.executionBuilder) {
+        throw Error("External builder not configured");
+      }
+
+      // should only send active or pending validator to builder
+      // Spec: https://ethereum.github.io/builder-specs/#/Builder/registerValidator
+      const headState = chain.getHeadState();
+      const currentEpoch = chain.clock.currentEpoch;
+
+      const filteredRegistrations = registrations.filter((registration) => {
+        const {pubkey} = registration.message;
+        const validatorIndex = headState.epochCtx.pubkey2index.get(pubkey);
+        if (validatorIndex === null) return false;
+
+        const validator = headState.validators.getReadonly(validatorIndex);
+        const status = getValidatorStatus(validator, currentEpoch);
+        return (
+          status === "active_exiting" ||
+          status === "active_ongoing" ||
+          status === "active_slashed" ||
+          status === "pending_initialized" ||
+          status === "pending_queued"
+        );
+      });
+
+      await chain.executionBuilder.registerValidator(currentEpoch, filteredRegistrations);
+
+      logger.debug("Forwarded validator registrations to connected builder", {
+        epoch: currentEpoch,
+        count: filteredRegistrations.length,
+      });
+    },
+  };
+}