@lodestar/validator 1.35.0-dev.83de5b8dea → 1.35.0-dev.8689cc3545

package/src/services/externalSignerSync.ts

@@ -0,0 +1,81 @@
+import {PublicKey} from "@chainsafe/blst";
+import {ChainForkConfig} from "@lodestar/config";
+import {SLOTS_PER_EPOCH} from "@lodestar/params";
+import {fromHex, toPrintableUrl} from "@lodestar/utils";
+import {externalSignerGetKeys} from "../util/externalSignerClient.js";
+import {LoggerVc} from "../util/index.js";
+import {SignerType, ValidatorStore} from "./validatorStore.js";
+
+export type ExternalSignerOptions = {
+  url?: string;
+  fetch?: boolean;
+  fetchInterval?: number;
+};
+
+/**
+ * This service is responsible for keeping the keys managed by the connected
+ * external signer and the validator client in sync by adding newly discovered keys
+ * and removing no longer present keys on external signer from the validator store.
+ */
+export function pollExternalSignerPubkeys(
+  config: ChainForkConfig,
+  logger: LoggerVc,
+  signal: AbortSignal,
+  validatorStore: ValidatorStore,
+  opts?: ExternalSignerOptions
+): void {
+  const externalSigner = opts ?? {};
+
+  if (!externalSigner.url || !externalSigner.fetch) {
+    return; // Disabled
+  }
+
+  async function fetchExternalSignerPubkeys(): Promise<void> {
+    // External signer URL is already validated earlier
+    const externalSignerUrl = externalSigner.url as string;
+    const printableUrl = toPrintableUrl(externalSignerUrl);
+
+    try {
+      logger.debug("Fetching public keys from external signer", {url: printableUrl});
+      const externalPubkeys = await externalSignerGetKeys(externalSignerUrl);
+      assertValidPubkeysHex(externalPubkeys);
+      logger.debug("Received public keys from external signer", {url: printableUrl, count: externalPubkeys.length});
+
+      const localPubkeys = validatorStore.getRemoteSignerPubkeys(externalSignerUrl);
+      logger.debug("Local public keys stored for external signer", {url: printableUrl, count: localPubkeys.length});
+
+      const localPubkeysSet = new Set(localPubkeys);
+      for (const pubkey of externalPubkeys) {
+        if (!localPubkeysSet.has(pubkey)) {
+          await validatorStore.addSigner({type: SignerType.Remote, pubkey, url: externalSignerUrl});
+          logger.info("Added remote signer", {pubkey, url: printableUrl});
+        }
+      }
+
+      const externalPubkeysSet = new Set(externalPubkeys);
+      for (const pubkey of localPubkeys) {
+        if (!externalPubkeysSet.has(pubkey)) {
+          validatorStore.removeSigner(pubkey);
+          logger.info("Removed remote signer", {pubkey, url: printableUrl});
+        }
+      }
+    } catch (e) {
+      logger.error("Failed to fetch public keys from external signer", {url: printableUrl}, e as Error);
+    }
+  }
+
+  const interval = setInterval(
+    fetchExternalSignerPubkeys,
+    externalSigner.fetchInterval ??
+      // Once per epoch by default
+      SLOTS_PER_EPOCH * config.SECONDS_PER_SLOT * 1000
+  );
+  signal.addEventListener("abort", () => clearInterval(interval), {once: true});
+}
+
+function assertValidPubkeysHex(pubkeysHex: string[]): void {
+  for (const pubkeyHex of pubkeysHex) {
+    const pubkeyBytes = fromHex(pubkeyHex);
+    PublicKey.fromBytes(pubkeyBytes, true);
+  }
+}

package/src/services/indices.ts

@@ -0,0 +1,165 @@
+import {ApiClient, routes} from "@lodestar/api";
+import {ValidatorIndex} from "@lodestar/types";
+import {Logger, MapDef, toPubkeyHex} from "@lodestar/utils";
+import {Metrics} from "../metrics.js";
+import {batchItems} from "../util/index.js";
+
+/**
+ * This is to prevent the "Request body is too large" issue for http post.
+ * Typical servers accept up to 1MB (2 ** 20 bytes) of request body, for example fastify and nginx.
+ * A hex encoded public key with "0x"-prefix has a size of 98 bytes + 2 bytes to account for commas
+ * and other JSON padding. `Math.floor(2 ** 20 / 100) == 10485`, we can send up to ~10k keys per request.
+ */
+const PUBKEYS_PER_REQUEST = 10_000;
+
+// To assist with readability
+type PubkeyHex = string;
+
+// To assist with logging statuses, we only log the statuses that are not active_exiting or withdrawal_possible
+type SimpleValidatorStatus = "pending" | "active" | "exited" | "withdrawn";
+
+const statusToSimpleStatusMapping = (status: routes.beacon.ValidatorStatus): SimpleValidatorStatus => {
+  switch (status) {
+    case "active_exiting":
+    case "active_slashed":
+    case "active_ongoing":
+      return "active";
+
+    case "withdrawal_possible":
+    case "exited_slashed":
+    case "exited_unslashed":
+      return "exited";
+
+    case "pending_initialized":
+    case "pending_queued":
+      return "pending";
+
+    case "withdrawal_done":
+      return "withdrawn";
+  }
+};
+
+export class IndicesService {
+  readonly index2pubkey = new Map<ValidatorIndex, PubkeyHex>();
+  /** Indexed by pubkey in hex 0x prefixed */
+  readonly pubkey2index = new Map<PubkeyHex, ValidatorIndex>();
+  // Request indices once
+  private pollValidatorIndicesPromise: Promise<ValidatorIndex[]> | null = null;
+
+  constructor(
+    private readonly logger: Logger,
+    private readonly api: ApiClient,
+    private readonly metrics: Metrics | null
+  ) {
+    if (metrics) {
+      metrics.indices.addCollect(() => metrics.indices.set(this.index2pubkey.size));
+    }
+  }
+
+  get indexCount(): number {
+    return this.index2pubkey.size;
+  }
+
+  /** Returns the validator index for a given validator pubkey */
+  getValidatorIndex(pubKey: PubkeyHex): ValidatorIndex | undefined {
+    return this.pubkey2index.get(pubKey);
+  }
+
+  /** Return all known indices from the validatorStore pubkeys */
+  getAllLocalIndices(): ValidatorIndex[] {
+    return Array.from(this.index2pubkey.keys());
+  }
+
+  /** Return true if `index` is active part of this validator client */
+  hasValidatorIndex(index: ValidatorIndex): boolean {
+    return this.index2pubkey.has(index);
+  }
+
+  async pollValidatorIndices(pubkeysHex: PubkeyHex[]): Promise<ValidatorIndex[]> {
+    // Ensures pollValidatorIndicesInternal() is not called more than once at the same time.
+    // AttestationDutiesService, SyncCommitteeDutiesService and DoppelgangerService will call this function at the same time, so this will
+    // cache the promise and return it to the second caller, preventing calling the API twice for the same data.
+    if (this.pollValidatorIndicesPromise) {
+      return this.pollValidatorIndicesPromise;
+    }
+
+    this.pollValidatorIndicesPromise = this.pollValidatorIndicesInternal(pubkeysHex).finally(() => {
+      // Once the pollValidatorIndicesInternal() resolves or rejects null the cached promise so it can be called again.
+      this.pollValidatorIndicesPromise = null;
+    });
+    return this.pollValidatorIndicesPromise;
+  }
+
+  removeForKey(pubkey: PubkeyHex): boolean {
+    for (const [index, value] of this.index2pubkey) {
+      if (value === pubkey) {
+        this.index2pubkey.delete(index);
+      }
+    }
+    return this.pubkey2index.delete(pubkey);
+  }
+
+  /** Iterate through all the voting pubkeys in the `ValidatorStore` and attempt to learn any unknown
+      validator indices. Returns the new discovered indexes */
+  private async pollValidatorIndicesInternal(pubkeysHex: PubkeyHex[]): Promise<ValidatorIndex[]> {
+    const pubkeysHexToDiscover = pubkeysHex.filter((pubkey) => !this.pubkey2index.has(pubkey));
+
+    if (pubkeysHexToDiscover.length === 0) {
+      return [];
+    }
+
+    // Query the remote BN to resolve a pubkey to a validator index.
+    // support up to 10k pubkeys per poll
+    const pubkeysHexBatches = batchItems(pubkeysHexToDiscover, {batchSize: PUBKEYS_PER_REQUEST});
+
+    const newIndices: number[] = [];
+    for (const pubkeysHexBatch of pubkeysHexBatches) {
+      const validatorIndicesArr = await this.fetchValidatorIndices(pubkeysHexBatch);
+      newIndices.push(...validatorIndicesArr);
+    }
+
+    this.metrics?.discoveredIndices.inc(newIndices.length);
+
+    return newIndices;
+  }
+
+  private async fetchValidatorIndices(pubkeysHex: string[]): Promise<ValidatorIndex[]> {
+    const validators = (await this.api.beacon.postStateValidators({stateId: "head", validatorIds: pubkeysHex})).value();
+
+    const newIndices = [];
+
+    const allValidatorStatuses = new MapDef<SimpleValidatorStatus, number>(() => 0);
+
+    for (const validator of validators) {
+      // Group all validators by status
+      const status = statusToSimpleStatusMapping(validator.status);
+      allValidatorStatuses.set(status, allValidatorStatuses.getOrDefault(status) + 1);
+
+      const pubkeyHex = toPubkeyHex(validator.validator.pubkey);
+      if (!this.pubkey2index.has(pubkeyHex)) {
+        this.logger.info("Validator seen on beacon chain", {
+          validatorIndex: validator.index,
+          pubKey: pubkeyHex,
+        });
+        this.pubkey2index.set(pubkeyHex, validator.index);
+        this.index2pubkey.set(validator.index, pubkeyHex);
+        newIndices.push(validator.index);
+      }
+    }
+
+    // The number of validators that are not in the beacon chain
+    const pendingCount = pubkeysHex.length - validators.length;
+
+    allValidatorStatuses.set("pending", allValidatorStatuses.getOrDefault("pending") + pendingCount);
+
+    // Retrieve the number of validators for each status
+    const statuses = Object.fromEntries(Array.from(allValidatorStatuses.entries()).filter((entry) => entry[1] > 0));
+
+    // The total number of validators
+    const total = pubkeysHex.length;
+
+    this.logger.info("Validator statuses", {...statuses, total});
+
+    return newIndices;
+  }
+}

package/src/services/prepareBeaconProposer.ts

@@ -0,0 +1,119 @@
+import {ApiClient, routes} from "@lodestar/api";
+import {BeaconConfig} from "@lodestar/config";
+import {GENESIS_EPOCH, SLOTS_PER_EPOCH} from "@lodestar/params";
+import {Epoch, bellatrix} from "@lodestar/types";
+import {Metrics} from "../metrics.js";
+import {IClock, LoggerVc, batchItems} from "../util/index.js";
+import {ValidatorStore} from "./validatorStore.js";
+
+const REGISTRATION_CHUNK_SIZE = 512;
+/**
+ * This service is responsible for registering validators to beacon node with the
+ * proposer data (currently `feeRecipient`) so that it can issue advance fcUs to
+ * the engine for building execution payload with transactions.
+ *
+ * This needs to be done every epoch because the BN will cache it at most for
+ * two epochs.
+ */
+export function pollPrepareBeaconProposer(
+  config: BeaconConfig,
+  logger: LoggerVc,
+  api: ApiClient,
+  clock: IClock,
+  validatorStore: ValidatorStore,
+  _metrics: Metrics | null
+): void {
+  async function prepareBeaconProposer(epoch: Epoch): Promise<void> {
+    // Before bellatrix we don't need to update this data on bn/builder
+    if (epoch < config.BELLATRIX_FORK_EPOCH - 1) return;
+
+    // prepareBeaconProposer is not as time sensitive as attesting.
+    // Poll indices first, then call api.validator.prepareBeaconProposer once
+    await validatorStore.pollValidatorIndices().catch((e: Error) => {
+      logger.error("Error on pollValidatorIndices for prepareBeaconProposer", {epoch}, e);
+    });
+
+    const indicesChunks = batchItems(validatorStore.getAllLocalIndices(), {batchSize: REGISTRATION_CHUNK_SIZE});
+
+    for (const indices of indicesChunks) {
+      try {
+        const proposers = indices.map(
+          (index): routes.validator.ProposerPreparationData => ({
+            validatorIndex: index,
+            feeRecipient: validatorStore.getFeeRecipientByIndex(index),
+          })
+        );
+        (await api.validator.prepareBeaconProposer({proposers})).assertOk();
+        logger.debug("Registered proposers with beacon node", {epoch, count: proposers.length});
+      } catch (e) {
+        logger.error("Failed to register proposers with beacon node", {epoch}, e as Error);
+      }
+    }
+  }
+
+  clock.runEveryEpoch(prepareBeaconProposer);
+}
+
+/**
+ * This service is responsible for registering validators with the mev builder as they
+ * might prepare and keep ready the execution payloads of just registered validators.
+ *
+ * This needs to be done every epoch because the builder(s) will cache it at most for
+ * two epochs.
+ */
+export function pollBuilderValidatorRegistration(
+  config: BeaconConfig,
+  logger: LoggerVc,
+  api: ApiClient,
+  clock: IClock,
+  validatorStore: ValidatorStore,
+  _metrics: Metrics | null
+): void {
+  async function registerValidator(epoch: Epoch): Promise<void> {
+    // Don't send validator registrations pre-genesis as mev-boost-relay will reject
+    // those registrations anyways if timestamp is before genesis time and we wanna
+    // avoid caching and re-sending them in subsequent requests
+    if (epoch < GENESIS_EPOCH) return;
+
+    // Before bellatrix we don't need to update this data on bn/builder
+    if (epoch < config.BELLATRIX_FORK_EPOCH - 1) return;
+    const slot = epoch * SLOTS_PER_EPOCH;
+
+    // registerValidator is not as time sensitive as attesting.
+    // Poll indices first, then call api.validator.registerValidator once
+    await validatorStore.pollValidatorIndices().catch((e: Error) => {
+      logger.error("Error on pollValidatorIndices for registerValidator", {epoch}, e);
+    });
+    const pubkeyHexes = validatorStore
+      .getAllLocalIndices()
+      .map((index) => validatorStore.getPubkeyOfIndex(index))
+      .filter(
+        (pubkeyHex): pubkeyHex is string =>
+          pubkeyHex !== undefined &&
+          validatorStore.getBuilderSelectionParams(pubkeyHex).selection !==
+            routes.validator.BuilderSelection.ExecutionOnly
+      );
+
+    if (pubkeyHexes.length > 0) {
+      const pubkeyHexesChunks = batchItems(pubkeyHexes, {batchSize: REGISTRATION_CHUNK_SIZE});
+
+      for (const pubkeyHexes of pubkeyHexesChunks) {
+        try {
+          const registrations = await Promise.all(
+            pubkeyHexes.map((pubkeyHex): Promise<bellatrix.SignedValidatorRegistrationV1> => {
+              const feeRecipient = validatorStore.getFeeRecipient(pubkeyHex);
+              const gasLimit = validatorStore.getGasLimit(pubkeyHex);
+              return validatorStore.getValidatorRegistration(pubkeyHex, {feeRecipient, gasLimit}, slot);
+            })
+          );
+          (await api.validator.registerValidator({registrations})).assertOk();
+          logger.info("Published validator registrations to builder", {epoch, count: registrations.length});
+        } catch (e) {
+          logger.error("Failed to publish validator registrations to builder", {epoch}, e as Error);
+        }
+      }
+    }
+  }
+
+  clock.runEveryEpoch(registerValidator);
+}

package/src/services/syncCommittee.ts

@@ -0,0 +1,317 @@
+import {ApiClient, routes} from "@lodestar/api";
+import {ChainForkConfig} from "@lodestar/config";
+import {computeEpochAtSlot, isSyncCommitteeAggregator} from "@lodestar/state-transition";
+import {BLSSignature, CommitteeIndex, Root, Slot, altair} from "@lodestar/types";
+import {sleep} from "@lodestar/utils";
+import {Metrics} from "../metrics.js";
+import {PubkeyHex} from "../types.js";
+import {IClock, LoggerVc} from "../util/index.js";
+import {ChainHeaderTracker} from "./chainHeaderTracker.js";
+import {ValidatorEventEmitter} from "./emitter.js";
+import {SyncCommitteeDutiesService, SyncDutyAndProofs} from "./syncCommitteeDuties.js";
+import {SyncingStatusTracker} from "./syncingStatusTracker.js";
+import {SubcommitteeDuty, groupSyncDutiesBySubcommitteeIndex} from "./utils.js";
+import {ValidatorStore} from "./validatorStore.js";
+
+export type SyncCommitteeServiceOpts = {
+  scAfterBlockDelaySlotFraction?: number;
+  distributedAggregationSelection?: boolean;
+};
+
+/**
+ * Service that sets up and handles validator sync duties.
+ */
+export class SyncCommitteeService {
+  private readonly dutiesService: SyncCommitteeDutiesService;
+
+  constructor(
+    private readonly config: ChainForkConfig,
+    private readonly logger: LoggerVc,
+    private readonly api: ApiClient,
+    private readonly clock: IClock,
+    private readonly validatorStore: ValidatorStore,
+    private readonly emitter: ValidatorEventEmitter,
+    private readonly chainHeaderTracker: ChainHeaderTracker,
+    readonly syncingStatusTracker: SyncingStatusTracker,
+    private readonly metrics: Metrics | null,
+    private readonly opts?: SyncCommitteeServiceOpts
+  ) {
+    this.dutiesService = new SyncCommitteeDutiesService(
+      config,
+      logger,
+      api,
+      clock,
+      validatorStore,
+      syncingStatusTracker,
+      metrics,
+      {
+        distributedAggregationSelection: opts?.distributedAggregationSelection,
+      }
+    );
+
+    // At most every slot, check existing duties from SyncCommitteeDutiesService and run tasks
+    clock.runEverySlot(this.runSyncCommitteeTasks);
+  }
+
+  removeDutiesForKey(pubkey: PubkeyHex): void {
+    this.dutiesService.removeDutiesForKey(pubkey);
+  }
+
+  private runSyncCommitteeTasks = async (slot: Slot, signal: AbortSignal): Promise<void> => {
+    try {
+      // Before altair fork no need to check duties
+      if (computeEpochAtSlot(slot) < this.config.ALTAIR_FORK_EPOCH) {
+        return;
+      }
+
+      // Fetch info first so a potential delay is absorbed by the sleep() below
+      const dutiesAtSlot = await this.dutiesService.getDutiesAtSlot(slot);
+      if (dutiesAtSlot.length === 0) {
+        return;
+      }
+
+      if (this.opts?.distributedAggregationSelection) {
+        // Validator in distributed cluster only has a key share, not the full private key.
+        // The partial selection proofs must be exchanged for combined selection proofs by
+        // calling submitSyncCommitteeSelections on the distributed validator middleware client.
+        // This will run in parallel to other sync committee tasks but must be finished before starting
+        // sync committee contributions as it is required to correctly determine if validator is aggregator
+        // and to produce a ContributionAndProof that can be threshold aggregated by the middleware client.
+        this.runDistributedAggregationSelectionTasks(dutiesAtSlot, slot, signal).catch((e) =>
+          this.logger.error("Error on sync committee aggregation selection", {slot}, e)
+        );
+      }
+
+      // unlike Attestation, SyncCommitteeSignature could be published asap
+      // especially with lodestar, it's very busy at 1/3 of slot
+      // see https://github.com/ChainSafe/lodestar/issues/4608
+      await Promise.race([sleep(this.clock.msToSlot(slot + 1 / 3), signal), this.emitter.waitForBlockSlot(slot)]);
+      this.metrics?.syncCommitteeStepCallProduceMessage.observe(this.clock.secFromSlot(slot + 1 / 3));
+
+      // Step 1. Download, sign and publish an `SyncCommitteeMessage` for each validator.
+      //         Differs from AttestationService, `SyncCommitteeMessage` are equal for all
+      const beaconBlockRoot = await this.produceAndPublishSyncCommittees(slot, dutiesAtSlot);
+
+      // Step 2. If an attestation was produced, make an aggregate.
+      // First, wait until the `aggregation_production_instant` (2/3rds of the way though the slot)
+      await sleep(this.clock.msToSlot(slot + 2 / 3), signal);
+      this.metrics?.syncCommitteeStepCallProduceAggregate.observe(this.clock.secFromSlot(slot + 2 / 3));
+
+      // await for all so if the Beacon node is overloaded it auto-throttles
+      // TODO: This approach is conservative to reduce the node's load, review
+      const dutiesBySubcommitteeIndex = groupSyncDutiesBySubcommitteeIndex(dutiesAtSlot);
+      await Promise.all(
+        Array.from(dutiesBySubcommitteeIndex.entries()).map(async ([subcommitteeIndex, duties]) => {
+          if (duties.length === 0) return;
+          // Then download, sign and publish a `SignedAggregateAndProof` for each
+          // validator that is elected to aggregate for this `slot` and `subcommitteeIndex`.
+          await this.produceAndPublishAggregates(slot, subcommitteeIndex, beaconBlockRoot, duties).catch((e: Error) => {
+            this.logger.error("Error on SyncCommitteeContribution", {slot, index: subcommitteeIndex}, e);
+          });
+        })
+      );
+    } catch (e) {
+      this.logger.error("Error on runSyncCommitteeTasks", {slot}, e as Error);
+    }
+  };
+
+  /**
+   * Performs the first step of the attesting process: downloading `SyncCommittee` objects,
+   * signing them and returning them to the validator.
+   *
+   * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/altair/validator.md#sync-committee-messages
+   *
+   * Only one `SyncCommittee` is downloaded from the BN. It is then signed by each
+   * validator and the list of individually-signed `SyncCommittee` objects is returned to the BN.
+   */
+  private async produceAndPublishSyncCommittees(slot: Slot, duties: SyncDutyAndProofs[]): Promise<Root> {
+    const logCtx = {slot};
+
+    // /eth/v1/beacon/blocks/:blockId/root -> at slot -1
+
+    // Produce one attestation data per slot and subcommitteeIndex
+    // Spec: the validator should prepare a SyncCommitteeMessage for the previous slot (slot - 1)
+    // as soon as they have determined the head block of slot - 1
+
+    const blockRoot: Uint8Array =
+      this.chainHeaderTracker.getCurrentChainHead(slot) ??
+      (await this.api.beacon.getBlockRoot({blockId: "head"})).value().root;
+
+    const signatures: altair.SyncCommitteeMessage[] = [];
+
+    await Promise.all(
+      duties.map(async ({duty}) => {
+        const logCtxValidator = {...logCtx, validatorIndex: duty.validatorIndex};
+        try {
+          signatures.push(
+            await this.validatorStore.signSyncCommitteeSignature(duty.pubkey, duty.validatorIndex, slot, blockRoot)
+          );
+          this.logger.debug("Signed SyncCommitteeMessage", logCtxValidator);
+        } catch (e) {
+          this.logger.error("Error signing SyncCommitteeMessage", logCtxValidator, e as Error);
+        }
+      })
+    );
+
+    // by default we want to submit SyncCommitteeSignature asap after we receive block
+    // provide a delay option just in case any client implementation validate the existence of block in
+    // SyncCommitteeSignature gossip validation.
+    const msToOneThirdSlot = this.clock.msToSlot(slot + 1 / 3);
+    const afterBlockDelayMs = 1000 * this.clock.secondsPerSlot * (this.opts?.scAfterBlockDelaySlotFraction ?? 0);
+    const toDelayMs = Math.min(msToOneThirdSlot, afterBlockDelayMs);
+    if (toDelayMs > 0) {
+      await sleep(toDelayMs);
+    }
+
+    this.metrics?.syncCommitteeStepCallPublishMessage.observe(this.clock.secFromSlot(slot + 1 / 3));
+
+    if (signatures.length > 0) {
+      try {
+        (await this.api.beacon.submitPoolSyncCommitteeSignatures({signatures})).assertOk();
+        this.logger.info("Published SyncCommitteeMessage", {...logCtx, count: signatures.length});
+        this.metrics?.publishedSyncCommitteeMessage.inc(signatures.length);
+      } catch (e) {
+        this.logger.error("Error publishing SyncCommitteeMessage", logCtx, e as Error);
+      }
+    }
+
+    return blockRoot;
+  }
+
+  /**
+   * Performs the second step of the attesting process: downloading an aggregated `SyncCommittee`,
+   * converting it into a `SignedAggregateAndProof` and returning it to the BN.
+   *
+   * https://github.com/ethereum/consensus-specs/blob/v1.1.10/specs/altair/validator.md#sync-committee-contributions
+   *
+   * Only one aggregated `SyncCommittee` is downloaded from the BN. It is then signed
+   * by each validator and the list of individually-signed `SignedAggregateAndProof` objects is
+   * returned to the BN.
+   */
+  private async produceAndPublishAggregates(
+    slot: Slot,
+    subcommitteeIndex: CommitteeIndex,
+    beaconBlockRoot: Root,
+    duties: SubcommitteeDuty[]
+  ): Promise<void> {
+    const logCtx = {slot, index: subcommitteeIndex};
+
+    // No validator is aggregator, skip
+    if (duties.every(({selectionProof}) => selectionProof === null)) {
+      return;
+    }
+
+    this.logger.verbose("Producing SyncCommitteeContribution", logCtx);
+    const res = await this.api.validator.produceSyncCommitteeContribution({slot, subcommitteeIndex, beaconBlockRoot});
+
+    const signedContributions: altair.SignedContributionAndProof[] = [];
+
+    await Promise.all(
+      duties.map(async ({duty, selectionProof}) => {
+        const logCtxValidator = {...logCtx, validatorIndex: duty.validatorIndex};
+        try {
+          // Produce signed contributions only for validators that are subscribed aggregators.
+          if (selectionProof !== null) {
+            signedContributions.push(
+              await this.validatorStore.signContributionAndProof(duty, selectionProof, res.value())
+            );
+            this.logger.debug("Signed SyncCommitteeContribution", logCtxValidator);
+          }
+        } catch (e) {
+          this.logger.error("Error signing SyncCommitteeContribution", logCtxValidator, e as Error);
+        }
+      })
+    );
+
+    this.metrics?.syncCommitteeStepCallPublishAggregate.observe(this.clock.secFromSlot(slot + 2 / 3));
+
+    if (signedContributions.length > 0) {
+      try {
+        (
+          await this.api.validator.publishContributionAndProofs({contributionAndProofs: signedContributions})
+        ).assertOk();
+        this.logger.info("Published SyncCommitteeContribution", {...logCtx, count: signedContributions.length});
+        this.metrics?.publishedSyncCommitteeContribution.inc(signedContributions.length);
+      } catch (e) {
+        this.logger.error("Error publishing SyncCommitteeContribution", logCtx, e as Error);
+      }
+    }
+  }
+
+  /**
+   * Performs additional sync committee contribution tasks required if validator is part of distributed cluster
+   *
+   * 1. Exchange partial for combined selection proofs
+   * 2. Determine validators that should produce sync committee contribution
+   * 3. Mutate duty objects to set selection proofs for aggregators
+   *
+   * See https://docs.google.com/document/d/1q9jOTPcYQa-3L8luRvQJ-M0eegtba4Nmon3dpO79TMk/mobilebasic
+   */
+  private async runDistributedAggregationSelectionTasks(
+    duties: SyncDutyAndProofs[],
+    slot: number,
+    signal: AbortSignal
+  ): Promise<void> {
+    const partialSelections: routes.validator.SyncCommitteeSelection[] = [];
+
+    for (const {duty, selectionProofs} of duties) {
+      const validatorSelections: routes.validator.SyncCommitteeSelection[] = selectionProofs.map(
+        ({subcommitteeIndex, partialSelectionProof}) => ({
+          validatorIndex: duty.validatorIndex,
+          slot,
+          subcommitteeIndex,
+          selectionProof: partialSelectionProof as BLSSignature,
+        })
+      );
+      partialSelections.push(...validatorSelections);
+    }
+
+    this.logger.debug("Submitting partial sync committee selection proofs", {slot, count: partialSelections.length});
+
+    const res = await Promise.race([
+      this.api.validator.submitSyncCommitteeSelections({selections: partialSelections}),
+      // Exit sync committee contributions flow if there is no response after 2/3 of slot.
+      // This is in contrast to attestations aggregations flow which is already exited at 1/3 of the slot
+      // because for sync committee is not required to resubscribe to subnets as beacon node will assume
+      // validator always aggregates. This allows us to wait until we have to produce sync committee contributions.
+      // Note that the sync committee contributions flow is not explicitly exited but rather will be skipped
+      // due to the fact that calculation of `is_sync_committee_aggregator` in SyncCommitteeDutiesService is not done
+      // and selectionProof is set to null, meaning no validator will be considered an aggregator.
+      sleep(this.clock.msToSlot(slot + 2 / 3), signal),
+    ]);
+
+    if (!res) {
+      throw new Error("Failed to receive combined selection proofs before 2/3 of slot");
+    }
+
+    const combinedSelections = res.value();
+    this.logger.debug("Received combined sync committee selection proofs", {slot, count: combinedSelections.length});
+
+    for (const dutyAndProofs of duties) {
+      const {validatorIndex, subnets} = dutyAndProofs.duty;
+
+      for (const subnet of subnets) {
+        const logCtxValidator = {slot, index: subnet, validatorIndex};
+
+        const combinedSelection = combinedSelections.find(
+          (s) => s.validatorIndex === validatorIndex && s.slot === slot && s.subcommitteeIndex === subnet
+        );
+
+        if (!combinedSelection) {
+          this.logger.warn("Did not receive combined sync committee selection proof", logCtxValidator);
+          continue;
+        }
+
+        const isAggregator = isSyncCommitteeAggregator(combinedSelection.selectionProof);
+
+        if (isAggregator) {
+          const selectionProofObject = dutyAndProofs.selectionProofs.find((p) => p.subcommitteeIndex === subnet);
+          if (selectionProofObject) {
+            // Update selection proof by mutating proof objects in duty object
+            selectionProofObject.selectionProof = combinedSelection.selectionProof;
+          }
+        }
+      }
+    }
+  }
+}