@lodestar/validator 1.35.0-dev.c85be4e26c → 1.35.0-dev.c9deb9b59f

package/src/buckets.ts

@@ -0,0 +1,30 @@
+// Buckets are separate database namespaces
+export enum Bucket {
+  // validator
+  // validator = 16, // DEPRECATED on v0.11.0
+  // lastProposedBlock = 17, // DEPRECATED on v0.11.0
+  // proposedAttestations = 18, // DEPRECATED on v0.11.0
+  // validator slashing protection
+  slashingProtectionBlockBySlot = 20,
+  slashingProtectionAttestationByTarget = 21,
+  slashingProtectionAttestationLowerBound = 22,
+  slashingProtectionMinSpanDistance = 23,
+  slashingProtectionMaxSpanDistance = 24,
+  // allForks_pendingBlock = 25, // Root -> SignedBeaconBlock // DEPRECATED on v0.30.0
+
+  validator_metaData = 41,
+}
+
+export function getBucketNameByValue<T extends Bucket>(enumValue: T): keyof typeof Bucket {
+  const keys = Object.keys(Bucket).filter((x) => {
+    if (Number.isNaN(parseInt(x))) {
+      return Bucket[x as keyof typeof Bucket] === enumValue;
+    }
+
+    return false;
+  }) as (keyof typeof Bucket)[];
+  if (keys.length > 0) {
+    return keys[0];
+  }
+  throw new Error("Missing bucket for value " + enumValue);
+}

package/src/defaults.ts

@@ -0,0 +1,8 @@
+export default {
+  db: {
+    name: "LodestarValidatorDB",
+  },
+  rpc: {
+    rpcUrl: "http://127.0.0.1:9545",
+  },
+};

package/src/genesis.ts

@@ -0,0 +1,19 @@
+import {ApiClient} from "@lodestar/api";
+import {Genesis} from "@lodestar/types/phase0";
+import {Logger, sleep} from "@lodestar/utils";
+
+/** The time between polls when waiting for genesis */
+const WAITING_FOR_GENESIS_POLL_MS = 12 * 1000;
+
+export async function waitForGenesis(api: ApiClient, logger: Logger, signal?: AbortSignal): Promise<Genesis> {
+  while (true) {
+    try {
+      return (await api.beacon.getGenesis()).value();
+    } catch (e) {
+      // TODO: Search for a 404 error which indicates that genesis has not yet occurred.
+      // Note: Lodestar API does not become online after genesis is found
+      logger.info("Waiting for genesis", {message: (e as Error).message});
+      await sleep(WAITING_FOR_GENESIS_POLL_MS, signal);
+    }
+  }
+}

package/src/index.ts

@@ -0,0 +1,22 @@
+export {waitForGenesis} from "./genesis.js";
+export {type Metrics, getMetrics} from "./metrics.js";
+export * from "./repositories/index.js";
+export type {
+  ProposerConfig,
+  Signer,
+  SignerLocal,
+  SignerRemote,
+  ValidatorProposerConfig,
+} from "./services/validatorStore.js";
+export {MAX_BUILDER_BOOST_FACTOR, SignerType, ValidatorStore, defaultOptions} from "./services/validatorStore.js";
+export * from "./slashingProtection/index.js";
+// Types
+export type {ProcessShutdownCallback} from "./types.js";
+// Remote signer client
+export {
+  SignableMessageType,
+  externalSignerGetKeys,
+  externalSignerPostSignature,
+  externalSignerUpCheck,
+} from "./util/externalSignerClient.js";
+export {Validator, type ValidatorOptions} from "./validator.js";

package/src/metrics.ts

@@ -0,0 +1,417 @@
+import {routes} from "@lodestar/api";
+import {MetricsRegisterExtra} from "@lodestar/utils";
+
+export enum MessageSource {
+  forward = "forward",
+  publish = "publish",
+}
+
+export enum BeaconHealth {
+  READY = 0,
+  SYNCING = 1,
+  ERROR = 2,
+}
+
+export type Metrics = ReturnType<typeof getMetrics>;
+
+export type LodestarGitData = {
+  /** "0.16.0 developer/feature-1 ac99f2b5" */
+  version: string;
+  /** "4f816b16dfde718e2d74f95f2c8292596138c248" */
+  commit: string;
+  /** "holesky" */
+  network: string;
+};
+
+/**
+ * A collection of metrics used by the validator client
+ */
+export function getMetrics(register: MetricsRegisterExtra, gitData: LodestarGitData) {
+  // Using function style instead of class to prevent having to re-declare all MetricsPrometheus types.
+
+  // Track version, same as https://github.com/ChainSafe/lodestar/blob/6df28de64f12ea90b341b219229a47c8a25c9343/packages/lodestar/src/metrics/metrics/lodestar.ts#L17
+  register
+    .gauge<LodestarGitData>({
+      name: "lodestar_version",
+      help: "Lodestar version",
+      labelNames: Object.keys(gitData) as [keyof LodestarGitData],
+    })
+    .set(gitData, 1);
+
+  return {
+    defaultConfiguration: register.gauge<{
+      builderSelection: routes.validator.BuilderSelection;
+      broadcastValidation: routes.beacon.BroadcastValidation;
+    }>({
+      name: "vc_default_configuration",
+      help: "Default validator configuration",
+      labelNames: ["builderSelection", "broadcastValidation"],
+    }),
+
+    // Attestation journey:
+    // - Wait for block or ATTESTATION_DUE_BPS, call prepare attestation
+    // - Get attestation, sign, call publish
+    // - Wait AGGREGATE_DUE_BPS, call prepare aggregate
+    // - Get aggregate, sign, call publish
+
+    attesterStepCallProduceAttestation: register.histogram({
+      name: "vc_attester_step_call_produce_attestation_seconds",
+      help: "Time between ATTESTATION_DUE_BPS of slot and call produce attestation",
+      // Attester flow can start early if block is imported before ATTESTATION_DUE_BPS of the slot
+      // This measure is critical so we need very good resolution around the 0 second mark
+      buckets: [-3, -1, 0, 1, 2, 3, 6, 12],
+    }),
+    attesterStepCallPublishAttestation: register.histogram({
+      name: "vc_attester_step_call_publish_attestation_seconds",
+      help: "Time between ATTESTATION_DUE_BPS of slot and call publish attestation",
+      buckets: [-3, -1, 0, 1, 2, 3, 6, 12],
+    }),
+
+    attesterStepCallProduceAggregate: register.histogram({
+      name: "vc_attester_step_call_produce_aggregate_seconds",
+      help: "Time between AGGREGATE_DUE_BPS of slot and call produce aggregate",
+      // Aggregate production starts at AGGREGATE_DUE_BPS the earliest.
+      // Track values close to 0 (expected) in greater resolution, values over 12 overflow into the next slot.
+      buckets: [0.5, 1, 2, 3, 6, 12],
+    }),
+    attesterStepCallPublishAggregate: register.histogram({
+      name: "vc_attester_step_call_publish_aggregate_seconds",
+      help: "Time between AGGREGATE_DUE_BPS of slot and call publish aggregate",
+      buckets: [0.5, 1, 2, 3, 6, 12],
+    }),
+
+    syncCommitteeStepCallProduceMessage: register.histogram({
+      name: "vc_sync_committee_step_call_produce_message_seconds",
+      help: "Time between SYNC_MESSAGE_DUE_BPS of slot and call produce message",
+      // Max wait time is SYNC_MESSAGE_DUE_BPS of slot
+      buckets: [0.5, 1, 2, 3, 6, 12],
+    }),
+    syncCommitteeStepCallPublishMessage: register.histogram({
+      name: "vc_sync_committee_step_call_publish_message_seconds",
+      help: "Time between SYNC_MESSAGE_DUE_BPS of slot and call publish message",
+      buckets: [0.5, 1, 2, 3, 6, 12],
+    }),
+
+    syncCommitteeStepCallProduceAggregate: register.histogram({
+      name: "vc_sync_committee_step_call_produce_aggregate_seconds",
+      help: "Time between CONTRIBUTION_DUE_BPS of slot and call produce aggregate",
+      // Min wait time is CONTRIBUTION_DUE_BPS of slot
+      buckets: [0.5, 1, 2, 3, 6, 12],
+    }),
+    syncCommitteeStepCallPublishAggregate: register.histogram({
+      name: "vc_sync_committee_step_call_publish_aggregate_seconds",
+      help: "Time between CONTRIBUTION_DUE_BPS of slot and call publish aggregate",
+      buckets: [0.5, 1, 2, 3, 6, 12],
+    }),
+
+    // Min wait time it 0. CallProduceBlock step is a bit redundant since it must be 0, but let's check
+    proposerStepCallProduceBlock: register.histogram({
+      name: "vc_proposer_step_call_produce_block_seconds",
+      help: "Time between start of slot and call produce block",
+      buckets: [0.5, 1, 2, 3, 6, 8],
+    }),
+    proposerStepCallPublishBlock: register.histogram({
+      name: "vc_proposer_step_call_publish_block_seconds",
+      help: "Time between start of slot and call publish block",
+      buckets: [0.5, 1, 2, 3, 6, 8],
+    }),
+
+    // AttestationService
+
+    publishedAttestations: register.gauge({
+      name: "vc_published_attestations_total",
+      help: "Total published attestations",
+    }),
+
+    publishedAggregates: register.gauge({
+      name: "vc_published_aggregates_total",
+      help: "Total published aggregates",
+    }),
+
+    numParticipantsInAggregate: register.histogram({
+      name: "vc_attestation_service_participants_in_aggregate_total",
+      help: "Number of attestations in the published AggregatedAttestation",
+      buckets: [1, 25, 50, 100, 250, 400, 500, 600],
+    }),
+
+    attestaterError: register.gauge<{error: "produce" | "sign" | "publish"}>({
+      name: "vc_attestation_service_errors",
+      help: "Total errors in AttestationService",
+      labelNames: ["error"],
+    }),
+
+    // AttestationDutiesService
+
+    attesterDutiesCount: register.gauge({
+      name: "vc_attester_duties_count",
+      help: "Current count of duties in AttestationDutiesService",
+    }),
+
+    attesterDutiesEpochCount: register.gauge({
+      name: "vc_attester_duties_epoch_count",
+      help: "Current count of epoch duties in AttestationDutiesService",
+    }),
+
+    attesterDutiesReorg: register.gauge({
+      name: "vc_attestation_duties_reorg_total",
+      help: "Total count of instances the attester duties dependent root changed",
+    }),
+
+    attesterDutiesNextSlot: register.gauge({
+      // Metric is used by Rocket Pool dashboard (18391) to determine seconds until next attestation.
+      // It works without requiring any modification to the dashboard as the metric name is the
+      // same as Lighthouse uses for this.
+      name: "vc_attestation_duty_slot",
+      help: "Slot of next scheduled attestation duty",
+    }),
+
+    // BlockProposingService
+
+    blocksProduced: register.gauge({
+      name: "vc_block_produced_total",
+      help: "Total count of blocks produced",
+    }),
+
+    blocksPublished: register.gauge({
+      name: "vc_block_published_total",
+      help: "Total count of blocks published",
+    }),
+
+    blockProposingErrors: register.gauge<{error: "produce" | "publish"}>({
+      name: "vc_block_proposing_errors_total",
+      help: "Total count of errors producing or publishing a block",
+      labelNames: ["error"],
+    }),
+
+    // BlockDutiesService
+
+    proposerDutiesEpochCount: register.gauge({
+      name: "vc_proposer_duties_epoch_count",
+      help: "Current count of epoch duties in BlockDutiesService",
+    }),
+
+    proposerDutiesReorg: register.gauge({
+      name: "vc_proposer_duties_reorg_total",
+      help: "Total count of instances the proposer duties dependent root changed",
+    }),
+
+    newProposalDutiesDetected: register.gauge({
+      name: "vc_new_proposal_duties_detected_total",
+      help: "Total count of times new proposal duties were detected",
+    }),
+
+    // IndicesService
+
+    indices: register.gauge({
+      name: "vc_indices_count",
+      help: "Current count of indices in IndicesService",
+    }),
+
+    discoveredIndices: register.gauge({
+      name: "vc_discovered_indices_total",
+      help: "Total count of validator indices discovered",
+    }),
+
+    // SyncCommitteeService
+
+    publishedSyncCommitteeMessage: register.gauge({
+      name: "vc_published_sync_committee_message_total",
+      help: "Total published SyncCommitteeMessage",
+    }),
+
+    publishedSyncCommitteeContribution: register.gauge({
+      name: "vc_published_sync_committee_contribution_total",
+      help: "Total published SyncCommitteeContribution",
+    }),
+
+    // SyncCommitteeDutiesService
+
+    syncCommitteeDutiesCount: register.gauge({
+      name: "vc_sync_committee_duties_count",
+      help: "Current count of duties in SyncCommitteeDutiesService",
+    }),
+
+    syncCommitteeDutiesEpochCount: register.gauge({
+      name: "vc_sync_committee_duties_epoch_count",
+      help: "Current count of epoch duties in SyncCommitteeDutiesService",
+    }),
+
+    syncCommitteeDutiesReorg: register.gauge({
+      name: "vc_sync_committee_duties_reorg_total",
+      help: "Total count of instances the sync committee duties dependent root changed",
+    }),
+
+    // ValidatorStore
+
+    signers: register.gauge({
+      name: "vc_signers_count",
+      help: "Total count of instances the sync committee duties dependent root changed",
+    }),
+
+    localSignTime: register.histogram({
+      name: "vc_local_sign_time_seconds",
+      help: "Histogram of sign time for any signature with local signer",
+      // When using a local keystores, signing time is ~ 1ms
+      buckets: [0.0001, 0.001, 0.01, 0.1],
+    }),
+
+    remoteSignTime: register.histogram({
+      name: "vc_remote_sign_time_seconds",
+      help: "Histogram of sign time for any signature with remote signer",
+      // When using a remote signer sign time can be ~ 50-500ms
+      buckets: [0.01, 0.1, 1, 5],
+    }),
+
+    remoteSignErrors: register.gauge({
+      name: "vc_remote_sign_errors_total",
+      help: "Total count of errors calling a remote signer",
+    }),
+
+    signError: register.gauge({
+      name: "vc_sign_errors_total",
+      help: "Total count of errors calling a signer",
+    }),
+
+    slashingProtectionBlockError: register.gauge({
+      name: "vc_slashing_protection_block_errors_total",
+      help: "Total count of errors on slashingProtection.checkAndInsertBlockProposal",
+    }),
+
+    slashingProtectionAttestationError: register.gauge({
+      name: "vc_slashing_protection_attestation_errors_total",
+      help: "Total count of errors on slashingProtection.checkAndInsertAttestation",
+    }),
+
+    // REST API client
+
+    beaconHealth: register.gauge({
+      name: "vc_beacon_health",
+      help: `Current health status of the beacon(s) the validator is connected to. ${renderEnumNumeric(BeaconHealth)}`,
+    }),
+
+    restApiClient: {
+      requestTime: register.histogram<{routeId: string}>({
+        name: "vc_rest_api_client_request_time_seconds",
+        help: "Histogram of REST API client request time by routeId",
+        labelNames: ["routeId"],
+        // Expected times are ~ 50-500ms, but in an overload NodeJS they can be greater
+        buckets: [0.01, 0.1, 1, 2, 5],
+      }),
+
+      streamTime: register.histogram<{routeId: string}>({
+        name: "vc_rest_api_client_stream_time_seconds",
+        help: "Histogram of REST API client streaming time by routeId",
+        labelNames: ["routeId"],
+        // Expected times are ~ 50-500ms, but in an overload NodeJS they can be greater
+        buckets: [0.01, 0.1, 1, 2, 5],
+      }),
+
+      requestErrors: register.gauge<{routeId: string; baseUrl: string}>({
+        name: "vc_rest_api_client_request_errors_total",
+        help: "Total count of errors on REST API client requests by routeId",
+        labelNames: ["routeId", "baseUrl"],
+      }),
+
+      requestToFallbacks: register.gauge<{routeId: string; baseUrl: string}>({
+        name: "vc_rest_api_client_request_to_fallbacks_total",
+        help: "Total count of requests to fallback URLs on REST API by routeId",
+        labelNames: ["routeId", "baseUrl"],
+      }),
+
+      urlsScore: register.gauge<{urlIndex: number; baseUrl: string}>({
+        name: "vc_rest_api_client_urls_score",
+        help: "Current score of REST API URLs by url index",
+        labelNames: ["urlIndex", "baseUrl"],
+      }),
+    },
+
+    keymanagerApiRest: {
+      responseTime: register.histogram<{operationId: string}>({
+        name: "vc_keymanager_api_rest_response_time_seconds",
+        help: "REST API time to fulfill a request by operationId",
+        labelNames: ["operationId"],
+        // Request times range between 1ms to 100ms in normal conditions. Can get to 1-5 seconds if overloaded
+        buckets: [0.01, 0.1, 1],
+      }),
+      requests: register.gauge<{operationId: string}>({
+        name: "vc_keymanager_api_rest_requests_total",
+        help: "REST API total count requests by operationId",
+        labelNames: ["operationId"],
+      }),
+      errors: register.gauge<{operationId: string}>({
+        name: "vc_keymanager_api_rest_errors_total",
+        help: "REST API total count of errors by operationId",
+        labelNames: ["operationId"],
+      }),
+      // Metrics for HttpActiveSocketsTracker, defined there
+      activeSockets: register.gauge({
+        name: "vc_keymanager_api_rest_active_sockets_count",
+        help: "REST API current count of active sockets",
+      }),
+      socketsBytesRead: register.gauge({
+        name: "vc_keymanager_api_rest_sockets_bytes_read_total",
+        help: "REST API total count of bytes read on all sockets",
+      }),
+      socketsBytesWritten: register.gauge({
+        name: "vc_keymanager_api_rest_sockets_bytes_written_total",
+        help: "REST API total count of bytes written on all sockets",
+      }),
+    },
+
+    db: {
+      dbReadReq: register.gauge<{bucket: string}>({
+        name: "vc_db_read_req_total",
+        help: "Total count of db read requests, may read 0 or more items",
+        labelNames: ["bucket"],
+      }),
+      dbReadItems: register.gauge<{bucket: string}>({
+        name: "vc_db_read_items_total",
+        help: "Total count of db read items, item = key | value | entry",
+        labelNames: ["bucket"],
+      }),
+      dbWriteReq: register.gauge<{bucket: string}>({
+        name: "vc_db_write_req_total",
+        help: "Total count of db write requests, may write 0 or more items",
+        labelNames: ["bucket"],
+      }),
+      dbWriteItems: register.gauge<{bucket: string}>({
+        name: "vc_db_write_items_total",
+        help: "Total count of db write items",
+        labelNames: ["bucket"],
+      }),
+      dbSizeTotal: register.gauge({
+        name: "vc_db_size_bytes_total",
+        help: "Approximate number of bytes of file system space used by db",
+      }),
+      dbApproximateSizeTime: register.histogram({
+        name: "vc_db_approximate_size_time_seconds",
+        help: "Time to approximate db size in seconds",
+        buckets: [0.0001, 0.001, 0.01, 0.1, 1],
+      }),
+    },
+
+    doppelganger: {
+      statusCount: register.gauge<{status: string}>({
+        name: "vc_doppelganger_validator_status_count",
+        help: "Count of validators per status",
+        labelNames: ["status"],
+      }),
+      epochsChecked: register.gauge({
+        name: "vc_doppelganger_epochs_checked_total",
+        help: "Total count of epochs checked",
+      }),
+    },
+  };
+}
+
+export function renderEnumNumeric(obj: Record<string, unknown>): string {
+  const out: string[] = [];
+
+  for (const [key, value] of Object.entries(obj)) {
+    if (typeof value === "number") {
+      out.push(`${key}=${value}`);
+    }
+  }
+
+  return out.join(", ");
+}

package/src/repositories/index.ts

@@ -0,0 +1 @@
+export * from "./metaDataRepository.js";

package/src/repositories/metaDataRepository.ts

@@ -0,0 +1,42 @@
+import {DbReqOpts, encodeKey} from "@lodestar/db";
+import {Root, UintNum64, ssz} from "@lodestar/types";
+import {Bucket, getBucketNameByValue} from "../buckets.js";
+import {LodestarValidatorDatabaseController} from "../types.js";
+
+const GENESIS_VALIDATORS_ROOT = Buffer.from("GENESIS_VALIDATORS_ROOT");
+const GENESIS_TIME = Buffer.from("GENESIS_TIME");
+
+/**
+ * Store MetaData of validator.
+ */
+export class MetaDataRepository {
+  protected bucket = Bucket.validator_metaData;
+
+  private readonly bucketId = getBucketNameByValue(this.bucket);
+  private readonly dbReqOpts: DbReqOpts = {bucketId: this.bucketId};
+
+  constructor(protected db: LodestarValidatorDatabaseController) {
+    this.dbReqOpts = {bucketId: this.bucketId};
+  }
+
+  async getGenesisValidatorsRoot(): Promise<Root | null> {
+    return this.db.get(this.encodeKey(GENESIS_VALIDATORS_ROOT), this.dbReqOpts);
+  }
+
+  async setGenesisValidatorsRoot(genesisValidatorsRoot: Root): Promise<void> {
+    await this.db.put(this.encodeKey(GENESIS_VALIDATORS_ROOT), genesisValidatorsRoot, this.dbReqOpts);
+  }
+
+  async getGenesisTime(): Promise<UintNum64 | null> {
+    const bytes = await this.db.get(this.encodeKey(GENESIS_TIME), this.dbReqOpts);
+    return bytes ? ssz.UintNum64.deserialize(bytes) : null;
+  }
+
+  async setGenesisTime(genesisTime: UintNum64): Promise<void> {
+    await this.db.put(this.encodeKey(GENESIS_TIME), ssz.UintNum64.serialize(genesisTime), this.dbReqOpts);
+  }
+
+  private encodeKey(key: Uint8Array): Uint8Array {
+    return encodeKey(this.bucket, key);
+  }
+}