@lodestar/beacon-node 1.50.0-dev.b4eaa62b30 → 1.50.0-dev.b726a05a4d

package/src/metrics/metrics/lodestar.ts

@@ -20,6 +20,7 @@ import {InsertOutcome} from "../../chain/opPools/types.js";
 import {RegenCaller, RegenFnName} from "../../chain/regen/interface.js";
 import {ReprocessStatus} from "../../chain/reprocess.js";
 import {RejectReason} from "../../chain/seenCache/seenAttestationData.js";
+import type {ShufflingPromiseCancelReason} from "../../chain/shufflingCache.js";
 import {CacheItemType} from "../../chain/stateCache/types.js";
 import {OpSource} from "../../chain/validatorMonitor.js";
 import type {FlatFileStoreOperation} from "../../db/flatFileStore/metrics.js";
@@ -1176,6 +1177,11 @@ export function createLodestarMetrics(
       help: "The total result of calling notifyForkchoiceUpdate execution engine api",
       labelNames: ["result"],
     }),
+    engineApiRequests: register.counter<{transport: string}>({
+      name: "lodestar_execution_engine_api_requests_total",
+      help: "Total engine API requests by transport",
+      labelNames: ["transport"],
+    }),
     opPool: {
       aggregatedAttestationPool: {
         size: register.gauge({
@@ -1281,6 +1287,10 @@ export function createLodestarMetrics(
             name: "lodestar_oppool_aggregated_attestation_pool_packed_attestations_seen_committees_total",
             help: "Total number of committees for which all members are seen when producing packed attestations",
           }),
+          missingShufflingCommittees: register.gauge({
+            name: "lodestar_oppool_aggregated_attestation_pool_packed_attestations_missing_shuffling_committees_total",
+            help: "Total number of committees skipped due to missing shuffling when producing packed attestations",
+          }),
         },
       },
       attestationPool: {
@@ -1614,6 +1624,19 @@ export function createLodestarMetrics(
         name: "lodestar_shuffling_cache_size",
         help: "Shuffling cache size",
       }),
+      epochs: register.gauge({
+        name: "lodestar_shuffling_cache_epochs",
+        help: "Number of epochs in shuffling cache",
+      }),
+      prunedShufflings: register.counter({
+        name: "lodestar_shuffling_cache_pruned_shufflings_total",
+        help: "Total number of shufflings pruned from shuffling cache",
+      }),
+      cancelledPromises: register.counter<{reason: ShufflingPromiseCancelReason}>({
+        name: "lodestar_shuffling_cache_cancelled_promises_total",
+        help: "Total number of shuffling promises resolved with null",
+        labelNames: ["reason"],
+      }),
       insertPromiseCount: register.gauge({
         name: "lodestar_shuffling_cache_insert_promise_count",
         help: "Total number of times insertPromise is called",

package/src/metrics/metrics.ts

@@ -9,27 +9,42 @@ import {RegistryMetricCreator} from "./utils/registryMetricCreator.js";
 
 export type Metrics = BeaconMetrics &
   ForkChoiceMetrics &
-  BeaconStateTransitionMetrics &
-  LodestarMetrics & {register: RegistryMetricCreator; close: () => void};
+  LodestarMetrics & {
+    /** Null when we use native state transition metrics instead */
+    stateTransition: BeaconStateTransitionMetrics | null;
+    register: RegistryMetricCreator;
+    close: () => void;
+  };
+
+export type CreateMetricsOptions = {
+  /*
+   * Toggle inclusion of state transition metrics.
+   *
+   * False when native state transition is used, so we
+   * can grab metrics from the native implementation instead.
+   * */
+  includeStateTransitionMetrics?: boolean;
+  collectNodeMetrics?: boolean;
+};
 
 export function createMetrics(
   opts: MetricsOptions,
   genesisTime: number,
   externalRegistries: Registry[] = [],
-  {collectNodeMetrics = true}: {collectNodeMetrics?: boolean} = {}
+  createOpts: CreateMetricsOptions = {}
 ): Metrics {
   const register = new RegistryMetricCreator();
   const beacon = createBeaconMetrics(register);
   const forkChoice = getForkChoiceMetrics(register);
   const lodestar = createLodestarMetrics(register, opts.metadata, genesisTime);
-  const stateTransition = getMetrics(register);
+  const stateTransition = createOpts.includeStateTransitionMetrics === false ? null : getMetrics(register);
 
   const onUnhandledRejection = (_error: unknown): void => {
     lodestar.unhandledPromiseRejections.inc();
   };
   process.on("unhandledRejection", onUnhandledRejection);
 
-  const nodeJsMetricsClose = collectNodeMetrics ? collectNodeJSMetrics(register) : (): void => {};
+  const nodeJsMetricsClose = createOpts.collectNodeMetrics === false ? (): void => {} : collectNodeJSMetrics(register);
   const close = (): void => {
     process.removeListener("unhandledRejection", onUnhandledRejection);
     nodeJsMetricsClose();
@@ -46,7 +61,7 @@ export function createMetrics(
     ...beacon,
     ...forkChoice,
     ...lodestar,
-    ...stateTransition,
+    stateTransition,
     register,
     close,
   };

package/src/network/core/events.ts

@@ -1,21 +1,30 @@
 import EventEmitter from "node:events";
 import {ResponseIncoming, ResponseOutgoing} from "@lodestar/reqresp";
-import {AsyncIterableEventBus, IteratorEvent, RequestEvent} from "../../util/asyncIterableToEvents.js";
+import {
+  AsyncIterableEventBus,
+  IteratorEvent,
+  RequestCancelEvent,
+  RequestEvent,
+} from "../../util/asyncIterableToEvents.js";
 import {StrictEventEmitterSingleArg} from "../../util/strictEvents.js";
 import {EventDirection} from "../events.js";
 import {IncomingRequestArgs, OutgoingRequestArgs} from "../reqresp/types.js";
 
 export enum ReqRespBridgeEvent {
   outgoingRequest = "reqresp.outgoingRequest",
+  outgoingRequestCancel = "reqresp.outgoingRequestCancel",
   outgoingResponse = "reqresp.outgoingResponse",
   incomingRequest = "reqresp.incomingRequest",
+  incomingRequestCancel = "reqresp.incomingRequestCancel",
   incomingResponse = "reqresp.incomingResponse",
 }
 
 export type ReqRespBridgeEventData = {
   [ReqRespBridgeEvent.outgoingRequest]: RequestEvent<OutgoingRequestArgs>;
+  [ReqRespBridgeEvent.outgoingRequestCancel]: RequestCancelEvent;
   [ReqRespBridgeEvent.outgoingResponse]: IteratorEvent<ResponseOutgoing>;
   [ReqRespBridgeEvent.incomingRequest]: RequestEvent<IncomingRequestArgs>;
+  [ReqRespBridgeEvent.incomingRequestCancel]: RequestCancelEvent;
   [ReqRespBridgeEvent.incomingResponse]: IteratorEvent<ResponseIncoming>;
 };
 
@@ -26,8 +35,10 @@ export class ReqRespBridgeEventBus extends (EventEmitter as {new (): IReqRespBri
 // NOTE: If the same event is on this two arrays it can create an infinite cycle
 export const reqRespBridgeEventDirection: Record<ReqRespBridgeEvent, EventDirection> = {
   [ReqRespBridgeEvent.outgoingRequest]: EventDirection.mainToWorker,
+  [ReqRespBridgeEvent.outgoingRequestCancel]: EventDirection.mainToWorker,
   [ReqRespBridgeEvent.outgoingResponse]: EventDirection.mainToWorker,
   [ReqRespBridgeEvent.incomingRequest]: EventDirection.workerToMain,
+  [ReqRespBridgeEvent.incomingRequestCancel]: EventDirection.workerToMain,
   [ReqRespBridgeEvent.incomingResponse]: EventDirection.workerToMain,
 };
 
@@ -36,8 +47,10 @@ export function getReqRespBridgeReqEvents(
 ): AsyncIterableEventBus<OutgoingRequestArgs, ResponseIncoming> {
   return {
     emitRequest: (data) => events.emit(ReqRespBridgeEvent.outgoingRequest, data),
+    emitRequestCancel: (data) => events.emit(ReqRespBridgeEvent.outgoingRequestCancel, data),
     emitResponse: (data) => events.emit(ReqRespBridgeEvent.incomingResponse, data),
     onRequest: (cb) => events.on(ReqRespBridgeEvent.outgoingRequest, cb),
+    onRequestCancel: (cb) => events.on(ReqRespBridgeEvent.outgoingRequestCancel, cb),
     onResponse: (cb) => events.on(ReqRespBridgeEvent.incomingResponse, cb),
   };
 }
@@ -47,8 +60,10 @@ export function getReqRespBridgeRespEvents(
 ): AsyncIterableEventBus<IncomingRequestArgs, ResponseOutgoing> {
   return {
     emitRequest: (data) => events.emit(ReqRespBridgeEvent.incomingRequest, data),
+    emitRequestCancel: (data) => events.emit(ReqRespBridgeEvent.incomingRequestCancel, data),
     emitResponse: (data) => events.emit(ReqRespBridgeEvent.outgoingResponse, data),
     onRequest: (cb) => events.on(ReqRespBridgeEvent.incomingRequest, cb),
+    onRequestCancel: (cb) => events.on(ReqRespBridgeEvent.incomingRequestCancel, cb),
     onResponse: (cb) => events.on(ReqRespBridgeEvent.outgoingResponse, cb),
   };
 }

package/src/network/metadata.ts

@@ -42,13 +42,13 @@ export class MetadataController {
   private onSetValue: (key: string, value: Uint8Array) => void;
   private networkConfig: NetworkConfig;
   private logger: Logger;
-  private _metadata: fulu.Metadata;
+  private metadata: fulu.Metadata;
 
   constructor(opts: MetadataOpts, modules: MetadataModules) {
     this.networkConfig = modules.networkConfig;
     this.logger = modules.logger;
     this.onSetValue = modules.onSetValue;
-    this._metadata = opts.metadata ?? {
+    this.metadata = opts.metadata ?? {
       ...ssz.fulu.Metadata.defaultValue(),
       custodyGroupCount: modules.networkConfig.custodyConfig.targetCustodyGroupCount,
     };
@@ -58,61 +58,61 @@ export class MetadataController {
     // updateEth2Field() MUST be called with clock epoch
     this.updateEth2Field(currentEpoch);
 
-    this.onSetValue(ENRKey.attnets, ssz.phase0.AttestationSubnets.serialize(this._metadata.attnets));
+    this.onSetValue(ENRKey.attnets, ssz.phase0.AttestationSubnets.serialize(this.metadata.attnets));
 
     const config = this.networkConfig.config;
 
     if (config.getForkSeq(computeStartSlotAtEpoch(currentEpoch)) >= ForkSeq.altair) {
       // Only persist syncnets if altair fork is already activated. If currentFork is altair but head is phase0
       // adding syncnets to the ENR is not a problem, we will just have a useless field for a few hours.
-      this.onSetValue(ENRKey.syncnets, ssz.altair.SyncSubnets.serialize(this._metadata.syncnets));
+      this.onSetValue(ENRKey.syncnets, ssz.altair.SyncSubnets.serialize(this.metadata.syncnets));
     }
 
     // Set CGC regardless of fork. It may be useful to clients before Fulu, and will be ignored otherwise.
-    this.onSetValue(ENRKey.cgc, serializeCgc(this._metadata.custodyGroupCount));
+    this.onSetValue(ENRKey.cgc, serializeCgc(this.metadata.custodyGroupCount));
   }
 
   get seqNumber(): bigint {
-    return this._metadata.seqNumber;
+    return this.metadata.seqNumber;
   }
 
   get syncnets(): BitArray {
-    return this._metadata.syncnets;
+    return this.metadata.syncnets;
   }
 
   set syncnets(syncnets: BitArray) {
     this.onSetValue(ENRKey.syncnets, ssz.altair.SyncSubnets.serialize(syncnets));
-    this._metadata.seqNumber++;
-    this._metadata.syncnets = syncnets;
+    this.metadata.seqNumber++;
+    this.metadata.syncnets = syncnets;
   }
 
   get attnets(): BitArray {
-    return this._metadata.attnets;
+    return this.metadata.attnets;
   }
 
   set attnets(attnets: BitArray) {
     this.onSetValue(ENRKey.attnets, ssz.phase0.AttestationSubnets.serialize(attnets));
-    this._metadata.seqNumber++;
-    this._metadata.attnets = attnets;
+    this.metadata.seqNumber++;
+    this.metadata.attnets = attnets;
   }
 
   get custodyGroupCount(): number {
-    return this._metadata.custodyGroupCount;
+    return this.metadata.custodyGroupCount;
   }
 
   set custodyGroupCount(custodyGroupCount: number) {
-    if (custodyGroupCount === this._metadata.custodyGroupCount) {
+    if (custodyGroupCount === this.metadata.custodyGroupCount) {
       return;
     }
     this.onSetValue(ENRKey.cgc, serializeCgc(custodyGroupCount));
     this.logger.debug("Updated cgc field in ENR", {custodyGroupCount});
-    this._metadata.seqNumber++;
-    this._metadata.custodyGroupCount = custodyGroupCount;
+    this.metadata.seqNumber++;
+    this.metadata.custodyGroupCount = custodyGroupCount;
   }
 
   /** Consumers that need the phase0.Metadata type can just ignore the .syncnets property */
   get json(): fulu.Metadata {
-    return this._metadata;
+    return this.metadata;
   }
 
   /**

package/src/network/peers/datastore.ts

@@ -16,23 +16,23 @@ type AwaitGenerator<T, TReturn = any, TNext = any> = Generator<T, TReturn, TNext
  * This has a memory issue because all peer data stays in memory and loaded at startup time
  * This is written for libp2p >=0.35, we maintain the same mechanism but with bounded data structure
  * This datastore includes a memory datastore and fallback to db datastore
- * Use an in-memory datastore with last accessed time and _maxMemoryItems, on start it's empty (lazy load)
+ * Use an in-memory datastore with last accessed time and maxMemoryItems, on start it's empty (lazy load)
  * - get: Search in-memory datastore first, if not found search from db.
  *     - If found from db, add back to the in-memory datastore
  *     - Update lastAccessedMs
- * - put: move oldest items from memory to db if there are more than _maxMemoryItems items in memory
- *     -  update memory datastore, only update db datastore if there are at least _threshold dirty items
+ * - put: move oldest items from memory to db if there are more than maxMemoryItems items in memory
+ *     -  update memory datastore, only update db datastore if there are at least threshold dirty items
  *     -  Update lastAccessedMs
  */
 export class Eth2PeerDataStore extends BaseDatastore {
-  private _dbDatastore: LevelDatastore;
-  private _memoryDatastore: Map<string, MemoryItem>;
+  private dbDatastore: LevelDatastore;
+  private memoryDatastore: Map<string, MemoryItem>;
   /** Same to PersistentPeerStore of the old libp2p implementation */
-  private _dirtyItems = new Set<string>();
+  private dirtyItems = new Set<string>();
   /** If there are more dirty items than threshold, commit data to db */
-  private _threshold: number;
+  private threshold: number;
   /** If there are more memory items than this, prune oldest ones from memory and move to db */
-  private _maxMemoryItems: number;
+  private maxMemoryItems: number;
 
   constructor(
     dbDatastore: LevelDatastore | string,
@@ -47,18 +47,18 @@ export class Eth2PeerDataStore extends BaseDatastore {
       throw Error(`Threshold ${threshold} should be at most maxMemoryItems ${maxMemoryItems}`);
     }
 
-    this._dbDatastore = typeof dbDatastore === "string" ? new LevelDatastore(dbDatastore) : dbDatastore;
-    this._memoryDatastore = new Map();
-    this._threshold = threshold;
-    this._maxMemoryItems = maxMemoryItems;
+    this.dbDatastore = typeof dbDatastore === "string" ? new LevelDatastore(dbDatastore) : dbDatastore;
+    this.memoryDatastore = new Map();
+    this.threshold = threshold;
+    this.maxMemoryItems = maxMemoryItems;
   }
 
   async open(): Promise<void> {
-    return this._dbDatastore.open();
+    return this.dbDatastore.open();
   }
 
   async close(): Promise<void> {
-    return this._dbDatastore.close();
+    return this.dbDatastore.close();
   }
 
   async put(key: Key, val: Uint8Array, _options?: AbortOptions): Promise<Key> {
@@ -67,23 +67,23 @@ export class Eth2PeerDataStore extends BaseDatastore {
 
   /**
    * Same interface to put with "fromDb" option, if this item is updated back from db
-   * Move oldest items from memory data store to db if it's over this._maxMemoryItems
+   * Move oldest items from memory data store to db if it's over this.maxMemoryItems
    */
   async _put(key: Key, val: Uint8Array, fromDb = false): Promise<Key> {
-    while (this._memoryDatastore.size >= this._maxMemoryItems) {
+    while (this.memoryDatastore.size >= this.maxMemoryItems) {
       // it's likely this is called only 1 time
       await this.pruneMemoryDatastore();
     }
 
     const keyStr = key.toString();
-    const memoryItem = this._memoryDatastore.get(keyStr);
+    const memoryItem = this.memoryDatastore.get(keyStr);
     if (memoryItem) {
       // update existing
       memoryItem.lastAccessedMs = Date.now();
       memoryItem.data = val;
     } else {
       // new
-      this._memoryDatastore.set(keyStr, {data: val, lastAccessedMs: Date.now()});
+      this.memoryDatastore.set(keyStr, {data: val, lastAccessedMs: Date.now()});
     }
 
     if (!fromDb) await this._addDirtyItem(keyStr);
@@ -98,15 +98,15 @@ export class Eth2PeerDataStore extends BaseDatastore {
    */
   async get(key: Key, options?: AbortOptions): Promise<Uint8Array> {
     const keyStr = key.toString();
-    const memoryItem = this._memoryDatastore.get(keyStr);
+    const memoryItem = this.memoryDatastore.get(keyStr);
     if (memoryItem) {
       memoryItem.lastAccessedMs = Date.now();
       return memoryItem.data;
     }
 
     // this throws error if not found
-    const dbValue = await this._dbDatastore.get(key, options);
-    // don't call this._memoryDatastore.set directly
+    const dbValue = await this.dbDatastore.get(key, options);
+    // don't call this.memoryDatastore.set directly
     // we want to get through prune() logic with fromDb as true
     await this._put(key, dbValue, true);
     return dbValue;
@@ -125,30 +125,30 @@ export class Eth2PeerDataStore extends BaseDatastore {
   }
 
   async delete(key: Key, options?: AbortOptions): Promise<void> {
-    this._memoryDatastore.delete(key.toString());
-    await this._dbDatastore.delete(key, options);
+    this.memoryDatastore.delete(key.toString());
+    await this.dbDatastore.delete(key, options);
   }
 
   async *_all(q: Query, options?: AbortOptions): AwaitGenerator<Pair> {
-    for (const [key, value] of this._memoryDatastore.entries()) {
+    for (const [key, value] of this.memoryDatastore.entries()) {
       yield {
         key: new Key(key),
         value: value.data,
       };
     }
-    yield* this._dbDatastore.query(q, options);
+    yield* this.dbDatastore.query(q, options);
   }
 
   async *_allKeys(q: KeyQuery, options?: AbortOptions): AwaitGenerator<Key> {
-    for (const key of this._memoryDatastore.keys()) {
+    for (const key of this.memoryDatastore.keys()) {
       yield new Key(key);
     }
-    yield* this._dbDatastore.queryKeys(q, options);
+    yield* this.dbDatastore.queryKeys(q, options);
   }
 
   private async _addDirtyItem(keyStr: string): Promise<void> {
-    this._dirtyItems.add(keyStr);
-    if (this._dirtyItems.size >= this._threshold) {
+    this.dirtyItems.add(keyStr);
+    if (this.dirtyItems.size >= this.threshold) {
       try {
         await this._commitData();
       } catch (_e) {}
@@ -156,15 +156,15 @@ export class Eth2PeerDataStore extends BaseDatastore {
   }
 
   private async _commitData(): Promise<void> {
-    const batch = this._dbDatastore.batch();
-    for (const keyStr of this._dirtyItems) {
-      const memoryItem = this._memoryDatastore.get(keyStr);
+    const batch = this.dbDatastore.batch();
+    for (const keyStr of this.dirtyItems) {
+      const memoryItem = this.memoryDatastore.get(keyStr);
       if (memoryItem) {
         batch.put(new Key(keyStr), memoryItem.data);
       }
     }
     await batch.commit();
-    this._dirtyItems.clear();
+    this.dirtyItems.clear();
   }
 
   /**
@@ -175,7 +175,7 @@ export class Eth2PeerDataStore extends BaseDatastore {
     let oldestKey: string | undefined = undefined;
     let oldestValue: Uint8Array | undefined = undefined;
 
-    for (const [key, value] of this._memoryDatastore) {
+    for (const [key, value] of this.memoryDatastore) {
       if (value.lastAccessedMs < oldestAccessedMs) {
         oldestAccessedMs = value.lastAccessedMs;
         oldestKey = key;
@@ -184,8 +184,8 @@ export class Eth2PeerDataStore extends BaseDatastore {
     }
 
     if (oldestKey && oldestValue) {
-      await this._dbDatastore.put(new Key(oldestKey), oldestValue);
-      this._memoryDatastore.delete(oldestKey);
+      await this.dbDatastore.put(new Key(oldestKey), oldestValue);
+      this.memoryDatastore.delete(oldestKey);
     }
   }
 }