@lodestar/beacon-node 1.35.0-dev.98d359db41 → 1.35.0-dev.b42a298a7c

package/src/monitoring/properties.ts

@@ -1,152 +0,0 @@
-import {MetricObjectWithValues, MetricValue as MetricValueObject, Registry} from "prom-client";
-import {JsonRecord, JsonType, MetricValue, RecordValue} from "./types.js";
-
-type PropertyDefinition = {
-  /** Key of value to be sent to remote service */
-  jsonKey: string;
-  /** Description of the property */
-  description?: string;
-};
-
-type StaticPropertyDefinition<T extends RecordValue> = PropertyDefinition & {
-  /** Static value */
-  value: T;
-};
-
-type DynamicPropertyDefinition<T extends RecordValue> = PropertyDefinition & {
-  /** Value provider function */
-  provider: () => T;
-};
-
-type MetricPropertyDefinition<T extends RecordValue> = PropertyDefinition & {
-  /** Type of value to be sent to remote service */
-  jsonType: JsonType;
-  /** Name of the metric */
-  metricName: string;
-  /** Get value from metric with label */
-  withLabel?: {name: string; value: string};
-  /** Get value from label instead of metric value */
-  fromLabel?: string;
-  /** Range value to evaluate to true */
-  rangeValue?: number;
-  /** Evaluate to true if value is greater than or equal to threshold */
-  threshold?: number;
-  /** Function to format retrieved metric value */
-  formatter?: (value: MetricValue) => MetricValue;
-  /** Only fetch metric once and then use cached value */
-  cacheResult?: boolean;
-  /** Default value if metric does not exist */
-  defaultValue: T;
-};
-
-/**
- * Interface to be implemented by client stats properties
- */
-export interface ClientStatsProperty<T extends RecordValue> {
-  readonly definition: PropertyDefinition;
-
-  getRecord(register: Registry): JsonRecord<T> | Promise<JsonRecord<T>>;
-}
-
-/**
- * Static property that can be used to define hard-coded values
- */
-export class StaticProperty<T extends RecordValue> implements ClientStatsProperty<T> {
-  constructor(readonly definition: StaticPropertyDefinition<T>) {}
-
-  getRecord(): JsonRecord<T> {
-    return {key: this.definition.jsonKey, value: this.definition.value};
-  }
-}
-
-/**
- * Dynamic property that can be used to get value from a provider function
- */
-export class DynamicProperty<T extends RecordValue> implements ClientStatsProperty<T> {
-  constructor(readonly definition: DynamicPropertyDefinition<T>) {}
-
-  getRecord(): JsonRecord<T> {
-    return {key: this.definition.jsonKey, value: this.definition.provider()};
-  }
-}
-
-/**
- * Metric property that can be used to get value from an existing prometheus metric
- */
-export class MetricProperty<T extends RecordValue> implements ClientStatsProperty<T> {
-  private cachedValue?: T;
-
-  constructor(readonly definition: MetricPropertyDefinition<T>) {}
-
-  async getRecord(register: Registry): Promise<JsonRecord<T>> {
-    if (this.cachedValue != null) {
-      return {key: this.definition.jsonKey, value: this.cachedValue};
-    }
-
-    const metric = register.getSingleMetric(this.definition.metricName);
-
-    if (metric) {
-      const metricObject = await metric.get();
-
-      const metricValue = this.extractMetricValue(metricObject);
-
-      if (metricValue != null) {
-        const formattedValue = this.formatMetricValue(metricValue);
-
-        const typedValue = this.convertMetricValue(formattedValue) as T;
-
-        if (this.definition.cacheResult) {
-          this.cachedValue = typedValue;
-        }
-
-        return {key: this.definition.jsonKey, value: typedValue};
-      }
-    }
-
-    return {key: this.definition.jsonKey, value: this.definition.defaultValue};
-  }
-
-  private extractMetricValue(metricObject: MetricObjectWithValues<MetricValueObject<string>>): MetricValue | undefined {
-    const {withLabel, fromLabel} = this.definition;
-
-    if (withLabel) {
-      // get value from metric with specific label, e.g. protocol="global received"
-      return metricObject.values.find((v) => v.labels[withLabel.name] === withLabel.value)?.value;
-    }
-
-    if (fromLabel) {
-      // get value from label, e.g. lodestar_version{version="v1.3.0/2d0938e"} => v1.3.0/2d0938e
-      return metricObject.values[0].labels[fromLabel];
-    }
-
-    // metric value e.g. beacon_head_slot 5603174 => 5603174
-    return metricObject.values[0].value;
-  }
-
-  private formatMetricValue(value: MetricValue): MetricValue {
-    if (!this.definition.formatter) {
-      return value;
-    }
-    return this.definition.formatter(value);
-  }
-
-  private convertMetricValue(value: MetricValue): RecordValue {
-    if (typeof value === "number") {
-      switch (this.definition.jsonType) {
-        case JsonType.String:
-          return value.toString();
-        case JsonType.Number:
-          return Math.round(value);
-        case JsonType.Boolean:
-          if (this.definition.rangeValue != null) {
-            return value === this.definition.rangeValue;
-          }
-          if (this.definition.threshold != null) {
-            return value >= this.definition.threshold;
-          }
-          return value > 0;
-      }
-    }
-    return value;
-  }
-}

package/src/monitoring/service.ts

@@ -1,235 +0,0 @@
-import {Registry} from "prom-client";
-import {ErrorAborted, Histogram, Logger, TimeoutError, fetch} from "@lodestar/utils";
-import {RegistryMetricCreator} from "../metrics/index.js";
-import {createClientStats} from "./clientStats.js";
-import {MonitoringOptions, defaultMonitoringOptions} from "./options.js";
-import system from "./system.js";
-import {ClientStats} from "./types.js";
-
-type MonitoringData = Record<string, string | number | boolean>;
-
-export type RemoteServiceError = {
-  status: string;
-  data: null;
-};
-
-enum FetchAbortReason {
-  Close = "close",
-  Timeout = "timeout",
-}
-
-enum Status {
-  Started = "started",
-  Closed = "closed",
-}
-
-enum SendDataStatus {
-  Success = "success",
-  Error = "error",
-}
-
-export type Client = "beacon" | "validator";
-
-/**
- * Service for sending clients stats to a remote service (e.g. beaconcha.in)
- */
-export class MonitoringService {
-  private readonly clientStats: ClientStats[];
-  private readonly remoteServiceUrl: URL;
-  private readonly remoteServiceHost: string;
-  private readonly options: Required<MonitoringOptions>;
-  private readonly register: Registry;
-  private readonly logger: Logger;
-
-  private readonly collectDataMetric: Histogram;
-  private readonly sendDataMetric: Histogram<{status: SendDataStatus}>;
-
-  private status = Status.Started;
-  private initialDelayTimeout?: NodeJS.Timeout;
-  private monitoringInterval?: NodeJS.Timeout;
-  private fetchAbortController?: AbortController;
-  private pendingRequest?: Promise<void>;
-
-  constructor(
-    client: Client,
-    options: Required<Pick<MonitoringOptions, "endpoint">> & MonitoringOptions,
-    {register, logger}: {register: RegistryMetricCreator; logger: Logger}
-  ) {
-    this.options = {...defaultMonitoringOptions, ...options};
-    this.logger = logger;
-    this.register = register;
-    this.remoteServiceUrl = this.parseMonitoringEndpoint(this.options.endpoint);
-    this.remoteServiceHost = this.remoteServiceUrl.host;
-    this.clientStats = createClientStats(client, this.options.collectSystemStats);
-
-    this.collectDataMetric = register.histogram({
-      name: "lodestar_monitoring_collect_data_seconds",
-      help: "Time spent to collect monitoring data in seconds",
-      buckets: [0.001, 0.01, 0.1, 1, 5],
-    });
-
-    this.sendDataMetric = register.histogram({
-      name: "lodestar_monitoring_send_data_seconds",
-      help: "Time spent to send monitoring data to remote service in seconds",
-      labelNames: ["status"],
-      buckets: [0.3, 0.5, 1, Math.floor(this.options.requestTimeout / 1000)],
-    });
-
-    this.initialDelayTimeout = setTimeout(async () => {
-      await this.send();
-      this.nextMonitoringInterval();
-    }, this.options.initialDelay);
-
-    this.logger.info("Started monitoring service", {
-      // do not log full URL as it may contain secrets
-      remote: this.remoteServiceHost,
-      machine: this.remoteServiceUrl.searchParams.get("machine"),
-      interval: this.options.interval,
-    });
-  }
-
-  /**
-   * Stop sending client stats
-   */
-  close(): void {
-    if (this.status === Status.Closed) return;
-    this.status = Status.Closed;
-
-    if (this.initialDelayTimeout) {
-      clearTimeout(this.initialDelayTimeout);
-    }
-    if (this.monitoringInterval) {
-      clearTimeout(this.monitoringInterval);
-    }
-    if (this.pendingRequest) {
-      this.fetchAbortController?.abort(FetchAbortReason.Close);
-    }
-  }
-
-  /**
-   * Collect and send client stats
-   */
-  async send(): Promise<void> {
-    if (!this.pendingRequest) {
-      this.pendingRequest = (async () => {
-        try {
-          const data = await this.collectData();
-
-          const res = await this.sendData(data);
-
-          if (!res.ok) {
-            const error = (await res.json()) as RemoteServiceError;
-            throw new Error(error.status);
-          }
-
-          this.logger.debug(`Sent client stats to ${this.remoteServiceHost}`, {data: JSON.stringify(data)});
-        } catch (e) {
-          this.logger.warn(`Failed to send client stats to ${this.remoteServiceHost}`, {reason: (e as Error).message});
-        } finally {
-          this.pendingRequest = undefined;
-        }
-      })();
-    }
-
-    await this.pendingRequest;
-  }
-
-  private async collectData(): Promise<MonitoringData[]> {
-    const timer = this.collectDataMetric.startTimer();
-    const data: MonitoringData[] = [];
-    const recordPromises = [];
-
-    if (this.options.collectSystemStats) {
-      await system.collectData(this.logger);
-    }
-
-    for (const [i, s] of this.clientStats.entries()) {
-      data[i] = {};
-
-      recordPromises.push(
-        ...Object.values(s).map(async (property) => {
-          const record = await property.getRecord(this.register);
-          data[i][record.key] = record.value;
-        })
-      );
-    }
-
-    await Promise.all(recordPromises).finally(timer);
-
-    return data;
-  }
-
-  private async sendData(data: MonitoringData[]): Promise<Response> {
-    const timer = this.sendDataMetric.startTimer();
-    this.fetchAbortController = new AbortController();
-
-    const timeout = setTimeout(
-      () => this.fetchAbortController?.abort(FetchAbortReason.Timeout),
-      this.options.requestTimeout
-    );
-
-    let res: Response | undefined;
-
-    try {
-      res = await fetch(this.remoteServiceUrl, {
-        method: "POST",
-        headers: {"Content-Type": "application/json"},
-        body: JSON.stringify(data),
-        signal: this.fetchAbortController.signal,
-      });
-
-      return res;
-    } catch (e) {
-      const {signal} = this.fetchAbortController;
-
-      if (!signal.aborted) {
-        // error was thrown by fetch
-        throw e;
-      }
-
-      // error was thrown by abort signal
-      if (signal.reason === FetchAbortReason.Close) {
-        throw new ErrorAborted("request");
-      }
-      if (signal.reason === FetchAbortReason.Timeout) {
-        throw new TimeoutError("request");
-      }
-      throw e;
-    } finally {
-      timer({status: res?.ok ? SendDataStatus.Success : SendDataStatus.Error});
-      clearTimeout(timeout);
-    }
-  }
-
-  private nextMonitoringInterval(): void {
-    if (this.status === Status.Closed) return;
-    // ensure next interval only starts after previous request has finished
-    // else we might send next request too early and run into rate limit errors
-    this.monitoringInterval = setTimeout(async () => {
-      await this.send();
-      this.nextMonitoringInterval();
-    }, this.options.interval);
-  }
-
-  private parseMonitoringEndpoint(endpoint: string): URL {
-    if (!endpoint) {
-      throw new Error(`Monitoring endpoint is empty or undefined: ${endpoint}`);
-    }
-
-    try {
-      const url = new URL(endpoint);
-
-      if (url.protocol === "http:") {
-        this.logger.warn(
-          "Insecure monitoring endpoint, please make sure to always use a HTTPS connection in production"
-        );
-      } else if (url.protocol !== "https:") {
-        throw new Error();
-      }
-
-      return url;
-    } catch (_e) {
-      throw new Error(`Monitoring endpoint must be a valid URL: ${endpoint}`);
-    }
-  }
-}

package/src/monitoring/system.ts

@@ -1,146 +0,0 @@
-// We want to keep `system` export to be used as namespace
-import os from "node:os";
-import path from "node:path";
-// We want to keep `system` export as it's more readable and easier to understand
-import system from "systeminformation";
-import {Logger} from "@lodestar/utils";
-
-type MiscOs = "lin" | "win" | "mac" | "unk";
-
-/**
- * Singleton class to collect and provide system information
- */
-class System {
-  // static data only needs to be collected once
-  private staticDataCollected = false;
-  // disk I/O is not measurable in some environments
-  private diskIOMeasurable = true;
-
-  cpuCores = 0;
-  cpuThreads = 0;
-  cpuNodeSystemSecondsTotal = 0;
-  cpuNodeUserSecondsTotal = 0;
-  // Note: CPU I/O wait is not measured by os.cpus()
-  cpuNodeIOWaitSecondsTotal = 0;
-  cpuNodeIdleSecondsTotal = 0;
-  memoryNodeBytesTotal = 0;
-  memoryNodeBytesFree = 0;
-  memoryNodeBytesCached = 0;
-  memoryNodeBytesBuffers = 0;
-  diskNodeBytesTotal = 0;
-  diskNodeBytesFree = 0;
-  // Note: disk I/O seconds is currently unused by beaconcha.in
-  diskNodeIOSeconds = 0;
-  diskNodeReadsTotal = 0;
-  diskNodeWritesTotal = 0;
-  networkNodeBytesTotalReceive = 0;
-  networkNodeBytesTotalTransmit = 0;
-  miscNodeBootTsSeconds = 0;
-  miscOs: MiscOs = "unk";
-
-  /**
-   * Collect system data and update cached values
-   */
-  async collectData(logger: Logger): Promise<void> {
-    const debug = (dataType: string, e: Error): void => logger.debug(`Failed to collect ${dataType} data`, {}, e);
-
-    await Promise.all([
-      this.collectStaticData().catch((e) => debug("static system", e)),
-      this.collectCpuData().catch((e) => debug("CPU", e)),
-      this.collectMemoryData().catch((e) => debug("memory", e)),
-      this.collectDiskData().catch((e) => debug("disk", e)),
-      this.collectNetworkData().catch((e) => debug("network", e)),
-    ]);
-
-    this.miscNodeBootTsSeconds = this.getSystemBootTime();
-  }
-
-  private async collectStaticData(): Promise<void> {
-    if (this.staticDataCollected) return;
-
-    const cpu = await system.cpu();
-    // Note: inside container this might be inaccurate as
-    // physicalCores in some cases is the count of logical CPU cores
-    this.cpuCores = cpu.physicalCores;
-    this.cpuThreads = cpu.cores;
-
-    this.miscOs = this.getNormalizedOsVersion();
-
-    this.staticDataCollected = true;
-  }
-
-  private async collectCpuData(): Promise<void> {
-    const cpuTimes: Record<string, number> = {};
-
-    for (const cpu of os.cpus()) {
-      // sum up CPU times per mode and convert to seconds
-      for (const [mode, time] of Object.entries(cpu.times)) {
-        if (cpuTimes[mode] == null) cpuTimes[mode] = 0;
-        cpuTimes[mode] += Math.floor(time / 1000);
-      }
-    }
-
-    // Note: currently beaconcha.in expects system CPU seconds to be everything
-    this.cpuNodeSystemSecondsTotal = Object.values(cpuTimes).reduce((total, time) => total + time, 0);
-    this.cpuNodeUserSecondsTotal = cpuTimes.user;
-    this.cpuNodeIdleSecondsTotal = cpuTimes.idle;
-  }
-
-  private async collectMemoryData(): Promise<void> {
-    const memory = await system.mem();
-    this.memoryNodeBytesTotal = memory.total;
-    this.memoryNodeBytesFree = memory.free;
-    this.memoryNodeBytesCached = memory.cached;
-    this.memoryNodeBytesBuffers = memory.buffers;
-  }
-
-  private async collectDiskData(): Promise<void> {
-    const fileSystems = await system.fsSize();
-    // get file system root, on windows this is the name of the hard disk partition
-    const rootFs = process.platform === "win32" ? process.cwd().split(path.sep)[0] : "/";
-    // only consider root file system, if it does not exist use first entry in the list
-    const fileSystem = fileSystems.find((fs) => fs.mount === rootFs) ?? fileSystems[0];
-    this.diskNodeBytesTotal = fileSystem.size;
-    this.diskNodeBytesFree = fileSystem.available;
-
-    if (this.diskIOMeasurable) {
-      const disk = await system.disksIO();
-      if (disk != null && disk.rIO !== 0) {
-        // Note: rIO and wIO might not be available inside container
-        // see https://github.com/sebhildebrandt/systeminformation/issues/777
-        this.diskNodeReadsTotal = disk.rIO;
-        this.diskNodeWritesTotal = disk.wIO;
-      } else {
-        this.diskIOMeasurable = false;
-      }
-    }
-  }
-
-  private async collectNetworkData(): Promise<void> {
-    // defaults to first external network interface
-    const [network] = await system.networkStats();
-    // Note: rx_bytes and tx_bytes will be inaccurate if process
-    // runs inside container as it only captures local network traffic
-    this.networkNodeBytesTotalReceive = network.rx_bytes;
-    this.networkNodeBytesTotalTransmit = network.tx_bytes;
-  }
-
-  private getNormalizedOsVersion(): MiscOs {
-    switch (process.platform) {
-      case "linux":
-        return "lin";
-      case "darwin":
-        return "mac";
-      case "win32":
-        return "win";
-      default:
-        return "unk";
-    }
-  }
-
-  private getSystemBootTime(): number {
-    return Math.floor(Date.now() / 1000 - os.uptime());
-  }
-}
-
-export default new System();

package/src/monitoring/types.ts

@@ -1,21 +0,0 @@
-import {ClientStatsProperty} from "./properties.js";
-
-export type MetricValue = string | number;
-
-export type RecordValue = string | number | boolean;
-
-export type JsonRecord<T extends RecordValue> = {key: string; value: T};
-
-export type ClientStats = Record<string, ClientStatsProperty<RecordValue>>;
-
-export enum ProcessType {
-  BeaconNode = "beaconnode",
-  Validator = "validator",
-  System = "system",
-}
-
-export enum JsonType {
-  String,
-  Number,
-  Boolean,
-}

package/src/network/core/events.ts

@@ -1,59 +0,0 @@
-import EventEmitter from "node:events";
-import {ResponseIncoming, ResponseOutgoing} from "@lodestar/reqresp";
-import {AsyncIterableEventBus, IteratorEvent, RequestEvent} from "../../util/asyncIterableToEvents.js";
-import {StrictEventEmitterSingleArg} from "../../util/strictEvents.js";
-import {EventDirection} from "../../util/workerEvents.js";
-import {IncomingRequestArgs, OutgoingRequestArgs} from "../reqresp/types.js";
-
-export enum ReqRespBridgeEvent {
-  outgoingRequest = "reqresp.outgoingRequest",
-  outgoingResponse = "reqresp.outgoingResponse",
-  incomingRequest = "reqresp.incomingRequest",
-  incomingResponse = "reqresp.incomingResponse",
-}
-
-export type ReqRespBridgeEventData = {
-  [ReqRespBridgeEvent.outgoingRequest]: RequestEvent<OutgoingRequestArgs>;
-  [ReqRespBridgeEvent.outgoingResponse]: IteratorEvent<ResponseOutgoing>;
-  [ReqRespBridgeEvent.incomingRequest]: RequestEvent<IncomingRequestArgs>;
-  [ReqRespBridgeEvent.incomingResponse]: IteratorEvent<ResponseIncoming>;
-};
-
-type IReqRespBridgeEventBus = StrictEventEmitterSingleArg<ReqRespBridgeEventData>;
-
-export class ReqRespBridgeEventBus extends (EventEmitter as {new (): IReqRespBridgeEventBus}) {}
-
-// NOTE: If the same event is on this two arrays it can create an infinite cycle
-export const reqRespBridgeEventDirection: Record<ReqRespBridgeEvent, EventDirection> = {
-  [ReqRespBridgeEvent.outgoingRequest]: EventDirection.mainToWorker,
-  [ReqRespBridgeEvent.outgoingResponse]: EventDirection.mainToWorker,
-  [ReqRespBridgeEvent.incomingRequest]: EventDirection.workerToMain,
-  [ReqRespBridgeEvent.incomingResponse]: EventDirection.workerToMain,
-};
-
-export function getReqRespBridgeReqEvents(
-  events: IReqRespBridgeEventBus
-): AsyncIterableEventBus<OutgoingRequestArgs, ResponseIncoming> {
-  return {
-    emitRequest: (data) => events.emit(ReqRespBridgeEvent.outgoingRequest, data),
-    emitResponse: (data) => events.emit(ReqRespBridgeEvent.incomingResponse, data),
-    onRequest: (cb) => events.on(ReqRespBridgeEvent.outgoingRequest, cb),
-    onResponse: (cb) => events.on(ReqRespBridgeEvent.incomingResponse, cb),
-  };
-}
-
-export function getReqRespBridgeRespEvents(
-  events: IReqRespBridgeEventBus
-): AsyncIterableEventBus<IncomingRequestArgs, ResponseOutgoing> {
-  return {
-    emitRequest: (data) => events.emit(ReqRespBridgeEvent.incomingRequest, data),
-    emitResponse: (data) => events.emit(ReqRespBridgeEvent.outgoingResponse, data),
-    onRequest: (cb) => events.on(ReqRespBridgeEvent.incomingRequest, cb),
-    onResponse: (cb) => events.on(ReqRespBridgeEvent.outgoingResponse, cb),
-  };
-}
-
-export enum NetworkWorkerThreadEventType {
-  networkEvent = "networkEvent",
-  reqRespBridgeEvents = "reqRespBridgeEvents",
-}

package/src/network/core/index.ts

@@ -1,3 +0,0 @@
-export * from "./networkCore.js";
-export * from "./networkCoreWorkerHandler.js";
-export * from "./types.js";