@oneuptime/common 11.6.1 → 11.7.0

package/Server/Utils/Monitor/NetworkDeviceMetricUtil.ts

@@ -0,0 +1,343 @@
+import logger from "../Logger";
+import TelemetryUtil from "../Telemetry/Telemetry";
+import MetricService from "../../Services/MetricService";
+import GlobalConfigService from "../../Services/GlobalConfigService";
+import GlobalConfig from "../../../Models/DatabaseModels/GlobalConfig";
+import { MetricPointType } from "../../../Models/AnalyticsModels/Metric";
+import ServiceType from "../../../Types/Telemetry/ServiceType";
+import MetricType from "../../../Models/DatabaseModels/MetricType";
+import Dictionary from "../../../Types/Dictionary";
+import { JSONObject } from "../../../Types/JSON";
+import MonitorMetricType from "../../../Types/Monitor/MonitorMetricType";
+import SnmpInterface from "../../../Types/Monitor/SnmpMonitor/SnmpInterface";
+import SnmpMonitorResponse, {
+  SnmpOidResponse,
+} from "../../../Types/Monitor/SnmpMonitor/SnmpMonitorResponse";
+import ObjectID from "../../../Types/ObjectID";
+import OneUptimeDate from "../../../Types/Date";
+import CaptureSpan from "../Telemetry/CaptureSpan";
+
+/*
+ * Emits device-scoped metrics from each device walk — availability,
+ * response time, per-interface status/bandwidth/utilization/errors, and
+ * polled health-OID values. Series are keyed to the NetworkDevice
+ * (primaryEntityId = deviceId, attributes.networkDeviceId), NOT to a
+ * monitor, so the device pages can chart health for every registered
+ * device — including devices no monitor watches. Mirrors the SNMP sections
+ * of MonitorMetricUtil, which continue to emit monitor-scoped series for
+ * monitors that alert on the device.
+ */
+export default class NetworkDeviceMetricUtil {
+  /*
+   * Caps mirror MonitorMetricUtil: bound what a single walk can write —
+   * large routers can expose thousands of subinterfaces.
+   */
+  private static readonly maxInterfaceSeries: number = 200;
+  private static readonly maxOidSeries: number = 50;
+
+  // Retention handling mirrors MonitorMetricUtil (shared GlobalConfig knob).
+  private static readonly DEFAULT_RETENTION_DAYS: number = 30;
+  private static cachedRetentionDays: number | null = null;
+  private static lastCacheRefresh: Date | null = null;
+  private static readonly CACHE_TTL_MS: number = 5 * 60 * 1000;
+
+  private static async getRetentionDays(): Promise<number> {
+    const now: Date = OneUptimeDate.getCurrentDate();
+
+    if (
+      this.cachedRetentionDays !== null &&
+      this.lastCacheRefresh !== null &&
+      now.getTime() - this.lastCacheRefresh.getTime() < this.CACHE_TTL_MS
+    ) {
+      return this.cachedRetentionDays;
+    }
+
+    try {
+      const globalConfig: GlobalConfig | null =
+        await GlobalConfigService.findOneBy({
+          query: {
+            _id: ObjectID.getZeroObjectID().toString(),
+          },
+          props: {
+            isRoot: true,
+          },
+          select: {
+            monitorMetricRetentionInDays: true,
+          },
+        });
+
+      if (
+        globalConfig &&
+        globalConfig.monitorMetricRetentionInDays !== undefined &&
+        globalConfig.monitorMetricRetentionInDays !== null
+      ) {
+        this.cachedRetentionDays = globalConfig.monitorMetricRetentionInDays;
+      } else {
+        this.cachedRetentionDays = this.DEFAULT_RETENTION_DAYS;
+      }
+    } catch (err) {
+      logger.error(err);
+      this.cachedRetentionDays = this.DEFAULT_RETENTION_DAYS;
+    }
+
+    this.lastCacheRefresh = now;
+    return this.cachedRetentionDays;
+  }
+
+  @CaptureSpan()
+  public static async saveWalkMetrics(data: {
+    projectId: ObjectID;
+    networkDeviceId: ObjectID;
+    deviceName: string | undefined;
+    probeId: ObjectID | undefined;
+    snmpResponse: SnmpMonitorResponse | undefined;
+    responseTimeInMs: number | undefined;
+    isOnline: boolean | undefined;
+  }): Promise<void> {
+    const metricRows: Array<JSONObject> = [];
+    const metricNameServiceNameMap: Dictionary<MetricType> = {};
+
+    const retentionDays: number =
+      await NetworkDeviceMetricUtil.getRetentionDays();
+    const retentionDate: Date = OneUptimeDate.addRemoveDays(
+      OneUptimeDate.getCurrentDate(),
+      retentionDays,
+    );
+
+    const baseAttributes: JSONObject = {
+      projectId: data.projectId.toString(),
+      networkDeviceId: data.networkDeviceId.toString(),
+    };
+
+    if (data.deviceName) {
+      baseAttributes["deviceName"] = data.deviceName;
+    }
+
+    if (data.probeId) {
+      baseAttributes["probeId"] = data.probeId.toString();
+    }
+
+    const pushMetric: (metric: {
+      metricName: string;
+      value: number | null | undefined;
+      description: string;
+      unit: string;
+      extraAttributes?: JSONObject | undefined;
+    }) => void = (metric: {
+      metricName: string;
+      value: number | null | undefined;
+      description: string;
+      unit: string;
+      extraAttributes?: JSONObject | undefined;
+    }) => {
+      if (metric.value === undefined || metric.value === null) {
+        return;
+      }
+
+      if (typeof metric.value !== "number" || !isFinite(metric.value)) {
+        return;
+      }
+
+      metricRows.push(
+        NetworkDeviceMetricUtil.buildDeviceMetricRow({
+          projectId: data.projectId,
+          networkDeviceId: data.networkDeviceId,
+          metricName: metric.metricName,
+          value: metric.value,
+          retentionDate: retentionDate,
+          attributes: {
+            ...baseAttributes,
+            ...(metric.extraAttributes || {}),
+          },
+        }),
+      );
+
+      const metricType: MetricType = new MetricType();
+      metricType.name = metric.metricName;
+      metricType.description = metric.description;
+      metricType.unit = metric.unit;
+
+      metricNameServiceNameMap[metric.metricName] = metricType;
+    };
+
+    if (data.isOnline !== undefined) {
+      pushMetric({
+        metricName: MonitorMetricType.IsOnline,
+        value: data.isOnline ? 1 : 0,
+        description: "SNMP reachability of this network device (1 up, 0 down)",
+        unit: "",
+      });
+    }
+
+    if (data.responseTimeInMs !== undefined) {
+      pushMetric({
+        metricName: MonitorMetricType.ResponseTime,
+        value: data.responseTimeInMs,
+        description: "SNMP response time of this network device",
+        unit: "ms",
+      });
+    }
+
+    const snmpInterfaces: Array<SnmpInterface> | undefined =
+      data.snmpResponse?.interfaces;
+
+    if (snmpInterfaces && snmpInterfaces.length > 0) {
+      const interfacesToEmit: Array<SnmpInterface> = snmpInterfaces.slice(
+        0,
+        NetworkDeviceMetricUtil.maxInterfaceSeries,
+      );
+
+      if (interfacesToEmit.length < snmpInterfaces.length) {
+        logger.warn(
+          `Device ${data.networkDeviceId.toString()}: emitting metrics for first ${interfacesToEmit.length} of ${snmpInterfaces.length} SNMP interfaces`,
+        );
+      }
+
+      for (const snmpInterface of interfacesToEmit) {
+        const interfaceAttributes: JSONObject = {
+          interfaceName: snmpInterface.name,
+          interfaceIndex: snmpInterface.interfaceIndex.toString(),
+        };
+
+        pushMetric({
+          metricName: MonitorMetricType.SnmpInterfaceOperStatus,
+          value: snmpInterface.isOperationallyUp ? 1 : 0,
+          description: "SNMP interface operational status (1 up, 0 down)",
+          unit: "",
+          extraAttributes: interfaceAttributes,
+        });
+        pushMetric({
+          metricName: MonitorMetricType.SnmpInterfaceInBitsPerSecond,
+          value: snmpInterface.inBitsPerSecond,
+          description: "SNMP interface inbound bandwidth",
+          unit: "bps",
+          extraAttributes: interfaceAttributes,
+        });
+        pushMetric({
+          metricName: MonitorMetricType.SnmpInterfaceOutBitsPerSecond,
+          value: snmpInterface.outBitsPerSecond,
+          description: "SNMP interface outbound bandwidth",
+          unit: "bps",
+          extraAttributes: interfaceAttributes,
+        });
+        pushMetric({
+          metricName: MonitorMetricType.SnmpInterfaceUtilizationPercent,
+          value: snmpInterface.utilizationPercent,
+          description: "SNMP interface utilization",
+          unit: "%",
+          extraAttributes: interfaceAttributes,
+        });
+        pushMetric({
+          metricName: MonitorMetricType.SnmpInterfaceErrorsPerSecond,
+          value: snmpInterface.errorsPerSecond,
+          description: "SNMP interface errors per second",
+          unit: "errors/s",
+          extraAttributes: interfaceAttributes,
+        });
+      }
+    }
+
+    /*
+     * Polled health-OID values (vendor-template CPU/memory/temperature and
+     * custom OIDs configured on the device). One series per OID, keyed by
+     * the oid/oidName attributes; non-numeric values (strings, OIDs, MACs)
+     * are skipped — they carry no chartable signal.
+     */
+    const snmpOidResponses: Array<SnmpOidResponse> | undefined =
+      data.snmpResponse?.oidResponses;
+
+    if (snmpOidResponses && snmpOidResponses.length > 0) {
+      const oidResponsesToEmit: Array<SnmpOidResponse> = snmpOidResponses.slice(
+        0,
+        NetworkDeviceMetricUtil.maxOidSeries,
+      );
+
+      if (oidResponsesToEmit.length < snmpOidResponses.length) {
+        logger.warn(
+          `Device ${data.networkDeviceId.toString()}: emitting metrics for first ${oidResponsesToEmit.length} of ${snmpOidResponses.length} SNMP OID responses`,
+        );
+      }
+
+      for (const oidResponse of oidResponsesToEmit) {
+        const numericValue: number | undefined =
+          typeof oidResponse.value === "number" && isFinite(oidResponse.value)
+            ? oidResponse.value
+            : undefined;
+
+        if (numericValue === undefined) {
+          continue;
+        }
+
+        pushMetric({
+          metricName: MonitorMetricType.SnmpOidValue,
+          value: numericValue,
+          description: "Value of a polled SNMP OID",
+          unit: "",
+          extraAttributes: {
+            oid: oidResponse.oid,
+            oidName: oidResponse.name || oidResponse.oid,
+          },
+        });
+      }
+    }
+
+    if (metricRows.length === 0) {
+      return;
+    }
+
+    await MetricService.insertJsonRows(metricRows);
+
+    TelemetryUtil.indexMetricNameServiceNameMap({
+      projectId: data.projectId,
+      metricNameServiceNameMap: metricNameServiceNameMap,
+    }).catch((err: Error) => {
+      logger.error(err);
+    });
+  }
+
+  // Row shape must stay in lockstep with MonitorMetricUtil.buildMonitorMetricRow.
+  private static buildDeviceMetricRow(data: {
+    projectId: ObjectID;
+    networkDeviceId: ObjectID;
+    metricName: string;
+    value: number;
+    retentionDate: Date;
+    attributes: JSONObject;
+  }): JSONObject {
+    const ingestionDate: Date = OneUptimeDate.getCurrentDate();
+    const ingestionTimestamp: string =
+      OneUptimeDate.toClickhouseDateTime(ingestionDate);
+    const timeUnixNano: string =
+      OneUptimeDate.toUnixNano(ingestionDate).toString();
+
+    const attributes: JSONObject = { ...data.attributes };
+    const attributeKeys: Array<string> =
+      TelemetryUtil.getAttributeKeys(attributes);
+
+    return {
+      _id: ObjectID.generateTimeOrdered().toString(),
+      createdAt: ingestionTimestamp,
+      projectId: data.projectId.toString(),
+      primaryEntityId: data.networkDeviceId.toString(),
+      primaryEntityType: ServiceType.NetworkDevice,
+      name: data.metricName,
+      aggregationTemporality: null,
+      metricPointType: MetricPointType.Sum,
+      time: ingestionTimestamp,
+      startTime: null,
+      timeUnixNano: timeUnixNano,
+      startTimeUnixNano: null,
+      attributes: attributes,
+      attributeKeys: attributeKeys,
+      isMonotonic: null,
+      count: null,
+      sum: null,
+      min: null,
+      max: null,
+      bucketCounts: [],
+      explicitBounds: [],
+      value: data.value,
+      retentionDate: OneUptimeDate.toClickhouseDateTime(data.retentionDate),
+    } as JSONObject;
+  }
+}

package/Server/Utils/Monitor/NetworkDeviceWalkUtil.ts

@@ -0,0 +1,251 @@
+import Monitor from "../../../Models/DatabaseModels/Monitor";
+import NetworkDevice from "../../../Models/DatabaseModels/NetworkDevice";
+import MonitorService from "../../Services/MonitorService";
+import NetworkDeviceService from "../../Services/NetworkDeviceService";
+import MonitorResourceUtil from "./MonitorResource";
+import NetworkDeviceMetricUtil from "./NetworkDeviceMetricUtil";
+import NetworkInventoryUtil from "./NetworkInventoryUtil";
+import SnmpInterfaceRateUtil from "./SnmpInterfaceRateUtil";
+import LIMIT_MAX from "../../../Types/Database/LimitMax";
+import MonitorStep from "../../../Types/Monitor/MonitorStep";
+import MonitorType from "../../../Types/Monitor/MonitorType";
+import SnmpMonitorResponse from "../../../Types/Monitor/SnmpMonitor/SnmpMonitorResponse";
+import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
+import { JSONObject } from "../../../Types/JSON";
+import ObjectID from "../../../Types/ObjectID";
+import OneUptimeDate from "../../../Types/Date";
+import logger from "../Logger";
+
+/*
+ * Server half of device-owned polling.
+ *
+ * The device's assigned probe walks every polling-enabled NetworkDevice on
+ * the device's own schedule and reports each walk here. This util then:
+ *
+ *   1. computes interface rates against the device's previous walk
+ *      (NetworkDevice.lastWalkLog),
+ *   2. syncs the device/interface/endpoint inventory and lastSeenAt,
+ *   3. emits device-scoped metrics (so health charts work with zero
+ *      monitors), and
+ *   4. fans the walk out to every Network Device monitor referencing the
+ *      device — each gets an event-driven ProbeMonitorResponse evaluated
+ *      through the ordinary criteria pipeline (incidents, alerts, status).
+ *
+ * Monitors are pure alerting config here: they never poll, and deleting
+ * every monitor leaves the device fully inventoried and charted.
+ */
+export default class NetworkDeviceWalkUtil {
+  public static async processWalkResult(data: {
+    probeId: ObjectID;
+    networkDeviceId: ObjectID;
+    snmpResponse: SnmpMonitorResponse;
+    monitoredAt: Date;
+  }): Promise<void> {
+    /*
+     * Scope the lookup to the reporting probe (the auth middleware only
+     * proves the caller is SOME valid probe) — without this any probe that
+     * learned a foreign device id could overwrite another project's
+     * inventory or trigger its monitors.
+     */
+    const device: NetworkDevice | null = await NetworkDeviceService.findOneBy({
+      query: {
+        _id: data.networkDeviceId,
+        probeId: data.probeId,
+      },
+      select: {
+        _id: true,
+        projectId: true,
+        name: true,
+        siteId: true,
+        lastWalkLog: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    if (!device || !device.id || !device.projectId) {
+      logger.warn(
+        `Device walk ignored: device ${data.networkDeviceId.toString()} is not assigned to probe ${data.probeId.toString()}.`,
+      );
+      return;
+    }
+
+    const isOnline: boolean | undefined = data.snmpResponse.isOnline;
+
+    /*
+     * Interface rates are counter deltas against the previous walk. Compute
+     * them on the FULL interface set, before inventory pruning, so muted
+     * ports still get fresh rates in the inventory table.
+     */
+    SnmpInterfaceRateUtil.attachInterfaceRates({
+      snmpResponse: data.snmpResponse,
+      previousWalkLog: device.lastWalkLog as JSONObject | undefined,
+    });
+
+    /*
+     * Persist this walk as the next delta baseline — but only when it
+     * actually walked interfaces. Keeping the last GOOD counters across
+     * failed polls means rates resume with a correct (longer-window) delta
+     * instead of losing a cycle.
+     */
+    if (
+      data.snmpResponse.interfaces &&
+      data.snmpResponse.interfaces.length > 0
+    ) {
+      await NetworkDeviceService.updateOneById({
+        id: device.id,
+        data: {
+          lastWalkLog: {
+            snmpResponse: JSON.parse(JSON.stringify(data.snmpResponse)),
+            monitoredAt: OneUptimeDate.getCurrentDate(),
+          },
+        } as any,
+        props: {
+          isRoot: true,
+        },
+      });
+    }
+
+    /*
+     * Inventory sync also prunes data.snmpResponse.interfaces down to
+     * monitored ports, so the metrics and criteria below only see ports the
+     * user cares about.
+     */
+    await NetworkInventoryUtil.updateFromWalk({
+      projectId: device.projectId,
+      deviceId: device.id,
+      snmpResponse: data.snmpResponse,
+      isOnline: isOnline,
+    });
+
+    try {
+      await NetworkDeviceMetricUtil.saveWalkMetrics({
+        projectId: device.projectId,
+        networkDeviceId: device.id,
+        deviceName: device.name,
+        probeId: data.probeId,
+        snmpResponse: data.snmpResponse,
+        responseTimeInMs: data.snmpResponse.responseTimeInMs,
+        isOnline: isOnline,
+      });
+    } catch (err) {
+      // Metrics must never fail the walk pipeline.
+      logger.error(
+        `Failed to save device metrics for ${device.id.toString()}:`,
+      );
+      logger.error(err);
+    }
+
+    // --- Fan out to the monitors alerting on this device ---
+    const monitors: Array<Monitor> =
+      await NetworkDeviceWalkUtil.findMonitorsWatchingDevices({
+        projectId: device.projectId,
+        deviceIds: [device.id.toString()],
+      });
+
+    for (const monitor of monitors) {
+      if (
+        monitor.disableActiveMonitoring ||
+        monitor.disableActiveMonitoringBecauseOfManualIncident ||
+        monitor.disableActiveMonitoringBecauseOfScheduledMaintenanceEvent
+      ) {
+        continue;
+      }
+
+      if (!monitor.id || !monitor.projectId) {
+        continue;
+      }
+
+      for (const monitorStep of monitor.monitorSteps?.data
+        ?.monitorStepsInstanceArray || []) {
+        const referencedDeviceId: string | undefined =
+          monitorStep.data?.networkDeviceMonitor?.networkDeviceId;
+
+        if (
+          !referencedDeviceId ||
+          referencedDeviceId !== device.id.toString()
+        ) {
+          continue;
+        }
+
+        const walkResponse: ProbeMonitorResponse = {
+          projectId: monitor.projectId,
+          monitorId: monitor.id,
+          monitorStepId: monitorStep.id,
+          probeId: data.probeId,
+          snmpResponse: data.snmpResponse,
+          isOnline: isOnline,
+          responseTimeInMs: data.snmpResponse.responseTimeInMs,
+          isTimeout: data.snmpResponse.isTimeout,
+          failureCause: data.snmpResponse.failureCause || "",
+          monitoredAt: data.monitoredAt,
+          ingestedAt: OneUptimeDate.getCurrentDate(),
+        };
+
+        try {
+          await MonitorResourceUtil.monitorResource(walkResponse);
+        } catch (err) {
+          logger.error(
+            `Error evaluating device walk for monitor ${monitor.id.toString()}:`,
+          );
+          logger.error(err);
+        }
+      }
+    }
+
+    logger.debug(
+      `Device walk for ${device.id.toString()} processed: evaluated ${monitors.length} watching monitor(s).`,
+    );
+  }
+
+  /*
+   * Every enabled Network Device monitor in the project whose steps
+   * reference one of the given devices. Shared by the walk fan-out above
+   * and the SNMP trap fan-out — the monitor↔device link lives in step JSON,
+   * so this fetches the project's Network Device monitors and filters
+   * server-side.
+   */
+  public static async findMonitorsWatchingDevices(data: {
+    projectId: ObjectID;
+    deviceIds: Array<string>;
+  }): Promise<Array<Monitor>> {
+    if (data.deviceIds.length === 0) {
+      return [];
+    }
+
+    const deviceIdSet: Set<string> = new Set(data.deviceIds);
+
+    const monitors: Array<Monitor> = await MonitorService.findBy({
+      query: {
+        projectId: data.projectId,
+        monitorType: MonitorType.NetworkDevice,
+      },
+      select: {
+        _id: true,
+        projectId: true,
+        monitorSteps: true,
+        disableActiveMonitoring: true,
+        disableActiveMonitoringBecauseOfManualIncident: true,
+        disableActiveMonitoringBecauseOfScheduledMaintenanceEvent: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    return monitors.filter((monitor: Monitor) => {
+      return (monitor.monitorSteps?.data?.monitorStepsInstanceArray || []).some(
+        (monitorStep: MonitorStep) => {
+          const referencedDeviceId: string | undefined =
+            monitorStep.data?.networkDeviceMonitor?.networkDeviceId;
+          return Boolean(
+            referencedDeviceId && deviceIdSet.has(referencedDeviceId),
+          );
+        },
+      );
+    });
+  }
+}