@oneuptime/common 11.4.0 → 11.5.0

package/Server/Utils/Monitor/MonitorResource.ts

@@ -5,10 +5,13 @@ import logger from "../Logger";
 import MonitorCriteriaEvaluator from "./MonitorCriteriaEvaluator";
 import MonitorLogUtil from "./MonitorLogUtil";
 import MonitorMetricUtil from "./MonitorMetricUtil";
+import NetworkInventoryUtil from "./NetworkInventoryUtil";
+import SnmpInterfaceRateUtil from "./SnmpInterfaceRateUtil";
 import DataToProcess from "./DataToProcess";
 import SortOrder from "../../../Types/BaseDatabase/SortOrder";
 import Dictionary from "../../../Types/Dictionary";
 import BadDataException from "../../../Types/Exception/BadDataException";
+import { JSONObject } from "../../../Types/JSON";
 import Semaphore, { SemaphoreMutex } from "../../Infrastructure/Semaphore";
 import IncomingMonitorRequest from "../../../Types/Monitor/IncomingMonitor/IncomingMonitorRequest";
 import MonitorCriteria from "../../../Types/Monitor/MonitorCriteria";
@@ -237,6 +240,16 @@ export default class MonitorResourceUtil {
       let probeName: string | undefined = undefined;
       const monitorName: string | undefined = monitor.name || undefined;
 
+      /*
+       * SNMP trap responses are event-driven, not check results. They are
+       * evaluated ONLY against trap criteria; they must not overwrite the
+       * last check's counters, participate in probe agreement, or reset the
+       * monitor to its default status when no criteria matches.
+       */
+      const isSnmpTrapEvent: boolean = Boolean(
+        (dataToProcess as ProbeMonitorResponse).snmpTrapResponse,
+      );
+
       // save the last log to MonitorProbe.
 
       // get last log. We do this because there are many monitoring steps and we need to store those.
@@ -275,26 +288,58 @@ export default class MonitorResourceUtil {
 
           probeName = monitorProbe.probe?.name || undefined;
 
-          await MonitorProbeService.updateOneBy({
-            query: {
-              monitorId: monitor.id!,
-              probeId: (dataToProcess as ProbeMonitorResponse).probeId!,
-            },
-            data: {
-              lastMonitoringLog: {
-                ...(monitorProbe.lastMonitoringLog || {}),
-                [(
-                  dataToProcess as ProbeMonitorResponse
-                ).monitorStepId.toString()]: {
-                  ...JSON.parse(JSON.stringify(dataToProcess)),
-                  monitoredAt: OneUptimeDate.getCurrentDate(),
-                },
-              } as any,
-            },
-            props: {
-              isRoot: true,
-            },
-          });
+          /*
+           * SNMP interface rates (bandwidth, utilization, errors/sec) are
+           * deltas against the previous check's counters — computed here,
+           * while the previous log is still available, so the computed
+           * values flow into metrics, criteria, and the stored log below.
+           */
+          if (monitor.monitorType === MonitorType.NetworkDevice) {
+            SnmpInterfaceRateUtil.attachInterfaceRates({
+              probeMonitorResponse: dataToProcess as ProbeMonitorResponse,
+              previousStepLog: (
+                monitorProbe.lastMonitoringLog as JSONObject | undefined
+              )?.[
+                (dataToProcess as ProbeMonitorResponse).monitorStepId.toString()
+              ] as JSONObject | undefined,
+            });
+
+            /*
+             * Sync the NetworkDevice/NetworkInterface inventory from the
+             * walk, then prune the response to monitored interfaces so
+             * criteria and metrics ignore muted ports. Trap events carry
+             * no walk data — nothing to sync.
+             */
+            if (!isSnmpTrapEvent) {
+              await NetworkInventoryUtil.updateFromWalk({
+                monitor: monitor,
+                dataToProcess: dataToProcess as ProbeMonitorResponse,
+              });
+            }
+          }
+
+          if (!isSnmpTrapEvent) {
+            await MonitorProbeService.updateOneBy({
+              query: {
+                monitorId: monitor.id!,
+                probeId: (dataToProcess as ProbeMonitorResponse).probeId!,
+              },
+              data: {
+                lastMonitoringLog: {
+                  ...(monitorProbe.lastMonitoringLog || {}),
+                  [(
+                    dataToProcess as ProbeMonitorResponse
+                  ).monitorStepId.toString()]: {
+                    ...JSON.parse(JSON.stringify(dataToProcess)),
+                    monitoredAt: OneUptimeDate.getCurrentDate(),
+                  },
+                } as any,
+              },
+              props: {
+                isRoot: true,
+              },
+            });
+          }
         }
       }
 
@@ -603,7 +648,12 @@ export default class MonitorResourceUtil {
       // Check probe agreement for probe-based monitors
       if (
         monitor.monitorType &&
-        MonitorTypeHelper.isProbableMonitor(monitor.monitorType)
+        MonitorTypeHelper.isProbableMonitor(monitor.monitorType) &&
+        /*
+         * Traps arrive on exactly one probe — other probes' polled state
+         * cannot corroborate them, so agreement would always veto the trap.
+         */
+        !isSnmpTrapEvent
       ) {
         const probeAgreementResult: ProbeAgreementResult =
           await MonitorResourceUtil.checkProbeAgreement({
@@ -874,6 +924,12 @@ export default class MonitorResourceUtil {
         });
       } else if (
         !response.criteriaMetId &&
+        /*
+         * A trap that matches no criteria is simply ignored — it must not
+         * reset the monitor to its default status (the polled checks own
+         * the monitor's state).
+         */
+        !isSnmpTrapEvent &&
         monitorSteps.data.defaultMonitorStatusId &&
         monitor.currentMonitorStatusId?.toString() !==
           monitorSteps.data.defaultMonitorStatusId.toString()

package/Server/Utils/Monitor/MonitorTemplateUtil.ts

@@ -245,7 +245,7 @@ export default class MonitorTemplateUtil {
         } as JSONObject;
       }
 
-      if (data.monitorType === MonitorType.SNMP) {
+      if (data.monitorType === MonitorType.NetworkDevice) {
         const snmpResponse: SnmpMonitorResponse | undefined = (
           data.dataToProcess as ProbeMonitorResponse
         ).snmpResponse;

package/Server/Utils/Monitor/NetworkDeviceHydrationUtil.ts

@@ -0,0 +1,201 @@
+import NetworkDevice from "../../../Models/DatabaseModels/NetworkDevice";
+import NetworkDeviceService from "../../Services/NetworkDeviceService";
+import MonitorSteps from "../../../Types/Monitor/MonitorSteps";
+import QueryHelper from "../../Types/Database/QueryHelper";
+import LIMIT_MAX from "../../../Types/Database/LimitMax";
+import MonitorType from "../../../Types/Monitor/MonitorType";
+import SnmpVersion from "../../../Types/Monitor/SnmpMonitor/SnmpVersion";
+import SnmpV3Auth from "../../../Types/Monitor/SnmpMonitor/SnmpV3Auth";
+import SnmpSecurityLevel from "../../../Types/Monitor/SnmpMonitor/SnmpSecurityLevel";
+import SnmpAuthProtocol from "../../../Types/Monitor/SnmpMonitor/SnmpAuthProtocol";
+import SnmpPrivProtocol from "../../../Types/Monitor/SnmpMonitor/SnmpPrivProtocol";
+import ObjectID from "../../../Types/ObjectID";
+import logger from "../Logger";
+
+/*
+ * Network Device monitor steps reference a NetworkDevice resource instead of
+ * carrying SNMP connection details. Probes are stateless and only understand
+ * concrete SNMP config, so before work is handed out the referenced device's
+ * hostname/credentials are hydrated into each step's `snmpMonitor` field.
+ */
+export interface HydratableMonitor {
+  id?: ObjectID | null | undefined;
+  monitorType?: MonitorType | undefined;
+  monitorSteps?: MonitorSteps | undefined;
+}
+
+export default class NetworkDeviceHydrationUtil {
+  // Accepts Monitor and MonitorTest alike (structural: monitorType + monitorSteps).
+  public static async hydrateNetworkDeviceMonitors(
+    monitors: Array<HydratableMonitor>,
+  ): Promise<void> {
+    const deviceIds: Set<string> = new Set();
+
+    for (const monitor of monitors) {
+      if (monitor.monitorType !== MonitorType.NetworkDevice) {
+        continue;
+      }
+
+      for (const step of monitor.monitorSteps?.data
+        ?.monitorStepsInstanceArray || []) {
+        const deviceId: string | undefined =
+          step.data?.networkDeviceMonitor?.networkDeviceId;
+        if (deviceId) {
+          deviceIds.add(deviceId);
+        }
+      }
+    }
+
+    if (deviceIds.size === 0) {
+      return;
+    }
+
+    const devices: Array<NetworkDevice> = await NetworkDeviceService.findBy({
+      query: {
+        _id: QueryHelper.any(Array.from(deviceIds)),
+      },
+      select: {
+        _id: true,
+        hostname: true,
+        snmpVersion: true,
+        snmpCommunityString: true,
+        snmpPort: true,
+        snmpV3Auth: true,
+        snmpV3SecurityLevel: true,
+        snmpV3Username: true,
+        snmpV3AuthProtocol: true,
+        snmpV3AuthKey: true,
+        snmpV3PrivProtocol: true,
+        snmpV3PrivKey: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    const devicesById: Map<string, NetworkDevice> = new Map();
+    for (const device of devices) {
+      if (device.id) {
+        devicesById.set(device.id.toString(), device);
+      }
+    }
+
+    for (const monitor of monitors) {
+      if (monitor.monitorType !== MonitorType.NetworkDevice) {
+        continue;
+      }
+
+      for (const step of monitor.monitorSteps?.data
+        ?.monitorStepsInstanceArray || []) {
+        const deviceId: string | undefined =
+          step.data?.networkDeviceMonitor?.networkDeviceId;
+
+        if (!deviceId || !step.data) {
+          continue;
+        }
+
+        const device: NetworkDevice | undefined = devicesById.get(deviceId);
+
+        if (!device || !device.hostname) {
+          logger.warn(
+            `Network Device monitor ${monitor.id?.toString()} references missing device ${deviceId}. Step will not be executable.`,
+          );
+          continue;
+        }
+
+        step.data.snmpMonitor = {
+          snmpVersion: NetworkDeviceHydrationUtil.parseSnmpVersion(
+            device.snmpVersion,
+          ),
+          hostname: device.hostname,
+          port: device.snmpPort || 161,
+          communityString: device.snmpCommunityString || undefined,
+          snmpV3Auth: NetworkDeviceHydrationUtil.buildSnmpV3Auth(device),
+          oids: step.data.networkDeviceMonitor?.oids || [],
+          timeout: 5000,
+          retries: 3,
+          monitorInterfaces:
+            step.data.networkDeviceMonitor?.monitorInterfaces !== false,
+        };
+      }
+    }
+  }
+
+  /*
+   * Assembles the SnmpV3Auth object the probe expects. Prefers the flattened
+   * snmpV3* columns (the current storage), falling back to the deprecated
+   * snmpV3Auth JSON column so devices created before the columns existed keep
+   * working. Returns undefined when no v3 username is configured.
+   */
+  private static buildSnmpV3Auth(
+    device: NetworkDevice,
+  ): SnmpV3Auth | undefined {
+    if (device.snmpV3Username) {
+      return {
+        securityLevel:
+          (device.snmpV3SecurityLevel as SnmpSecurityLevel) ||
+          SnmpSecurityLevel.NoAuthNoPriv,
+        username: device.snmpV3Username,
+        authProtocol:
+          (device.snmpV3AuthProtocol as SnmpAuthProtocol | undefined) ||
+          undefined,
+        authKey: device.snmpV3AuthKey || undefined,
+        privProtocol:
+          (device.snmpV3PrivProtocol as SnmpPrivProtocol | undefined) ||
+          undefined,
+        privKey: device.snmpV3PrivKey || undefined,
+      };
+    }
+
+    // Legacy devices stored the whole object in the snmpV3Auth JSON column.
+    const legacy: SnmpV3Auth | undefined = device.snmpV3Auth as
+      | SnmpV3Auth
+      | undefined;
+    if (legacy && legacy.username) {
+      return legacy;
+    }
+
+    return undefined;
+  }
+
+  // Tolerates both enum values ("2c") and enum keys ("V2c") in stored config.
+  private static parseSnmpVersion(value: string | undefined): SnmpVersion {
+    switch ((value || "").toLowerCase()) {
+      case "1":
+      case "v1":
+        return SnmpVersion.V1;
+      case "3":
+      case "v3":
+        return SnmpVersion.V3;
+      default:
+        return SnmpVersion.V2c;
+    }
+  }
+
+  /*
+   * Resolves which NetworkDevices (polled by the given probe) match a trap
+   * source IP. Used by the trap ingest to fan traps out to device monitors.
+   */
+  public static async findDevicesByProbeAndSource(data: {
+    probeId: ObjectID;
+    sourceIpAddress: string;
+  }): Promise<Array<NetworkDevice>> {
+    return NetworkDeviceService.findBy({
+      query: {
+        probeId: data.probeId,
+        hostname: data.sourceIpAddress,
+      },
+      select: {
+        _id: true,
+        projectId: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+  }
+}

package/Server/Utils/Monitor/NetworkInventoryUtil.ts

@@ -0,0 +1,215 @@
+import Monitor from "../../../Models/DatabaseModels/Monitor";
+import NetworkInterface from "../../../Models/DatabaseModels/NetworkInterface";
+import NetworkDeviceService from "../../Services/NetworkDeviceService";
+import NetworkInterfaceService from "../../Services/NetworkInterfaceService";
+import LIMIT_MAX from "../../../Types/Database/LimitMax";
+import MonitorStep from "../../../Types/Monitor/MonitorStep";
+import SnmpInterface from "../../../Types/Monitor/SnmpMonitor/SnmpInterface";
+import LldpNeighbor from "../../../Types/Monitor/SnmpMonitor/LldpNeighbor";
+import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
+import ObjectID from "../../../Types/ObjectID";
+import OneUptimeDate from "../../../Types/Date";
+import logger from "../Logger";
+
+/*
+ * Keeps the NetworkDevice / NetworkInterface inventory in sync with each
+ * interface walk, then prunes the in-flight response down to MONITORED
+ * interfaces so criteria and metrics only consider ports the user cares
+ * about. Inventory rows always reflect every walked interface; the
+ * isMonitored flag is user-owned and never overwritten here.
+ */
+export default class NetworkInventoryUtil {
+  public static async updateFromWalk(data: {
+    monitor: Monitor;
+    dataToProcess: ProbeMonitorResponse;
+  }): Promise<void> {
+    const step: MonitorStep | undefined =
+      data.monitor.monitorSteps?.data?.monitorStepsInstanceArray?.find(
+        (monitorStep: MonitorStep) => {
+          return (
+            monitorStep.id.toString() ===
+            data.dataToProcess.monitorStepId.toString()
+          );
+        },
+      );
+
+    const deviceIdAsString: string | undefined =
+      step?.data?.networkDeviceMonitor?.networkDeviceId;
+
+    if (!deviceIdAsString || !data.monitor.projectId) {
+      return;
+    }
+
+    const deviceId: ObjectID = new ObjectID(deviceIdAsString);
+    const walkedInterfaces: Array<SnmpInterface> =
+      data.dataToProcess.snmpResponse?.interfaces || [];
+    const systemInfo:
+      | { sysDescr?: string | undefined; sysName?: string | undefined }
+      | undefined = data.dataToProcess.snmpResponse?.systemInfo;
+    const lldpNeighbors: Array<LldpNeighbor> | undefined =
+      data.dataToProcess.snmpResponse?.lldpNeighbors;
+
+    const now: Date = OneUptimeDate.getCurrentDate();
+
+    try {
+      // --- Device enrichment + cached counts ---
+      const deviceUpdate: Record<string, unknown> = {
+        lastSeenAt: now,
+      };
+
+      if (systemInfo?.sysDescr) {
+        deviceUpdate["sysDescr"] = systemInfo.sysDescr.substring(0, 500);
+      }
+      if (systemInfo?.sysName) {
+        deviceUpdate["sysName"] = systemInfo.sysName.substring(0, 100);
+      }
+
+      /*
+       * Store the LLDP snapshot (capped) whenever the walk ran, even if it
+       * found nothing — clearing stale neighbors is as important as adding
+       * new ones for keeping the topology accurate.
+       */
+      if (lldpNeighbors !== undefined) {
+        deviceUpdate["lldpNeighbors"] = lldpNeighbors.slice(0, 256);
+      }
+
+      if (walkedInterfaces.length > 0) {
+        deviceUpdate["interfacesTotal"] = walkedInterfaces.length;
+        deviceUpdate["interfacesUp"] = walkedInterfaces.filter(
+          (walked: SnmpInterface) => {
+            return walked.isAdministrativelyUp && walked.isOperationallyUp;
+          },
+        ).length;
+        deviceUpdate["interfacesDown"] = walkedInterfaces.filter(
+          (walked: SnmpInterface) => {
+            return walked.isAdministrativelyUp && !walked.isOperationallyUp;
+          },
+        ).length;
+      }
+
+      await NetworkDeviceService.updateOneById({
+        id: deviceId,
+        data: deviceUpdate as any,
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (walkedInterfaces.length === 0) {
+        return;
+      }
+
+      // --- Interface upsert ---
+      const existingInterfaces: Array<NetworkInterface> =
+        await NetworkInterfaceService.findBy({
+          query: {
+            networkDeviceId: deviceId,
+          },
+          select: {
+            _id: true,
+            interfaceIndex: true,
+            isMonitored: true,
+          },
+          limit: LIMIT_MAX,
+          skip: 0,
+          props: {
+            isRoot: true,
+          },
+        });
+
+      const existingByIndex: Map<number, NetworkInterface> = new Map();
+      for (const existing of existingInterfaces) {
+        if (existing.interfaceIndex !== undefined) {
+          existingByIndex.set(existing.interfaceIndex, existing);
+        }
+      }
+
+      const unmonitoredIndexes: Set<number> = new Set();
+
+      for (const walked of walkedInterfaces) {
+        const existing: NetworkInterface | undefined = existingByIndex.get(
+          walked.interfaceIndex,
+        );
+
+        if (existing && existing.isMonitored === false) {
+          unmonitoredIndexes.add(walked.interfaceIndex);
+        }
+
+        const interfaceData: Record<string, unknown> = {
+          name: (walked.name || "").substring(0, 100),
+          alias: walked.alias ? walked.alias.substring(0, 100) : null,
+          isOperationallyUp: walked.isOperationallyUp,
+          isAdministrativelyUp: walked.isAdministrativelyUp,
+          speedInMbps:
+            walked.speedInBitsPerSecond !== undefined
+              ? walked.speedInBitsPerSecond / 1000000
+              : null,
+          inRateMbps:
+            walked.inBitsPerSecond !== undefined
+              ? Math.round((walked.inBitsPerSecond / 1000000) * 1000) / 1000
+              : null,
+          outRateMbps:
+            walked.outBitsPerSecond !== undefined
+              ? Math.round((walked.outBitsPerSecond / 1000000) * 1000) / 1000
+              : null,
+          utilizationPercent: walked.utilizationPercent ?? null,
+          errorsPerSecond: walked.errorsPerSecond ?? null,
+          lastSeenAt: now,
+        };
+
+        if (existing && existing.id) {
+          await NetworkInterfaceService.updateOneById({
+            id: existing.id,
+            data: interfaceData as any,
+            props: {
+              isRoot: true,
+            },
+          });
+        } else {
+          const newInterface: NetworkInterface = new NetworkInterface();
+          newInterface.projectId = data.monitor.projectId;
+          newInterface.networkDeviceId = deviceId;
+          newInterface.interfaceIndex = walked.interfaceIndex;
+          newInterface.name = (walked.name || "").substring(0, 100);
+          if (walked.alias) {
+            newInterface.alias = walked.alias.substring(0, 100);
+          }
+          newInterface.isMonitored = true;
+          newInterface.isOperationallyUp = walked.isOperationallyUp;
+          newInterface.isAdministrativelyUp = walked.isAdministrativelyUp;
+          if (walked.speedInBitsPerSecond !== undefined) {
+            newInterface.speedInMbps = walked.speedInBitsPerSecond / 1000000;
+          }
+          newInterface.lastSeenAt = now;
+
+          await NetworkInterfaceService.create({
+            data: newInterface,
+            props: {
+              isRoot: true,
+            },
+          });
+        }
+      }
+
+      /*
+       * Prune the in-flight response to monitored interfaces only, so
+       * criteria (interface down / utilization / errors) and per-interface
+       * metrics ignore ports the user muted. The inventory above keeps the
+       * full picture.
+       */
+      if (unmonitoredIndexes.size > 0 && data.dataToProcess.snmpResponse) {
+        data.dataToProcess.snmpResponse.interfaces = walkedInterfaces.filter(
+          (walked: SnmpInterface) => {
+            return !unmonitoredIndexes.has(walked.interfaceIndex);
+          },
+        );
+      }
+    } catch (err) {
+      // Inventory bookkeeping must never fail the check pipeline.
+      logger.error(
+        `Failed to update network inventory for device ${deviceIdAsString}:`,
+      );
+      logger.error(err);
+    }
+  }
+}