@oneuptime/common 12.0.32 → 12.0.33

package/Server/Utils/Monitor/NetworkDeviceMetricUtil.ts

@@ -13,6 +13,7 @@ import MetricType from "../../../Models/DatabaseModels/MetricType";
 import Dictionary from "../../../Types/Dictionary";
 import { JSONObject } from "../../../Types/JSON";
 import MonitorMetricType from "../../../Types/Monitor/MonitorMetricType";
+import PingMonitorResponse from "../../../Types/Monitor/PingMonitor/PingMonitorResponse";
 import SnmpInterface from "../../../Types/Monitor/SnmpMonitor/SnmpInterface";
 import SnmpMonitorResponse, {
   SnmpOidResponse,
@@ -22,14 +23,28 @@ import OneUptimeDate from "../../../Types/Date";
 import CaptureSpan from "../Telemetry/CaptureSpan";
 
 /*
- * Emits device-scoped metrics from each device walk — availability,
- * response time, per-interface status/bandwidth/utilization/errors, and
- * polled health-OID values. Series are keyed to the NetworkDevice
- * (primaryEntityId = deviceId, attributes.networkDeviceId), NOT to a
- * monitor, so the device pages can chart health for every registered
- * device — including devices no monitor watches. Mirrors the SNMP sections
- * of MonitorMetricUtil, which continue to emit monitor-scoped series for
- * monitors that alert on the device.
+ * Ping round-trip time of a network device, in ms.
+ *
+ * Ping monitors write their RTT into MonitorMetricType.ResponseTime, but on
+ * a network device that series is the SNMP walk's time (and is absent on a
+ * ping-only poll), so the RTT needs a series of its own. It sits in the
+ * `oneuptime.monitor.ping.*` namespace beside PacketLossPercent and Jitter,
+ * which ARE reused from the Ping monitor. MonitorMetricType has no member
+ * for it yet; this constant should graduate into that enum (and the device
+ * metric catalog) rather than be duplicated.
+ */
+export const NETWORK_DEVICE_PING_ROUND_TRIP_TIME_METRIC_NAME: string =
+  "oneuptime.monitor.ping.round.trip.time";
+
+/*
+ * Emits device-scoped metrics from each device poll — reachability, ping
+ * RTT / packet loss, SNMP walk time, per-interface
+ * status/bandwidth/utilization/errors, and polled health-OID values.
+ * Series are keyed to the NetworkDevice (primaryEntityId = deviceId,
+ * attributes.networkDeviceId), NOT to a monitor, so the device pages can
+ * chart health for every registered device — including devices no monitor
+ * watches. Mirrors the SNMP sections of MonitorMetricUtil, which continue
+ * to emit monitor-scoped series for monitors that alert on the device.
  */
 export default class NetworkDeviceMetricUtil {
   /*
@@ -103,9 +118,14 @@ export default class NetworkDeviceMetricUtil {
     networkDeviceId: ObjectID;
     deviceName: string | undefined;
     probeId: ObjectID | undefined;
+    // The SNMP walk when one ran; undefined on a ping-only poll.
     snmpResponse: SnmpMonitorResponse | undefined;
+    // The walk's time only - undefined on a ping-only poll, never the RTT.
     responseTimeInMs: number | undefined;
+    // Device reachability: answered ping OR the walk succeeded.
     isOnline: boolean | undefined;
+    // RTT / packet loss from the probe's ping, when it ran one.
+    pingResponse?: PingMonitorResponse | undefined;
   }): Promise<void> {
     const metricRows: Array<JSONObject> = [];
     const metricNameServiceNameMap: Dictionary<MetricType> = {};
@@ -177,16 +197,52 @@ export default class NetworkDeviceMetricUtil {
       pushMetric({
         metricName: MonitorMetricType.IsOnline,
         value: data.isOnline ? 1 : 0,
-        description: "SNMP reachability of this network device (1 up, 0 down)",
+        description:
+          "Reachability of this network device by ping or SNMP (1 up, 0 down)",
         unit: "",
       });
     }
 
+    /*
+     * Walk time only. A ping-only poll has no walk and writes no point
+     * here; its latency is the RTT series below. Mixing the two into one
+     * series would chart a 1 ms ICMP echo and a 400 ms 48-port walk as the
+     * same thing.
+     */
     if (data.responseTimeInMs !== undefined) {
       pushMetric({
         metricName: MonitorMetricType.ResponseTime,
         value: data.responseTimeInMs,
-        description: "SNMP response time of this network device",
+        description:
+          "SNMP walk time of this network device (absent when the poll ran no walk)",
+        unit: "ms",
+      });
+    }
+
+    /*
+     * The probe's ping, when it ran one. Packet loss and jitter reuse the
+     * Ping monitor's series so the same charts and criteria vocabulary
+     * apply; the RTT gets its own (see the constant's comment). pushMetric
+     * skips undefined values, so an older probe that reports no ping
+     * writes nothing here.
+     */
+    if (data.pingResponse) {
+      pushMetric({
+        metricName: NETWORK_DEVICE_PING_ROUND_TRIP_TIME_METRIC_NAME,
+        value: data.pingResponse.avgRoundTripTimeInMs,
+        description: "Ping round-trip time to this network device",
+        unit: "ms",
+      });
+      pushMetric({
+        metricName: MonitorMetricType.PacketLossPercent,
+        value: data.pingResponse.packetLossPercent,
+        description: "Ping packet loss to this network device",
+        unit: "%",
+      });
+      pushMetric({
+        metricName: MonitorMetricType.Jitter,
+        value: data.pingResponse.jitterInMs,
+        description: "Ping jitter to this network device",
         unit: "ms",
       });
     }

package/Server/Utils/Monitor/NetworkDeviceWalkUtil.ts

@@ -2,13 +2,18 @@ import Monitor from "../../../Models/DatabaseModels/Monitor";
 import NetworkDevice from "../../../Models/DatabaseModels/NetworkDevice";
 import MonitorService from "../../Services/MonitorService";
 import NetworkDeviceService from "../../Services/NetworkDeviceService";
+import Query from "../../Types/Database/Query";
+import QueryHelper from "../../Types/Database/QueryHelper";
 import MonitorResourceUtil from "./MonitorResource";
+import { NetworkDevicePollMode } from "./NetworkDeviceHydrationUtil";
 import NetworkDeviceMetricUtil from "./NetworkDeviceMetricUtil";
 import NetworkInventoryUtil from "./NetworkInventoryUtil";
 import SnmpInterfaceRateUtil from "./SnmpInterfaceRateUtil";
+import SortOrder from "../../../Types/BaseDatabase/SortOrder";
 import LIMIT_MAX from "../../../Types/Database/LimitMax";
 import MonitorStep from "../../../Types/Monitor/MonitorStep";
 import MonitorType from "../../../Types/Monitor/MonitorType";
+import PingMonitorResponse from "../../../Types/Monitor/PingMonitor/PingMonitorResponse";
 import SnmpMonitorResponse from "../../../Types/Monitor/SnmpMonitor/SnmpMonitorResponse";
 import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
 import { JSONObject } from "../../../Types/JSON";
@@ -16,18 +21,37 @@ import ObjectID from "../../../Types/ObjectID";
 import OneUptimeDate from "../../../Types/Date";
 import logger from "../Logger";
 
+/*
+ * What the probe's ping of a device found, as the probe reports it
+ * (PingMonitor.checkReachability): a verdict, the average RTT, the loss,
+ * and why it failed when it did. Packet counts are optional because the
+ * reachability ping is a short fixed burst whose counts the probe need not
+ * report; the loss percentage is the number that matters.
+ */
+export interface NetworkDevicePingResult {
+  isOnline: boolean;
+  avgRttMs?: number | undefined;
+  packetLossPercent?: number | undefined;
+  packetsSent?: number | undefined;
+  packetsReceived?: number | undefined;
+  failureCause?: string | undefined;
+}
+
 /*
  * Server half of device-owned polling.
  *
- * The device's assigned probe walks every polling-enabled NetworkDevice on
- * the device's own schedule and reports each walk here. This util then:
+ * The device's assigned probe polls every polling-enabled NetworkDevice on
+ * the device's own schedule - pinging it always, and walking it over SNMP
+ * as well when it has usable credentials - and reports each poll here. This
+ * util then:
  *
  *   1. computes interface rates against the device's previous walk
- *      (NetworkDevice.lastWalkLog),
- *   2. syncs the device/interface/endpoint inventory and lastSeenAt,
+ *      (NetworkDevice.lastWalkLog) when interfaces were walked,
+ *   2. syncs the device/interface/endpoint inventory, reachability and
+ *      lastSeenAt,
  *   3. emits device-scoped metrics (so health charts work with zero
  *      monitors), and
- *   4. fans the walk out to every Network Device monitor referencing the
+ *   4. fans the poll out to every Network Device monitor referencing the
  *      device — each gets an event-driven ProbeMonitorResponse evaluated
  *      through the ordinary criteria pipeline (incidents, alerts, status).
  *
@@ -35,10 +59,36 @@ import logger from "../Logger";
  * every monitor leaves the device fully inventoried and charted.
  */
 export default class NetworkDeviceWalkUtil {
+  /*
+   * One page of the monitor fan-out lookup. The indexed query rarely needs
+   * a second page (a device has a handful of monitors); the legacy JSON
+   * scan can, on a project with more than this many hand-made Network
+   * Device monitors.
+   */
+  private static readonly FAN_OUT_PAGE_SIZE: number = LIMIT_MAX;
+
   public static async processWalkResult(data: {
     probeId: ObjectID;
     networkDeviceId: ObjectID;
-    snmpResponse: SnmpMonitorResponse;
+    /*
+     * Device reachability: answered ping OR the walk succeeded. A caller
+     * that predates ping-first polling passes only a walk; its verdict is
+     * then the device's verdict.
+     */
+    isOnline?: boolean | undefined;
+    /*
+     * How the probe polled - "ping" (no credentials, ping only) or "snmp"
+     * (ping and a walk in parallel). Absent from a pre-ping-first caller,
+     * and read as "snmp" then: such a probe only ever walked.
+     */
+    pollMode?: NetworkDevicePollMode | undefined;
+    // The probe's ping, when it ran one (older probes send none).
+    pingResponse?: NetworkDevicePingResult | undefined;
+    /*
+     * The SNMP walk, when one ran - success or failure. Undefined on a
+     * ping-only poll, and never synthesized from the ping.
+     */
+    snmpResponse?: SnmpMonitorResponse | undefined;
     monitoredAt: Date;
   }): Promise<void> {
     /*
@@ -66,28 +116,45 @@ export default class NetworkDeviceWalkUtil {
 
     if (!device || !device.id || !device.projectId) {
       logger.warn(
-        `Device walk ignored: device ${data.networkDeviceId.toString()} is not assigned to probe ${data.probeId.toString()}.`,
+        `Device poll ignored: device ${data.networkDeviceId.toString()} is not assigned to probe ${data.probeId.toString()}.`,
       );
       return;
     }
 
-    const isOnline: boolean | undefined = data.snmpResponse.isOnline;
+    const pollMode: NetworkDevicePollMode = data.pollMode ?? "snmp";
+
+    /*
+     * The device verdict. A ping-first probe states it (ping OR walk); an
+     * older probe only walked, so its walk verdict IS the device verdict -
+     * and, matching the inventory's convention, a walk that reports no
+     * verdict at all counts as answered.
+     */
+    const isOnline: boolean =
+      data.isOnline ??
+      (data.snmpResponse ? data.snmpResponse.isOnline !== false : false);
+
+    const pingResponse: PingMonitorResponse | undefined =
+      NetworkDeviceWalkUtil.toPingMonitorResponse(data.pingResponse);
 
     /*
      * Interface rates are counter deltas against the previous walk. Compute
      * them on the FULL interface set, before inventory pruning, so muted
-     * ports still get fresh rates in the inventory table.
+     * ports still get fresh rates in the inventory table. Nothing to do on
+     * a poll that ran no walk.
      */
-    SnmpInterfaceRateUtil.attachInterfaceRates({
-      snmpResponse: data.snmpResponse,
-      previousWalkLog: device.lastWalkLog as JSONObject | undefined,
-    });
+    if (data.snmpResponse) {
+      SnmpInterfaceRateUtil.attachInterfaceRates({
+        snmpResponse: data.snmpResponse,
+        previousWalkLog: device.lastWalkLog as JSONObject | undefined,
+      });
+    }
 
     /*
      * Persist this walk as the next delta baseline — but only when it
      * actually walked interfaces. Keeping the last GOOD counters across
-     * failed polls means rates resume with a correct (longer-window) delta
-     * instead of losing a cycle.
+     * failed polls (and across ping-only polls, which walk nothing) means
+     * rates resume with a correct (longer-window) delta instead of losing
+     * a cycle.
      *
      * Hook-free single-statement write, deliberately. A 48-interface walk
      * log is ~28 KB of jsonb, stored out of line: the full updateOneById
@@ -126,6 +193,7 @@ export default class NetworkDeviceWalkUtil {
      * counter at all.
      */
     if (
+      data.snmpResponse &&
       data.snmpResponse.interfaces &&
       data.snmpResponse.interfaces.length > 0
     ) {
@@ -177,6 +245,7 @@ export default class NetworkDeviceWalkUtil {
       deviceId: device.id,
       snmpResponse: data.snmpResponse,
       isOnline: isOnline,
+      pollMode: pollMode,
     });
 
     try {
@@ -186,8 +255,10 @@ export default class NetworkDeviceWalkUtil {
         deviceName: device.name,
         probeId: data.probeId,
         snmpResponse: data.snmpResponse,
-        responseTimeInMs: data.snmpResponse.responseTimeInMs,
+        // Walk time only: a ping-only poll writes no response-time point.
+        responseTimeInMs: data.snmpResponse?.responseTimeInMs,
         isOnline: isOnline,
+        pingResponse: pingResponse,
       });
     } catch (err) {
       // Metrics must never fail the walk pipeline.
@@ -197,6 +268,19 @@ export default class NetworkDeviceWalkUtil {
       logger.error(err);
     }
 
+    /*
+     * Why the poll failed, for the incident/alert narrative. The walk's own
+     * cause is passed through whenever a walk ran (it is the more specific
+     * of the two, and the "SNMP walk failing" alert reads it); the ping's
+     * cause is only worth stating when the device is actually unreachable
+     * - on an ICMP-filtered device whose walk succeeds, "ping timed out" is
+     * not a failure of anything.
+     */
+    const failureCause: string =
+      data.snmpResponse?.failureCause ||
+      (!isOnline ? data.pingResponse?.failureCause : undefined) ||
+      "";
+
     // --- Fan out to the monitors alerting on this device ---
     const monitors: Array<Monitor> =
       await NetworkDeviceWalkUtil.findMonitorsWatchingDevices({
@@ -229,16 +313,23 @@ export default class NetworkDeviceWalkUtil {
           continue;
         }
 
+        /*
+         * `isOnline` is the device verdict; `snmpResponse` is the real walk
+         * or undefined - NEVER a synthesized failure, so walk-dependent
+         * criteria see "not evaluated" rather than "breaching" on a
+         * ping-only device. `responseTimeInMs` is the walk's time only.
+         */
         const walkResponse: ProbeMonitorResponse = {
           projectId: monitor.projectId,
           monitorId: monitor.id,
           monitorStepId: monitorStep.id,
           probeId: data.probeId,
           snmpResponse: data.snmpResponse,
+          pingResponse: pingResponse,
           isOnline: isOnline,
-          responseTimeInMs: data.snmpResponse.responseTimeInMs,
-          isTimeout: data.snmpResponse.isTimeout,
-          failureCause: data.snmpResponse.failureCause || "",
+          responseTimeInMs: data.snmpResponse?.responseTimeInMs,
+          isTimeout: data.snmpResponse?.isTimeout,
+          failureCause: failureCause,
           monitoredAt: data.monitoredAt,
           ingestedAt: OneUptimeDate.getCurrentDate(),
         };
@@ -247,7 +338,7 @@ export default class NetworkDeviceWalkUtil {
           await MonitorResourceUtil.monitorResource(walkResponse);
         } catch (err) {
           logger.error(
-            `Error evaluating device walk for monitor ${monitor.id.toString()}:`,
+            `Error evaluating device poll for monitor ${monitor.id.toString()}:`,
           );
           logger.error(err);
         }
@@ -255,16 +346,67 @@ export default class NetworkDeviceWalkUtil {
     }
 
     logger.debug(
-      `Device walk for ${device.id.toString()} processed: evaluated ${monitors.length} watching monitor(s).`,
+      `Device poll for ${device.id.toString()} (${pollMode}) processed: evaluated ${monitors.length} watching monitor(s).`,
     );
   }
 
   /*
-   * Every enabled Network Device monitor in the project whose steps
-   * reference one of the given devices. Shared by the walk fan-out above
-   * and the SNMP trap fan-out — the monitor↔device link lives in step JSON,
-   * so this fetches the project's Network Device monitors and filters
-   * server-side.
+   * The probe's ping result in the shape the criteria pipeline already
+   * speaks (ProbeMonitorResponse.pingResponse, as Ping monitors report it),
+   * so packet-loss criteria, metrics and evidence read it without a special
+   * case. Undefined when the probe ran no ping (older probes).
+   */
+  public static toPingMonitorResponse(
+    ping: NetworkDevicePingResult | undefined,
+  ): PingMonitorResponse | undefined {
+    if (!ping) {
+      return undefined;
+    }
+
+    /*
+     * Loss is the one number every reader wants; when the probe did not
+     * measure it, the verdict implies it - an answered ping lost nothing
+     * that matters, a silent one lost everything.
+     */
+    const packetLossPercent: number =
+      typeof ping.packetLossPercent === "number" &&
+      isFinite(ping.packetLossPercent)
+        ? ping.packetLossPercent
+        : ping.isOnline
+          ? 0
+          : 100;
+
+    return {
+      packetsSent: ping.packetsSent ?? 0,
+      packetsReceived: ping.packetsReceived ?? 0,
+      packetLossPercent: packetLossPercent,
+      avgRoundTripTimeInMs:
+        typeof ping.avgRttMs === "number" && isFinite(ping.avgRttMs)
+          ? ping.avgRttMs
+          : undefined,
+    };
+  }
+
+  /*
+   * Every enabled Network Device monitor in the project watching one of
+   * the given devices. Shared by the walk fan-out above and the SNMP trap
+   * fan-out.
+   *
+   * Two sources, unioned and deduped by id:
+   *
+   *   1. Indexed: monitors whose `autoProvisionedNetworkDeviceId` is one of
+   *      the devices - every monitor a template, an auto-import rule or an
+   *      alert policy provisioned. Served by the partial index on that
+   *      column, so it costs one index probe per device however many
+   *      monitors the project has.
+   *   2. Legacy: hand-made monitors, which carry the device link only in
+   *      their step JSON. Restricted to rows with NO
+   *      autoProvisionedNetworkDeviceId (the indexed query already covered
+   *      the rest), paged, and filtered server-side on the step JSON.
+   *
+   * The legacy scan is what this lookup used to be in full - one fetch of
+   * every Network Device monitor in the project per walk - and it shrinks
+   * as monitors move onto the column.
    */
   public static async findMonitorsWatchingDevices(data: {
     projectId: ObjectID;
@@ -276,27 +418,20 @@ export default class NetworkDeviceWalkUtil {
 
     const deviceIdSet: Set<string> = new Set(data.deviceIds);
 
-    const monitors: Array<Monitor> = await MonitorService.findBy({
-      query: {
+    const indexedMonitors: Array<Monitor> =
+      await NetworkDeviceWalkUtil.findNetworkDeviceMonitorsPaged({
         projectId: data.projectId,
         monitorType: MonitorType.NetworkDevice,
-      },
-      select: {
-        _id: true,
-        projectId: true,
-        monitorSteps: true,
-        disableActiveMonitoring: true,
-        disableActiveMonitoringBecauseOfManualIncident: true,
-        disableActiveMonitoringBecauseOfScheduledMaintenanceEvent: true,
-      },
-      limit: LIMIT_MAX,
-      skip: 0,
-      props: {
-        isRoot: true,
-      },
-    });
+        autoProvisionedNetworkDeviceId: QueryHelper.any(data.deviceIds),
+      });
 
-    return monitors.filter((monitor: Monitor) => {
+    const legacyMonitors: Array<Monitor> = (
+      await NetworkDeviceWalkUtil.findNetworkDeviceMonitorsPaged({
+        projectId: data.projectId,
+        monitorType: MonitorType.NetworkDevice,
+        autoProvisionedNetworkDeviceId: QueryHelper.isNull(),
+      })
+    ).filter((monitor: Monitor) => {
       return (monitor.monitorSteps?.data?.monitorStepsInstanceArray || []).some(
         (monitorStep: MonitorStep) => {
           const referencedDeviceId: string | undefined =
@@ -307,5 +442,64 @@ export default class NetworkDeviceWalkUtil {
         },
       );
     });
+
+    const seenMonitorIds: Set<string> = new Set();
+    const monitors: Array<Monitor> = [];
+
+    for (const monitor of [...indexedMonitors, ...legacyMonitors]) {
+      const monitorId: string | undefined = monitor.id?.toString();
+
+      if (!monitorId || seenMonitorIds.has(monitorId)) {
+        continue;
+      }
+
+      seenMonitorIds.add(monitorId);
+      monitors.push(monitor);
+    }
+
+    return monitors;
+  }
+
+  /*
+   * Every monitor matching the query, in id order so the pages stay
+   * stable while monitors are created or deleted underneath the scan.
+   */
+  private static async findNetworkDeviceMonitorsPaged(
+    query: Query<Monitor>,
+  ): Promise<Array<Monitor>> {
+    const monitors: Array<Monitor> = [];
+    let skip: number = 0;
+
+    for (;;) {
+      const page: Array<Monitor> = await MonitorService.findBy({
+        query: query,
+        select: {
+          _id: true,
+          projectId: true,
+          monitorSteps: true,
+          disableActiveMonitoring: true,
+          disableActiveMonitoringBecauseOfManualIncident: true,
+          disableActiveMonitoringBecauseOfScheduledMaintenanceEvent: true,
+        },
+        sort: {
+          _id: SortOrder.Ascending,
+        },
+        limit: NetworkDeviceWalkUtil.FAN_OUT_PAGE_SIZE,
+        skip: skip,
+        props: {
+          isRoot: true,
+        },
+      });
+
+      monitors.push(...page);
+
+      if (page.length < NetworkDeviceWalkUtil.FAN_OUT_PAGE_SIZE) {
+        break;
+      }
+
+      skip += page.length;
+    }
+
+    return monitors;
   }
 }