@oneuptime/common 12.0.24 → 12.0.26

package/Server/Utils/Monitor/MonitorResource.ts

@@ -25,7 +25,12 @@ import MonitorType, {
 import ServerMonitorResponse from "../../../Types/Monitor/ServerMonitor/ServerMonitorResponse";
 import ObjectID from "../../../Types/ObjectID";
 import { JSONObject } from "../../../Types/JSON";
-import ProbeApiIngestResponse from "../../../Types/Probe/ProbeApiIngestResponse";
+import ProbeApiIngestResponse, {
+  MatchedCriteriaResult,
+  PerSeriesCriteriaMatch,
+} from "../../../Types/Probe/ProbeApiIngestResponse";
+import Incident from "../../../Models/DatabaseModels/Incident";
+import Alert from "../../../Models/DatabaseModels/Alert";
 import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
 import Monitor from "../../../Models/DatabaseModels/Monitor";
 import MonitorProbe, {
@@ -965,6 +970,67 @@ export default class MonitorResourceUtil {
           });
         }
 
+        /*
+         * The criteria this evaluation is going to act on.
+         *
+         * Grouped monitors evaluate every criteria and can match several
+         * at once — host A critical while host B is only warning — so
+         * they hand back a list. Everything else keeps the historical
+         * single-winner shape, reconstructed here so both cases go down
+         * exactly the same code path below.
+         */
+        const criteriaFanOut: Array<MatchedCriteriaResult> =
+          response.matchedCriteria ?? [
+            {
+              criteriaId: response.criteriaMetId!,
+              rootCause: response.rootCause,
+              perSeriesMatches: response.perSeriesMatches ?? [],
+            },
+          ];
+
+        const isPerSeriesEvaluation: boolean =
+          response.perSeriesMatches !== undefined;
+
+        /*
+         * Every series breaching anything on this tick, and the same
+         * broken out per criteria.
+         *
+         * The resolve pass needs the per-criteria view: "is host B still
+         * breaching?" has to be answered against the criteria that
+         * raised host B's alert, not against whichever criteria happened
+         * to win the tick. Answering it against the winner resolved a
+         * still-valid warning alert the moment another host went
+         * critical. A criteria that ran and matched nothing maps to an
+         * empty set — that is what lets a recovered series resolve —
+         * while a criteria that never ran is absent, and its records are
+         * left alone.
+         */
+        const breachingSeriesFingerprintsByCriteriaId: Dictionary<Set<string>> =
+          {};
+
+        if (response.evaluatedCriteriaIds) {
+          for (const evaluatedCriteriaId of response.evaluatedCriteriaIds) {
+            breachingSeriesFingerprintsByCriteriaId[evaluatedCriteriaId] =
+              new Set<string>();
+          }
+        }
+
+        const allBreachingSeriesFingerprints: Set<string> = new Set<string>();
+
+        for (const matched of criteriaFanOut) {
+          const forCriteria: Set<string> =
+            breachingSeriesFingerprintsByCriteriaId[matched.criteriaId] ||
+            new Set<string>();
+
+          breachingSeriesFingerprintsByCriteriaId[matched.criteriaId] =
+            forCriteria;
+
+          for (const seriesMatch of matched.perSeriesMatches) {
+            forCriteria.add(seriesMatch.fingerprint);
+            allBreachingSeriesFingerprints.add(seriesMatch.fingerprint);
+          }
+        }
+
         /*
          * For grouped metric monitors, work out which breaching series
          * belong to a resource that is currently inside an ongoing
@@ -977,7 +1043,11 @@ export default class MonitorResourceUtil {
         const suppressedSeriesFingerprints: Set<string> =
           await MonitorMaintenanceSuppression.getSuppressedSeriesFingerprints({
             projectId: monitor.projectId!,
-            matchesPerSeries: response.perSeriesMatches,
+            matchesPerSeries: isPerSeriesEvaluation
+              ? criteriaFanOut.flatMap((matched: MatchedCriteriaResult) => {
+                  return matched.perSeriesMatches;
+                })
+              : undefined,
           });
 
         /*
@@ -1011,54 +1081,130 @@ export default class MonitorResourceUtil {
             probeName: probeName,
           });
 
-        await MonitorIncident.criteriaMetCreateIncidentsAndUpdateMonitorStatus({
-          monitor: monitor,
-          rootCause: response.rootCause,
-          dataToProcess: dataToProcess,
-          autoResolveCriteriaInstanceIdIncidentIdsDictionary,
-          criteriaInstance: matchedCriteriaInstance,
-          evaluationSummary: evaluationSummary,
-          monitorSummary: monitorSummary,
-          props: {
-            telemetryQuery: telemetryQuery,
-          },
-          matchesPerSeries: response.perSeriesMatches,
-          suppressedSeriesFingerprints,
-          dependencySuppression,
-          /*
-           * Incoming-request grouping is event-driven: a webhook describes
-           * only the keys in its payload, so absence from this tick is not
-           * recovery. Skip the snapshot absence-resolve pass; grouped
-           * incidents are resolved explicitly via the resolution block
-           * above. Per-key create + dedupe still run from matchesPerSeries.
-           */
-          disableSeriesAbsenceResolution:
-            monitor.monitorType === MonitorType.IncomingRequest,
-        });
+        /*
+         * Incoming-request grouping is event-driven: a webhook describes
+         * only the keys in its payload, so absence from this tick is not
+         * recovery. Skip the snapshot absence-resolve pass; grouped
+         * incidents/alerts are resolved explicitly via the resolution
+         * block above. Per-key create + dedupe still run from
+         * matchesPerSeries.
+         */
+        const disableSeriesAbsenceResolution: boolean =
+          monitor.monitorType === MonitorType.IncomingRequest;
 
-        await MonitorAlert.criteriaMetCreateAlertsAndUpdateMonitorStatus({
-          monitor: monitor,
-          rootCause: response.rootCause,
-          dataToProcess: dataToProcess,
-          autoResolveCriteriaInstanceIdAlertIdsDictionary,
-          criteriaInstance: criteriaInstanceAlertMap[response.criteriaMetId!]!,
-          evaluationSummary: evaluationSummary,
-          monitorSummary: monitorSummary,
-          props: {
-            telemetryQuery: telemetryQuery,
-          },
-          matchesPerSeries: response.perSeriesMatches,
-          suppressedSeriesFingerprints,
-          dependencySuppression,
-          /*
-           * Incoming-request grouping is event-driven (see the incident
-           * create call above): skip the snapshot absence-resolve pass for
-           * webhooks; grouped alerts are resolved explicitly via the
-           * resolution block above. Per-key create + dedupe still run.
-           */
-          disableSeriesAbsenceResolution:
-            monitor.monitorType === MonitorType.IncomingRequest,
-        });
+        const breachingSeriesFingerprints: Set<string> | undefined =
+          isPerSeriesEvaluation && !disableSeriesAbsenceResolution
+            ? allBreachingSeriesFingerprints
+            : undefined;
+
+        /*
+         * Resolve first, once, for the whole evaluation — then create.
+         *
+         * The resolve pass has to see every criteria's breaching set at
+         * the same time. Running it once per matching criteria would let
+         * each criteria absence-resolve the records the others just
+         * justified. The creators are handed the survivors so they never
+         * dedupe a new alert against one this pass has already closed.
+         */
+        const openIncidents: Array<Incident> =
+          await MonitorIncident.checkOpenIncidentsAndCloseIfResolved({
+            monitorId: monitor.id!,
+            autoResolveCriteriaInstanceIdIncidentIdsDictionary,
+            rootCause: response.rootCause,
+            criteriaInstance: matchedCriteriaInstance,
+            dataToProcess: dataToProcess,
+            evaluationSummary: evaluationSummary,
+            breachingSeriesFingerprints,
+            breachingSeriesFingerprintsByCriteriaId: response.matchedCriteria
+              ? breachingSeriesFingerprintsByCriteriaId
+              : undefined,
+            disableSeriesAbsenceResolution,
+          });
+
+        const openAlerts: Array<Alert> =
+          await MonitorAlert.checkOpenAlertsAndCloseIfResolved({
+            monitorId: monitor.id!,
+            autoResolveCriteriaInstanceIdAlertIdsDictionary,
+            rootCause: response.rootCause,
+            criteriaInstance: matchedCriteriaInstance,
+            dataToProcess: dataToProcess,
+            evaluationSummary: evaluationSummary,
+            breachingSeriesFingerprints,
+            breachingSeriesFingerprintsByCriteriaId: response.matchedCriteria
+              ? breachingSeriesFingerprintsByCriteriaId
+              : undefined,
+            disableSeriesAbsenceResolution,
+          });
+
+        /*
+         * De-escalation: a host that breaches both "critical" and
+         * "warning" gets one record, from the first (highest priority)
+         * criteria that claims it — criteria are in user-defined order
+         * and the first match is the one that also sets the monitor's
+         * status. Tracked separately for incidents and alerts because a
+         * criteria can create one without the other.
+         */
+        const seriesClaimedByIncidentCriteria: Set<string> = new Set<string>();
+        const seriesClaimedByAlertCriteria: Set<string> = new Set<string>();
+
+        for (const matched of criteriaFanOut) {
+          const criteriaInstance: MonitorCriteriaInstance | undefined =
+            criteriaInstanceMap[matched.criteriaId];
+
+          if (!criteriaInstance) {
+            continue;
+          }
+
+          await MonitorIncident.criteriaMetCreateIncidentsAndUpdateMonitorStatus(
+            {
+              monitor: monitor,
+              rootCause: matched.rootCause,
+              dataToProcess: dataToProcess,
+              autoResolveCriteriaInstanceIdIncidentIdsDictionary,
+              criteriaInstance: criteriaInstance,
+              evaluationSummary: evaluationSummary,
+              monitorSummary: monitorSummary,
+              props: {
+                telemetryQuery: telemetryQuery,
+              },
+              matchesPerSeries: isPerSeriesEvaluation
+                ? MonitorResourceUtil.claimUnclaimedSeries({
+                    matches: matched.perSeriesMatches,
+                    claimed: seriesClaimedByIncidentCriteria,
+                    isClaiming: criteriaInstance.data?.createIncidents === true,
+                  })
+                : undefined,
+              openIncidents,
+              suppressedSeriesFingerprints,
+              dependencySuppression,
+              disableSeriesAbsenceResolution,
+            },
+          );
+
+          await MonitorAlert.criteriaMetCreateAlertsAndUpdateMonitorStatus({
+            monitor: monitor,
+            rootCause: matched.rootCause,
+            dataToProcess: dataToProcess,
+            autoResolveCriteriaInstanceIdAlertIdsDictionary,
+            criteriaInstance: criteriaInstanceAlertMap[matched.criteriaId]!,
+            evaluationSummary: evaluationSummary,
+            monitorSummary: monitorSummary,
+            props: {
+              telemetryQuery: telemetryQuery,
+            },
+            matchesPerSeries: isPerSeriesEvaluation
+              ? MonitorResourceUtil.claimUnclaimedSeries({
+                  matches: matched.perSeriesMatches,
+                  claimed: seriesClaimedByAlertCriteria,
+                  isClaiming: criteriaInstance.data?.createAlerts === true,
+                })
+              : undefined,
+            openAlerts,
+            suppressedSeriesFingerprints,
+            dependencySuppression,
+            disableSeriesAbsenceResolution,
+          });
+        }
       } else if (
         !response.criteriaMetId &&
         /*
@@ -1361,6 +1507,37 @@ export default class MonitorResourceUtil {
   }
 
   @CaptureSpan()
+  /**
+   * Take the series this criteria may act on, given what earlier
+   * (higher-priority) criteria have already claimed.
+   *
+   * A host breaching both "critical" and "warning" should page once,
+   * from the more severe band. Criteria that create nothing claim
+   * nothing, so a passive recovery criteria never steals a series from
+   * the criteria that would have alerted on it.
+   */
+  private static claimUnclaimedSeries(input: {
+    matches: Array<PerSeriesCriteriaMatch>;
+    claimed: Set<string>;
+    isClaiming: boolean;
+  }): Array<PerSeriesCriteriaMatch> {
+    if (!input.isClaiming) {
+      return input.matches;
+    }
+
+    const unclaimed: Array<PerSeriesCriteriaMatch> = input.matches.filter(
+      (match: PerSeriesCriteriaMatch) => {
+        return !input.claimed.has(match.fingerprint);
+      },
+    );
+
+    for (const match of unclaimed) {
+      input.claimed.add(match.fingerprint);
+    }
+
+    return unclaimed;
+  }
+
   private static async checkProbeAgreement(input: {
     monitor: Monitor;
     monitorStep: MonitorStep;

package/Server/Utils/Monitor/NetworkDeviceWalkUtil.ts

@@ -88,12 +88,48 @@ export default class NetworkDeviceWalkUtil {
      * actually walked interfaces. Keeping the last GOOD counters across
      * failed polls means rates resume with a correct (longer-window) delta
      * instead of losing a cycle.
+     *
+     * Hook-free single-statement write, deliberately. A 48-interface walk
+     * log is ~28 KB of jsonb, stored out of line: the full updateOneById
+     * pipeline SELECTs the row before writing it (the column is in the
+     * payload), so every poll detoasted the PREVIOUS log a second time —
+     * it was already read above — only to overwrite it. Measured against
+     * the seeded 80,000-device fleet, that pre-read costs 8 buffer hits and
+     * ~0.6 ms where the same lookup without the column costs 4 and
+     * ~0.06 ms; at 267 walks/sec it is pure waste on the product's hottest
+     * table. It does NOT shrink the ~12 KB of WAL the rewrite itself emits
+     * (a plain one-column update is 144 B) — that is what storing less
+     * would fix, and it is a separate change.
+     *
+     * Nothing rides on the hooks for THIS write. NetworkDevice declares
+     * neither @EnableWorkflow nor @EnableAuditLog, so _updateBy's workflow,
+     * realtime and audit branches are inert for the model, and the payload
+     * carries exactly one column: no site-rule identity column
+     * (hostname/name/sysName), no siteId, no monitor binding, no
+     * monitoringMethod. Both of NetworkDeviceService's real update hooks —
+     * site-assignment re-evaluation and the monitor-status stamp — are
+     * therefore no-ops here. The inventory sync below still writes those
+     * columns through the hooked path, which is where they have to stay.
+     * Do not add a column to this payload: anything that lands here stops
+     * firing hooks silently.
+     *
+     * One thing IS lost, deliberately: `updateColumnsByIdWithoutHooks` does
+     * not bump the @VersionColumn, so this write leaves `version` alone where
+     * `updateOneById` incremented it. Nothing in the product reads
+     * NetworkDevice.version, and `sanitizeUpdateData` strips it from client
+     * payloads. Note this is the opposite call from the one the interface
+     * upsert makes a file away — NetworkInterfaceService hand-writes
+     * `version + 1` in its raw statements — and the difference is intended: a
+     * batched write REPLACES a path that bumped it and is the row's normal
+     * update path, whereas this is a delta-baseline cache write that happens
+     * every five minutes and would otherwise be the only thing moving the
+     * counter at all.
      */
     if (
       data.snmpResponse.interfaces &&
       data.snmpResponse.interfaces.length > 0
     ) {
-      await NetworkDeviceService.updateOneById({
+      await NetworkDeviceService.updateColumnsByIdWithoutHooks({
         id: device.id,
         data: {
           lastWalkLog: {
@@ -101,8 +137,15 @@ export default class NetworkDeviceWalkUtil {
             monitoredAt: OneUptimeDate.getCurrentDate(),
           },
         } as any,
-        props: {
-          isRoot: true,
+        /*
+         * The hooked path finds rows with `withDeleted: false`, so a device
+         * deleted between the read at the top of this walk and here was
+         * never written to. The raw path matches on `_id` alone, so keep
+         * that guard explicitly — a walk still in flight must not push a
+         * fresh log back onto a row the user just deleted.
+         */
+        expectedData: {
+          deletedAt: null,
         },
       });
     }

package/Server/Utils/Monitor/NetworkInventoryUtil.ts

@@ -1,9 +1,9 @@
 import NetworkDevice from "../../../Models/DatabaseModels/NetworkDevice";
-import NetworkInterface from "../../../Models/DatabaseModels/NetworkInterface";
 import NetworkDeviceService from "../../Services/NetworkDeviceService";
 import NetworkEndpointService from "../../Services/NetworkEndpointService";
-import NetworkInterfaceService from "../../Services/NetworkInterfaceService";
-import LIMIT_MAX from "../../../Types/Database/LimitMax";
+import NetworkInterfaceService, {
+  InterfaceWalkUpsertResult,
+} from "../../Services/NetworkInterfaceService";
 import SnmpInterface from "../../../Types/Monitor/SnmpMonitor/SnmpInterface";
 import SnmpMonitorResponse from "../../../Types/Monitor/SnmpMonitor/SnmpMonitorResponse";
 import LldpNeighbor from "../../../Types/Monitor/SnmpMonitor/LldpNeighbor";
@@ -252,107 +252,28 @@ export default class NetworkInventoryUtil {
       }
 
       if (walkedInterfaces.length > 0) {
-        // --- Interface upsert ---
-        const existingInterfaces: Array<NetworkInterface> =
-          await NetworkInterfaceService.findBy({
-            query: {
-              networkDeviceId: deviceId,
-            },
-            select: {
-              _id: true,
-              interfaceIndex: true,
-              isMonitored: true,
-            },
-            limit: LIMIT_MAX,
-            skip: 0,
-            props: {
-              isRoot: true,
-            },
+        /*
+         * --- Interface upsert ---
+         * One SELECT plus one batched INSERT/UPDATE per 500 rows, inside the
+         * service. This was a per-interface create()/updateOneById() loop —
+         * 101 statements for a 50-port switch, because _updateBy SELECTs
+         * before every UPDATE — which is a hard wall once an operator raises
+         * DEVICE_POLL_FETCH_LIMIT and the fleet actually polls on its
+         * configured interval. Interfaces that exist in inventory but were
+         * NOT in this walk are still left exactly as they were: nothing here
+         * deletes or ages them out.
+         */
+        const upsertResult: InterfaceWalkUpsertResult =
+          await NetworkInterfaceService.upsertWalkedInterfaces({
+            projectId: data.projectId,
+            deviceId: deviceId,
+            walkedInterfaces: walkedInterfaces,
+            now: now,
           });
 
-        const existingByIndex: Map<number, NetworkInterface> = new Map();
-        for (const existing of existingInterfaces) {
-          if (existing.interfaceIndex !== undefined) {
-            existingByIndex.set(existing.interfaceIndex, existing);
-          }
-        }
-
-        const unmonitoredIndexes: Set<number> = new Set();
-
-        for (const walked of walkedInterfaces) {
-          const existing: NetworkInterface | undefined = existingByIndex.get(
-            walked.interfaceIndex,
-          );
-
-          if (existing && existing.isMonitored === false) {
-            unmonitoredIndexes.add(walked.interfaceIndex);
-          }
-
-          const interfaceData: Record<string, unknown> = {
-            name: (walked.name || "").substring(0, 100),
-            alias: walked.alias ? walked.alias.substring(0, 100) : null,
-            macAddress: walked.macAddress
-              ? walked.macAddress.substring(0, 100)
-              : null,
-            interfaceType: walked.interfaceType ?? null,
-            isOperationallyUp: walked.isOperationallyUp,
-            isAdministrativelyUp: walked.isAdministrativelyUp,
-            speedInMbps:
-              walked.speedInBitsPerSecond !== undefined
-                ? walked.speedInBitsPerSecond / 1000000
-                : null,
-            inRateMbps:
-              walked.inBitsPerSecond !== undefined
-                ? Math.round((walked.inBitsPerSecond / 1000000) * 1000) / 1000
-                : null,
-            outRateMbps:
-              walked.outBitsPerSecond !== undefined
-                ? Math.round((walked.outBitsPerSecond / 1000000) * 1000) / 1000
-                : null,
-            utilizationPercent: walked.utilizationPercent ?? null,
-            errorsPerSecond: walked.errorsPerSecond ?? null,
-            lastSeenAt: now,
-          };
-
-          if (existing && existing.id) {
-            await NetworkInterfaceService.updateOneById({
-              id: existing.id,
-              data: interfaceData as any,
-              props: {
-                isRoot: true,
-              },
-            });
-          } else {
-            const newInterface: NetworkInterface = new NetworkInterface();
-            newInterface.projectId = data.projectId;
-            newInterface.networkDeviceId = deviceId;
-            newInterface.interfaceIndex = walked.interfaceIndex;
-            newInterface.name = (walked.name || "").substring(0, 100);
-            if (walked.alias) {
-              newInterface.alias = walked.alias.substring(0, 100);
-            }
-            if (walked.macAddress) {
-              newInterface.macAddress = walked.macAddress.substring(0, 100);
-            }
-            if (walked.interfaceType !== undefined) {
-              newInterface.interfaceType = walked.interfaceType;
-            }
-            newInterface.isMonitored = true;
-            newInterface.isOperationallyUp = walked.isOperationallyUp;
-            newInterface.isAdministrativelyUp = walked.isAdministrativelyUp;
-            if (walked.speedInBitsPerSecond !== undefined) {
-              newInterface.speedInMbps = walked.speedInBitsPerSecond / 1000000;
-            }
-            newInterface.lastSeenAt = now;
-
-            await NetworkInterfaceService.create({
-              data: newInterface,
-              props: {
-                isRoot: true,
-              },
-            });
-          }
-        }
+        const unmonitoredIndexes: Set<number> = new Set(
+          upsertResult.unmonitoredInterfaceIndexes,
+        );
 
         /*
          * Prune the in-flight response to monitored interfaces only, so