@oneuptime/common 10.0.71 → 10.0.72

package/Server/Utils/Monitor/Criteria/ServerMonitorCriteria.ts

@@ -208,6 +208,77 @@ export default class ServerMonitorCriteria {
       });
     }
 
+    if (
+      !(input.dataToProcess as ServerMonitorResponse)
+        .onlyCheckRequestReceivedAt &&
+      (input.criteriaFilter.checkOn === CheckOn.LoadAverage1Min ||
+        input.criteriaFilter.checkOn === CheckOn.LoadAverage5Min ||
+        input.criteriaFilter.checkOn === CheckOn.LoadAverage15Min)
+    ) {
+      threshold = CompareCriteria.convertToNumber(threshold);
+
+      const loadMetrics:
+        | { load1: number; load5: number; load15: number }
+        | undefined = (input.dataToProcess as ServerMonitorResponse)
+        .basicInfrastructureMetrics?.loadMetrics;
+
+      let currentLoad: number | undefined = undefined;
+      if (input.criteriaFilter.checkOn === CheckOn.LoadAverage1Min) {
+        currentLoad = loadMetrics?.load1;
+      } else if (input.criteriaFilter.checkOn === CheckOn.LoadAverage5Min) {
+        currentLoad = loadMetrics?.load5;
+      } else if (input.criteriaFilter.checkOn === CheckOn.LoadAverage15Min) {
+        currentLoad = loadMetrics?.load15;
+      }
+
+      const value: number | Array<number> =
+        (overTimeValue as Array<number>) || currentLoad || 0;
+
+      return CompareCriteria.compareCriteriaNumbers({
+        value: value,
+        threshold: threshold as number,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
+    if (
+      input.criteriaFilter.checkOn === CheckOn.SwapUsagePercent &&
+      !(input.dataToProcess as ServerMonitorResponse).onlyCheckRequestReceivedAt
+    ) {
+      threshold = CompareCriteria.convertToNumber(threshold);
+
+      const swapPercent: number | Array<number> =
+        (overTimeValue as Array<number>) ||
+        (input.dataToProcess as ServerMonitorResponse)
+          .basicInfrastructureMetrics?.memoryMetrics?.swapPercentUsed ||
+        0;
+
+      return CompareCriteria.compareCriteriaNumbers({
+        value: swapPercent,
+        threshold: threshold as number,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
+    if (
+      input.criteriaFilter.checkOn === CheckOn.CPUIoWaitPercent &&
+      !(input.dataToProcess as ServerMonitorResponse).onlyCheckRequestReceivedAt
+    ) {
+      threshold = CompareCriteria.convertToNumber(threshold);
+
+      const ioWaitPercent: number | Array<number> =
+        (overTimeValue as Array<number>) ||
+        (input.dataToProcess as ServerMonitorResponse)
+          .basicInfrastructureMetrics?.cpuMetrics?.timeIoWaitPercent ||
+        0;
+
+      return CompareCriteria.compareCriteriaNumbers({
+        value: ioWaitPercent,
+        threshold: threshold as number,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
     if (
       input.criteriaFilter.checkOn === CheckOn.ServerProcessName &&
       threshold &&

package/Server/Utils/Monitor/MonitorAlert.ts

@@ -26,6 +26,7 @@ import MonitorTemplateUtil from "./MonitorTemplateUtil";
 import { JSONObject } from "../../../Types/JSON";
 import OneUptimeDate from "../../../Types/Date";
 import MonitorEvaluationSummary from "../../../Types/Monitor/MonitorEvaluationSummary";
+import { PerSeriesCriteriaMatch } from "../../../Types/Probe/ProbeApiIngestResponse";
 
 export default class MonitorAlert {
   @CaptureSpan()
@@ -36,6 +37,7 @@ export default class MonitorAlert {
     criteriaInstance: MonitorCriteriaInstance | null;
     dataToProcess: DataToProcess;
     evaluationSummary?: MonitorEvaluationSummary | undefined;
+    breachingSeriesFingerprints?: Set<string> | undefined;
   }): Promise<Array<Alert>> {
     // check active alerts and if there are open alerts, do not cretae anothr alert.
     const openAlerts: Array<Alert> = await AlertService.findBy({
@@ -49,11 +51,14 @@ export default class MonitorAlert {
       limit: LIMIT_PER_PROJECT,
       select: {
         _id: true,
+        title: true,
         createdCriteriaId: true,
         projectId: true,
         alertNumber: true,
         alertNumberWithPrefix: true,
         currentAlertStateId: true,
+        seriesFingerprint: true,
+        seriesLabels: true,
       },
       props: {
         isRoot: true,
@@ -68,6 +73,7 @@ export default class MonitorAlert {
         autoResolveCriteriaInstanceIdAlertIdsDictionary:
           input.autoResolveCriteriaInstanceIdAlertIdsDictionary,
         criteriaInstance: input.criteriaInstance,
+        breachingSeriesFingerprints: input.breachingSeriesFingerprints,
       });
 
       if (shouldClose) {
@@ -106,6 +112,7 @@ export default class MonitorAlert {
     props: {
       telemetryQuery?: TelemetryQuery | undefined;
     };
+    matchesPerSeries?: Array<PerSeriesCriteriaMatch> | undefined;
   }): Promise<void> {
     const alertLogAttributes: LogAttributes = {
       projectId: input.monitor.projectId?.toString(),
@@ -116,6 +123,16 @@ export default class MonitorAlert {
       `${input.monitor.id?.toString()} - Check open alerts.`,
       alertLogAttributes,
     );
+
+    const breachingSeriesFingerprints: Set<string> | undefined =
+      input.matchesPerSeries
+        ? new Set<string>(
+            input.matchesPerSeries.map((m: PerSeriesCriteriaMatch) => {
+              return m.fingerprint;
+            }),
+          )
+        : undefined;
+
     // check active alerts and if there are open alerts, do not cretae anothr alert.
     const openAlerts: Array<Alert> =
       await this.checkOpenAlertsAndCloseIfResolved({
@@ -126,19 +143,31 @@ export default class MonitorAlert {
         criteriaInstance: input.criteriaInstance,
         dataToProcess: input.dataToProcess,
         evaluationSummary: input.evaluationSummary,
+        breachingSeriesFingerprints,
       });
 
-    if (input.criteriaInstance.data?.createAlerts) {
-      // create alerts
+    if (!input.criteriaInstance.data?.createAlerts) {
+      return;
+    }
 
-      for (const criteriaAlert of input.criteriaInstance.data?.alerts || []) {
-        // should create alert.
+    const seriesToProcess: Array<PerSeriesCriteriaMatch | undefined> =
+      input.matchesPerSeries && input.matchesPerSeries.length > 0
+        ? input.matchesPerSeries
+        : [undefined];
+
+    for (const criteriaAlert of input.criteriaInstance.data?.alerts || []) {
+      for (const seriesMatch of seriesToProcess) {
+        const seriesFingerprint: string | undefined = seriesMatch?.fingerprint;
+        const seriesLabels: JSONObject | undefined = seriesMatch?.labels;
+        const seriesRootCause: string =
+          seriesMatch?.rootCause || input.rootCause;
 
         const alreadyOpenAlert: Alert | undefined = openAlerts.find(
           (alert: Alert) => {
             return (
               alert.createdCriteriaId ===
-              input.criteriaInstance.data?.id.toString()
+                input.criteriaInstance.data?.id.toString() &&
+              (alert.seriesFingerprint || undefined) === seriesFingerprint
             );
           },
         );
@@ -156,9 +185,12 @@ export default class MonitorAlert {
         );
 
         if (hasAlreadyOpenAlert) {
+          const renderedAlertTitle: string =
+            alreadyOpenAlert?.title || criteriaAlert.title;
+
           input.evaluationSummary?.events.push({
             type: "alert-skipped",
-            title: `Alert already active: ${criteriaAlert.title}`,
+            title: `Alert already active: ${renderedAlertTitle}`,
             message:
               "Skipped creating a new alert because an active alert exists for this criteria.",
             relatedCriteriaId: input.criteriaInstance.data?.id,
@@ -183,6 +215,8 @@ export default class MonitorAlert {
           MonitorTemplateUtil.buildTemplateStorageMap({
             monitorType: input.monitor.monitorType!,
             dataToProcess: input.dataToProcess,
+            monitor: input.monitor,
+            seriesLabels,
           });
 
         alert.title = MonitorTemplateUtil.processTemplateString({
@@ -224,13 +258,20 @@ export default class MonitorAlert {
 
         alert.monitor = input.monitor;
         alert.projectId = input.monitor.projectId!;
-        alert.rootCause = input.rootCause;
+        alert.rootCause = seriesRootCause;
         alert.createdStateLog = JSON.parse(
           JSON.stringify(input.dataToProcess, null, 2),
         );
 
         alert.createdCriteriaId = input.criteriaInstance.data.id.toString();
 
+        if (seriesFingerprint) {
+          alert.seriesFingerprint = seriesFingerprint;
+        }
+        if (seriesLabels && Object.keys(seriesLabels).length > 0) {
+          alert.seriesLabels = seriesLabels;
+        }
+
         alert.onCallDutyPolicies =
           criteriaAlert.onCallPolicyIds?.map((id: ObjectID) => {
             const onCallPolicy: OnCallDutyPolicy = new OnCallDutyPolicy();
@@ -436,7 +477,50 @@ export default class MonitorAlert {
     openAlert: Alert;
     autoResolveCriteriaInstanceIdAlertIdsDictionary: Dictionary<Array<string>>;
     criteriaInstance: MonitorCriteriaInstance | null; // null if no criteia met.
+    breachingSeriesFingerprints?: Set<string> | undefined;
   }): boolean {
+    const openSeriesFingerprint: string | undefined =
+      input.openAlert.seriesFingerprint || undefined;
+
+    /*
+     * Per-series auto-resolve: when a breaching-series set is given and
+     * this alert has a fingerprint, resolve whenever the fingerprint is
+     * no longer in the set, independent of whether other series on the
+     * same monitor are still breaching.
+     */
+    if (
+      input.breachingSeriesFingerprints !== undefined &&
+      openSeriesFingerprint
+    ) {
+      const stillBreaching: boolean = input.breachingSeriesFingerprints.has(
+        openSeriesFingerprint,
+      );
+
+      if (stillBreaching) {
+        return false;
+      }
+
+      if (!input.openAlert.createdCriteriaId?.toString()) {
+        return false;
+      }
+
+      const autoResolveTemplates: Array<string> | undefined =
+        input.autoResolveCriteriaInstanceIdAlertIdsDictionary[
+          input.openAlert.createdCriteriaId.toString()
+        ];
+
+      /*
+       * Alert auto-resolve lists templates by criteria; presence of any
+       * template for this criteria means "this criteria's alerts are
+       * configured to auto-resolve", so resolve this series.
+       */
+      if (autoResolveTemplates && autoResolveTemplates.length > 0) {
+        return true;
+      }
+
+      return false;
+    }
+
     if (
       input.openAlert.createdCriteriaId?.toString() ===
       input.criteriaInstance?.data?.id.toString()

package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts

@@ -9,7 +9,9 @@ import SSLMonitorCriteria from "./Criteria/SSLMonitorCriteria";
 import ServerMonitorCriteria from "./Criteria/ServerMonitorCriteria";
 import SyntheticMonitoringCriteria from "./Criteria/SyntheticMonitor";
 import LogMonitorCriteria from "./Criteria/LogMonitorCriteria";
-import MetricMonitorCriteria from "./Criteria/MetricMonitorCriteria";
+import MetricMonitorCriteria, {
+  MetricSeriesEvaluationResult,
+} from "./Criteria/MetricMonitorCriteria";
 import TraceMonitorCriteria from "./Criteria/TraceMonitorCriteria";
 import ExceptionMonitorCriteria from "./Criteria/ExceptionMonitorCriteria";
 import ProfileMonitorCriteria from "./Criteria/ProfileMonitorCriteria";
@@ -31,7 +33,9 @@ import MonitorEvaluationSummary, {
   MonitorEvaluationEvent,
   MonitorEvaluationFilterResult,
 } from "../../../Types/Monitor/MonitorEvaluationSummary";
-import ProbeApiIngestResponse from "../../../Types/Probe/ProbeApiIngestResponse";
+import ProbeApiIngestResponse, {
+  PerSeriesCriteriaMatch,
+} from "../../../Types/Probe/ProbeApiIngestResponse";
 import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
 import RequestFailedDetails from "../../../Types/Probe/RequestFailedDetails";
 import IncomingMonitorRequest from "../../../Types/Monitor/IncomingMonitor/IncomingMonitorRequest";
@@ -50,6 +54,7 @@ import MetricMonitorResponse, {
   KubernetesAffectedResource,
   KubernetesResourceBreakdown,
 } from "../../../Types/Monitor/MetricMonitor/MetricMonitorResponse";
+import MetricSeriesResult from "../../../Types/Monitor/MetricMonitor/MetricSeriesResult";
 import MetricCriteriaContext, {
   MetricBreachingSample,
   MetricComponent,
@@ -156,6 +161,26 @@ ${contextBlock}
 **Cause**: ${(input.dataToProcess as ProbeMonitorResponse).failureCause || ""}
 `;
         }
+
+        /*
+         * When this is a metric-style monitor with per-series results,
+         * compute the per-series match list so MonitorResource can fan
+         * out one incident per affected series. We do this *after* the
+         * scalar criteriaMetId/rootCause is populated so legacy readers
+         * still see a usable response if perSeriesMatches is ignored.
+         */
+        const perSeriesMatches: Array<PerSeriesCriteriaMatch> =
+          MonitorCriteriaEvaluator.collectPerSeriesMatches({
+            dataToProcess: input.dataToProcess,
+            monitor: input.monitor,
+            monitorStep: input.monitorStep,
+            criteriaInstance,
+          });
+
+        if (perSeriesMatches.length > 0) {
+          input.probeApiIngestResponse.perSeriesMatches = perSeriesMatches;
+        }
+
         break;
       }
     }
@@ -163,6 +188,150 @@ ${contextBlock}
     return input.probeApiIngestResponse;
   }
 
+  /**
+   * For metric-backed monitors (Metrics/Kubernetes/Docker) with
+   * per-series aggregated results, re-evaluate the matched criteria
+   * once per series and return one entry per series that breached.
+   * Returns an empty array when the monitor is not series-aware or
+   * no series matched — the caller falls back to the single-incident
+   * path in that case.
+   */
+  private static collectPerSeriesMatches(input: {
+    dataToProcess: DataToProcess;
+    monitor: Monitor;
+    monitorStep: MonitorStep;
+    criteriaInstance: MonitorCriteriaInstance;
+  }): Array<PerSeriesCriteriaMatch> {
+    if (
+      input.monitor.monitorType !== MonitorType.Metrics &&
+      input.monitor.monitorType !== MonitorType.Kubernetes &&
+      input.monitor.monitorType !== MonitorType.Docker
+    ) {
+      return [];
+    }
+
+    const metricResponse: MetricMonitorResponse =
+      input.dataToProcess as MetricMonitorResponse;
+
+    const seriesBreakdown: Array<MetricSeriesResult> | undefined =
+      metricResponse.seriesBreakdown;
+
+    if (!seriesBreakdown || seriesBreakdown.length === 0) {
+      return [];
+    }
+
+    const criteriaId: string | undefined = input.criteriaInstance.data?.id;
+    if (!criteriaId) {
+      return [];
+    }
+
+    const filterCondition: FilterCondition =
+      input.criteriaInstance.data?.filterCondition || FilterCondition.All;
+
+    const filters: Array<CriteriaFilter> =
+      input.criteriaInstance.data?.filters || [];
+
+    /*
+     * Evaluate every metric-value filter against every series. We only
+     * handle CheckOn.MetricValue for now — the criteria evaluator's
+     * other filter types don't carry series information.
+     */
+    const metricFilters: Array<CriteriaFilter> = filters.filter(
+      (f: CriteriaFilter) => {
+        return f.checkOn === CheckOn.MetricValue;
+      },
+    );
+
+    if (metricFilters.length === 0) {
+      return [];
+    }
+
+    /*
+     * Key: fingerprint. Value: results of each metric filter for that
+     * series, tracked so we can apply FilterCondition.All/Any per series.
+     */
+    const resultsByFingerprint: Map<
+      string,
+      {
+        labels: JSONObject;
+        filterResults: Array<MetricSeriesEvaluationResult>;
+      }
+    > = new Map();
+
+    for (const criteriaFilter of metricFilters) {
+      const evaluations: Array<MetricSeriesEvaluationResult> =
+        MetricMonitorCriteria.evaluateAllSeries({
+          dataToProcess: input.dataToProcess,
+          criteriaFilter,
+          monitorStep: input.monitorStep,
+        });
+
+      for (const evaluation of evaluations) {
+        const fingerprint: string | undefined = evaluation.fingerprint;
+        if (!fingerprint) {
+          continue;
+        }
+
+        const existing:
+          | {
+              labels: JSONObject;
+              filterResults: Array<MetricSeriesEvaluationResult>;
+            }
+          | undefined = resultsByFingerprint.get(fingerprint);
+
+        if (existing) {
+          existing.filterResults.push(evaluation);
+        } else {
+          resultsByFingerprint.set(fingerprint, {
+            labels: evaluation.labels,
+            filterResults: [evaluation],
+          });
+        }
+      }
+    }
+
+    const matches: Array<PerSeriesCriteriaMatch> = [];
+
+    for (const [fingerprint, entry] of resultsByFingerprint) {
+      // Were all/any of the metric filters met for this series?
+      const matched: Array<MetricSeriesEvaluationResult> =
+        entry.filterResults.filter((r: MetricSeriesEvaluationResult) => {
+          return r.rootCause !== null;
+        });
+
+      const seriesMatched: boolean =
+        filterCondition === FilterCondition.All
+          ? matched.length === entry.filterResults.length &&
+            entry.filterResults.length > 0
+          : matched.length > 0;
+
+      if (!seriesMatched) {
+        continue;
+      }
+
+      /*
+       * For the per-series rootCause, concatenate the matched filter
+       * messages so the incident message reflects exactly what
+       * breached on this specific series.
+       */
+      const rootCauseLines: Array<string> = matched.map(
+        (r: MetricSeriesEvaluationResult) => {
+          return `- ${r.rootCause}`;
+        },
+      );
+
+      matches.push({
+        criteriaMetId: criteriaId,
+        fingerprint,
+        labels: entry.labels,
+        rootCause: rootCauseLines.join("\n"),
+        metricContext: matched[0]?.context,
+      });
+    }
+
+    return matches;
+  }
+
   private static async processMonitorCriteriaInstance(input: {
     dataToProcess: DataToProcess;
     monitorStep: MonitorStep;