@oneuptime/common 10.0.70 → 10.0.72

package/Server/Utils/Monitor/Criteria/ServerMonitorCriteria.ts

@@ -208,6 +208,77 @@ export default class ServerMonitorCriteria {
       });
     }
 
+    if (
+      !(input.dataToProcess as ServerMonitorResponse)
+        .onlyCheckRequestReceivedAt &&
+      (input.criteriaFilter.checkOn === CheckOn.LoadAverage1Min ||
+        input.criteriaFilter.checkOn === CheckOn.LoadAverage5Min ||
+        input.criteriaFilter.checkOn === CheckOn.LoadAverage15Min)
+    ) {
+      threshold = CompareCriteria.convertToNumber(threshold);
+
+      const loadMetrics:
+        | { load1: number; load5: number; load15: number }
+        | undefined = (input.dataToProcess as ServerMonitorResponse)
+        .basicInfrastructureMetrics?.loadMetrics;
+
+      let currentLoad: number | undefined = undefined;
+      if (input.criteriaFilter.checkOn === CheckOn.LoadAverage1Min) {
+        currentLoad = loadMetrics?.load1;
+      } else if (input.criteriaFilter.checkOn === CheckOn.LoadAverage5Min) {
+        currentLoad = loadMetrics?.load5;
+      } else if (input.criteriaFilter.checkOn === CheckOn.LoadAverage15Min) {
+        currentLoad = loadMetrics?.load15;
+      }
+
+      const value: number | Array<number> =
+        (overTimeValue as Array<number>) || currentLoad || 0;
+
+      return CompareCriteria.compareCriteriaNumbers({
+        value: value,
+        threshold: threshold as number,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
+    if (
+      input.criteriaFilter.checkOn === CheckOn.SwapUsagePercent &&
+      !(input.dataToProcess as ServerMonitorResponse).onlyCheckRequestReceivedAt
+    ) {
+      threshold = CompareCriteria.convertToNumber(threshold);
+
+      const swapPercent: number | Array<number> =
+        (overTimeValue as Array<number>) ||
+        (input.dataToProcess as ServerMonitorResponse)
+          .basicInfrastructureMetrics?.memoryMetrics?.swapPercentUsed ||
+        0;
+
+      return CompareCriteria.compareCriteriaNumbers({
+        value: swapPercent,
+        threshold: threshold as number,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
+    if (
+      input.criteriaFilter.checkOn === CheckOn.CPUIoWaitPercent &&
+      !(input.dataToProcess as ServerMonitorResponse).onlyCheckRequestReceivedAt
+    ) {
+      threshold = CompareCriteria.convertToNumber(threshold);
+
+      const ioWaitPercent: number | Array<number> =
+        (overTimeValue as Array<number>) ||
+        (input.dataToProcess as ServerMonitorResponse)
+          .basicInfrastructureMetrics?.cpuMetrics?.timeIoWaitPercent ||
+        0;
+
+      return CompareCriteria.compareCriteriaNumbers({
+        value: ioWaitPercent,
+        threshold: threshold as number,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
     if (
       input.criteriaFilter.checkOn === CheckOn.ServerProcessName &&
       threshold &&

package/Server/Utils/Monitor/MonitorAlert.ts

@@ -16,7 +16,9 @@ import { TelemetryQuery } from "../../../Types/Telemetry/TelemetryQuery";
 import { DisableAutomaticAlertCreation } from "../../EnvironmentConfig";
 import AlertService from "../../Services/AlertService";
 import AlertSeverityService from "../../Services/AlertSeverityService";
+import AlertStateService from "../../Services/AlertStateService";
 import AlertStateTimelineService from "../../Services/AlertStateTimelineService";
+import AlertState from "../../../Models/DatabaseModels/AlertState";
 import logger, { LogAttributes } from "../Logger";
 import CaptureSpan from "../Telemetry/CaptureSpan";
 import DataToProcess from "./DataToProcess";
@@ -24,6 +26,7 @@ import MonitorTemplateUtil from "./MonitorTemplateUtil";
 import { JSONObject } from "../../../Types/JSON";
 import OneUptimeDate from "../../../Types/Date";
 import MonitorEvaluationSummary from "../../../Types/Monitor/MonitorEvaluationSummary";
+import { PerSeriesCriteriaMatch } from "../../../Types/Probe/ProbeApiIngestResponse";
 
 export default class MonitorAlert {
   @CaptureSpan()
@@ -34,6 +37,7 @@ export default class MonitorAlert {
     criteriaInstance: MonitorCriteriaInstance | null;
     dataToProcess: DataToProcess;
     evaluationSummary?: MonitorEvaluationSummary | undefined;
+    breachingSeriesFingerprints?: Set<string> | undefined;
   }): Promise<Array<Alert>> {
     // check active alerts and if there are open alerts, do not cretae anothr alert.
     const openAlerts: Array<Alert> = await AlertService.findBy({
@@ -47,10 +51,14 @@ export default class MonitorAlert {
       limit: LIMIT_PER_PROJECT,
       select: {
         _id: true,
+        title: true,
         createdCriteriaId: true,
         projectId: true,
         alertNumber: true,
         alertNumberWithPrefix: true,
+        currentAlertStateId: true,
+        seriesFingerprint: true,
+        seriesLabels: true,
       },
       props: {
         isRoot: true,
@@ -65,6 +73,7 @@ export default class MonitorAlert {
         autoResolveCriteriaInstanceIdAlertIdsDictionary:
           input.autoResolveCriteriaInstanceIdAlertIdsDictionary,
         criteriaInstance: input.criteriaInstance,
+        breachingSeriesFingerprints: input.breachingSeriesFingerprints,
       });
 
       if (shouldClose) {
@@ -103,6 +112,7 @@ export default class MonitorAlert {
     props: {
       telemetryQuery?: TelemetryQuery | undefined;
     };
+    matchesPerSeries?: Array<PerSeriesCriteriaMatch> | undefined;
   }): Promise<void> {
     const alertLogAttributes: LogAttributes = {
       projectId: input.monitor.projectId?.toString(),
@@ -113,6 +123,16 @@ export default class MonitorAlert {
       `${input.monitor.id?.toString()} - Check open alerts.`,
       alertLogAttributes,
     );
+
+    const breachingSeriesFingerprints: Set<string> | undefined =
+      input.matchesPerSeries
+        ? new Set<string>(
+            input.matchesPerSeries.map((m: PerSeriesCriteriaMatch) => {
+              return m.fingerprint;
+            }),
+          )
+        : undefined;
+
     // check active alerts and if there are open alerts, do not cretae anothr alert.
     const openAlerts: Array<Alert> =
       await this.checkOpenAlertsAndCloseIfResolved({
@@ -123,19 +143,31 @@ export default class MonitorAlert {
         criteriaInstance: input.criteriaInstance,
         dataToProcess: input.dataToProcess,
         evaluationSummary: input.evaluationSummary,
+        breachingSeriesFingerprints,
       });
 
-    if (input.criteriaInstance.data?.createAlerts) {
-      // create alerts
+    if (!input.criteriaInstance.data?.createAlerts) {
+      return;
+    }
+
+    const seriesToProcess: Array<PerSeriesCriteriaMatch | undefined> =
+      input.matchesPerSeries && input.matchesPerSeries.length > 0
+        ? input.matchesPerSeries
+        : [undefined];
 
-      for (const criteriaAlert of input.criteriaInstance.data?.alerts || []) {
-        // should create alert.
+    for (const criteriaAlert of input.criteriaInstance.data?.alerts || []) {
+      for (const seriesMatch of seriesToProcess) {
+        const seriesFingerprint: string | undefined = seriesMatch?.fingerprint;
+        const seriesLabels: JSONObject | undefined = seriesMatch?.labels;
+        const seriesRootCause: string =
+          seriesMatch?.rootCause || input.rootCause;
 
         const alreadyOpenAlert: Alert | undefined = openAlerts.find(
           (alert: Alert) => {
             return (
               alert.createdCriteriaId ===
-              input.criteriaInstance.data?.id.toString()
+                input.criteriaInstance.data?.id.toString() &&
+              (alert.seriesFingerprint || undefined) === seriesFingerprint
             );
           },
         );
@@ -153,9 +185,12 @@ export default class MonitorAlert {
         );
 
         if (hasAlreadyOpenAlert) {
+          const renderedAlertTitle: string =
+            alreadyOpenAlert?.title || criteriaAlert.title;
+
           input.evaluationSummary?.events.push({
             type: "alert-skipped",
-            title: `Alert already active: ${criteriaAlert.title}`,
+            title: `Alert already active: ${renderedAlertTitle}`,
             message:
               "Skipped creating a new alert because an active alert exists for this criteria.",
             relatedCriteriaId: input.criteriaInstance.data?.id,
@@ -180,6 +215,8 @@ export default class MonitorAlert {
           MonitorTemplateUtil.buildTemplateStorageMap({
             monitorType: input.monitor.monitorType!,
             dataToProcess: input.dataToProcess,
+            monitor: input.monitor,
+            seriesLabels,
           });
 
         alert.title = MonitorTemplateUtil.processTemplateString({
@@ -221,13 +258,20 @@ export default class MonitorAlert {
 
         alert.monitor = input.monitor;
         alert.projectId = input.monitor.projectId!;
-        alert.rootCause = input.rootCause;
+        alert.rootCause = seriesRootCause;
         alert.createdStateLog = JSON.parse(
           JSON.stringify(input.dataToProcess, null, 2),
         );
 
         alert.createdCriteriaId = input.criteriaInstance.data.id.toString();
 
+        if (seriesFingerprint) {
+          alert.seriesFingerprint = seriesFingerprint;
+        }
+        if (seriesLabels && Object.keys(seriesLabels).length > 0) {
+          alert.seriesLabels = seriesLabels;
+        }
+
         alert.onCallDutyPolicies =
           criteriaAlert.onCallPolicyIds?.map((id: ObjectID) => {
             const onCallPolicy: OnCallDutyPolicy = new OnCallDutyPolicy();
@@ -328,6 +372,82 @@ export default class MonitorAlert {
         input.openAlert.projectId!,
       );
 
+    /*
+     * Skip the Resolved insert if the alert's timeline is already at or past
+     * the Resolved state in the project's workflow order. Two cases:
+     *  1. Latest timeline state is Resolved but Alert.currentAlertStateId is
+     *     stuck on an earlier state (partial-failure from a prior resolve).
+     *     Re-inserting Resolved would throw "Alert state cannot be same as
+     *     previous state" from AlertStateTimelineService.onBeforeCreate.
+     *  2. The project defines a custom state after Resolved (e.g. Closed) and
+     *     the alert has moved into it. Inserting Resolved would throw
+     *     "cannot transition to Resolved from Closed because Resolved is
+     *     before Closed in the order of alert states."
+     * Either failure bubbles up through ingest workers and causes monitors to
+     * flap. Reconcile Alert.currentAlertStateId if out of sync with the
+     * timeline, then return.
+     */
+    const [resolvedState, latestTimeline]: [
+      AlertState | null,
+      AlertStateTimeline | null,
+    ] = await Promise.all([
+      AlertStateService.findOneBy({
+        query: {
+          _id: resolvedStateId.toString(),
+        },
+        select: {
+          order: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      }),
+      AlertStateTimelineService.findOneBy({
+        query: {
+          alertId: input.openAlert.id!,
+        },
+        sort: {
+          startsAt: SortOrder.Descending,
+        },
+        select: {
+          alertStateId: true,
+          alertState: {
+            order: true,
+          },
+        },
+        props: {
+          isRoot: true,
+        },
+      }),
+    ]);
+
+    const latestOrder: number | undefined | null =
+      latestTimeline?.alertState?.order;
+    const resolvedOrder: number | undefined | null = resolvedState?.order;
+
+    if (
+      latestTimeline?.alertStateId &&
+      typeof latestOrder === "number" &&
+      typeof resolvedOrder === "number" &&
+      latestOrder >= resolvedOrder
+    ) {
+      if (
+        input.openAlert.currentAlertStateId?.toString() !==
+        latestTimeline.alertStateId.toString()
+      ) {
+        await AlertService.updateOneById({
+          id: input.openAlert.id!,
+          data: {
+            currentAlertStateId: latestTimeline.alertStateId,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      }
+      return;
+    }
+
     const alertStateTimeline: AlertStateTimeline = new AlertStateTimeline();
     alertStateTimeline.alertId = input.openAlert.id!;
     alertStateTimeline.alertStateId = resolvedStateId;
@@ -357,7 +477,50 @@ export default class MonitorAlert {
     openAlert: Alert;
     autoResolveCriteriaInstanceIdAlertIdsDictionary: Dictionary<Array<string>>;
     criteriaInstance: MonitorCriteriaInstance | null; // null if no criteia met.
+    breachingSeriesFingerprints?: Set<string> | undefined;
   }): boolean {
+    const openSeriesFingerprint: string | undefined =
+      input.openAlert.seriesFingerprint || undefined;
+
+    /*
+     * Per-series auto-resolve: when a breaching-series set is given and
+     * this alert has a fingerprint, resolve whenever the fingerprint is
+     * no longer in the set, independent of whether other series on the
+     * same monitor are still breaching.
+     */
+    if (
+      input.breachingSeriesFingerprints !== undefined &&
+      openSeriesFingerprint
+    ) {
+      const stillBreaching: boolean = input.breachingSeriesFingerprints.has(
+        openSeriesFingerprint,
+      );
+
+      if (stillBreaching) {
+        return false;
+      }
+
+      if (!input.openAlert.createdCriteriaId?.toString()) {
+        return false;
+      }
+
+      const autoResolveTemplates: Array<string> | undefined =
+        input.autoResolveCriteriaInstanceIdAlertIdsDictionary[
+          input.openAlert.createdCriteriaId.toString()
+        ];
+
+      /*
+       * Alert auto-resolve lists templates by criteria; presence of any
+       * template for this criteria means "this criteria's alerts are
+       * configured to auto-resolve", so resolve this series.
+       */
+      if (autoResolveTemplates && autoResolveTemplates.length > 0) {
+        return true;
+      }
+
+      return false;
+    }
+
     if (
       input.openAlert.createdCriteriaId?.toString() ===
       input.criteriaInstance?.data?.id.toString()

package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts

@@ -9,7 +9,9 @@ import SSLMonitorCriteria from "./Criteria/SSLMonitorCriteria";
 import ServerMonitorCriteria from "./Criteria/ServerMonitorCriteria";
 import SyntheticMonitoringCriteria from "./Criteria/SyntheticMonitor";
 import LogMonitorCriteria from "./Criteria/LogMonitorCriteria";
-import MetricMonitorCriteria from "./Criteria/MetricMonitorCriteria";
+import MetricMonitorCriteria, {
+  MetricSeriesEvaluationResult,
+} from "./Criteria/MetricMonitorCriteria";
 import TraceMonitorCriteria from "./Criteria/TraceMonitorCriteria";
 import ExceptionMonitorCriteria from "./Criteria/ExceptionMonitorCriteria";
 import ProfileMonitorCriteria from "./Criteria/ProfileMonitorCriteria";
@@ -31,7 +33,9 @@ import MonitorEvaluationSummary, {
   MonitorEvaluationEvent,
   MonitorEvaluationFilterResult,
 } from "../../../Types/Monitor/MonitorEvaluationSummary";
-import ProbeApiIngestResponse from "../../../Types/Probe/ProbeApiIngestResponse";
+import ProbeApiIngestResponse, {
+  PerSeriesCriteriaMatch,
+} from "../../../Types/Probe/ProbeApiIngestResponse";
 import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
 import RequestFailedDetails from "../../../Types/Probe/RequestFailedDetails";
 import IncomingMonitorRequest from "../../../Types/Monitor/IncomingMonitor/IncomingMonitorRequest";
@@ -50,6 +54,7 @@ import MetricMonitorResponse, {
   KubernetesAffectedResource,
   KubernetesResourceBreakdown,
 } from "../../../Types/Monitor/MetricMonitor/MetricMonitorResponse";
+import MetricSeriesResult from "../../../Types/Monitor/MetricMonitor/MetricSeriesResult";
 import MetricCriteriaContext, {
   MetricBreachingSample,
   MetricComponent,
@@ -156,6 +161,26 @@ ${contextBlock}
 **Cause**: ${(input.dataToProcess as ProbeMonitorResponse).failureCause || ""}
 `;
         }
+
+        /*
+         * When this is a metric-style monitor with per-series results,
+         * compute the per-series match list so MonitorResource can fan
+         * out one incident per affected series. We do this *after* the
+         * scalar criteriaMetId/rootCause is populated so legacy readers
+         * still see a usable response if perSeriesMatches is ignored.
+         */
+        const perSeriesMatches: Array<PerSeriesCriteriaMatch> =
+          MonitorCriteriaEvaluator.collectPerSeriesMatches({
+            dataToProcess: input.dataToProcess,
+            monitor: input.monitor,
+            monitorStep: input.monitorStep,
+            criteriaInstance,
+          });
+
+        if (perSeriesMatches.length > 0) {
+          input.probeApiIngestResponse.perSeriesMatches = perSeriesMatches;
+        }
+
         break;
       }
     }
@@ -163,6 +188,150 @@ ${contextBlock}
     return input.probeApiIngestResponse;
   }
 
+  /**
+   * For metric-backed monitors (Metrics/Kubernetes/Docker) with
+   * per-series aggregated results, re-evaluate the matched criteria
+   * once per series and return one entry per series that breached.
+   * Returns an empty array when the monitor is not series-aware or
+   * no series matched — the caller falls back to the single-incident
+   * path in that case.
+   */
+  private static collectPerSeriesMatches(input: {
+    dataToProcess: DataToProcess;
+    monitor: Monitor;
+    monitorStep: MonitorStep;
+    criteriaInstance: MonitorCriteriaInstance;
+  }): Array<PerSeriesCriteriaMatch> {
+    if (
+      input.monitor.monitorType !== MonitorType.Metrics &&
+      input.monitor.monitorType !== MonitorType.Kubernetes &&
+      input.monitor.monitorType !== MonitorType.Docker
+    ) {
+      return [];
+    }
+
+    const metricResponse: MetricMonitorResponse =
+      input.dataToProcess as MetricMonitorResponse;
+
+    const seriesBreakdown: Array<MetricSeriesResult> | undefined =
+      metricResponse.seriesBreakdown;
+
+    if (!seriesBreakdown || seriesBreakdown.length === 0) {
+      return [];
+    }
+
+    const criteriaId: string | undefined = input.criteriaInstance.data?.id;
+    if (!criteriaId) {
+      return [];
+    }
+
+    const filterCondition: FilterCondition =
+      input.criteriaInstance.data?.filterCondition || FilterCondition.All;
+
+    const filters: Array<CriteriaFilter> =
+      input.criteriaInstance.data?.filters || [];
+
+    /*
+     * Evaluate every metric-value filter against every series. We only
+     * handle CheckOn.MetricValue for now — the criteria evaluator's
+     * other filter types don't carry series information.
+     */
+    const metricFilters: Array<CriteriaFilter> = filters.filter(
+      (f: CriteriaFilter) => {
+        return f.checkOn === CheckOn.MetricValue;
+      },
+    );
+
+    if (metricFilters.length === 0) {
+      return [];
+    }
+
+    /*
+     * Key: fingerprint. Value: results of each metric filter for that
+     * series, tracked so we can apply FilterCondition.All/Any per series.
+     */
+    const resultsByFingerprint: Map<
+      string,
+      {
+        labels: JSONObject;
+        filterResults: Array<MetricSeriesEvaluationResult>;
+      }
+    > = new Map();
+
+    for (const criteriaFilter of metricFilters) {
+      const evaluations: Array<MetricSeriesEvaluationResult> =
+        MetricMonitorCriteria.evaluateAllSeries({
+          dataToProcess: input.dataToProcess,
+          criteriaFilter,
+          monitorStep: input.monitorStep,
+        });
+
+      for (const evaluation of evaluations) {
+        const fingerprint: string | undefined = evaluation.fingerprint;
+        if (!fingerprint) {
+          continue;
+        }
+
+        const existing:
+          | {
+              labels: JSONObject;
+              filterResults: Array<MetricSeriesEvaluationResult>;
+            }
+          | undefined = resultsByFingerprint.get(fingerprint);
+
+        if (existing) {
+          existing.filterResults.push(evaluation);
+        } else {
+          resultsByFingerprint.set(fingerprint, {
+            labels: evaluation.labels,
+            filterResults: [evaluation],
+          });
+        }
+      }
+    }
+
+    const matches: Array<PerSeriesCriteriaMatch> = [];
+
+    for (const [fingerprint, entry] of resultsByFingerprint) {
+      // Were all/any of the metric filters met for this series?
+      const matched: Array<MetricSeriesEvaluationResult> =
+        entry.filterResults.filter((r: MetricSeriesEvaluationResult) => {
+          return r.rootCause !== null;
+        });
+
+      const seriesMatched: boolean =
+        filterCondition === FilterCondition.All
+          ? matched.length === entry.filterResults.length &&
+            entry.filterResults.length > 0
+          : matched.length > 0;
+
+      if (!seriesMatched) {
+        continue;
+      }
+
+      /*
+       * For the per-series rootCause, concatenate the matched filter
+       * messages so the incident message reflects exactly what
+       * breached on this specific series.
+       */
+      const rootCauseLines: Array<string> = matched.map(
+        (r: MetricSeriesEvaluationResult) => {
+          return `- ${r.rootCause}`;
+        },
+      );
+
+      matches.push({
+        criteriaMetId: criteriaId,
+        fingerprint,
+        labels: entry.labels,
+        rootCause: rootCauseLines.join("\n"),
+        metricContext: matched[0]?.context,
+      });
+    }
+
+    return matches;
+  }
+
   private static async processMonitorCriteriaInstance(input: {
     dataToProcess: DataToProcess;
     monitorStep: MonitorStep;