@oneuptime/common 10.0.71 → 10.0.73

package/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.ts

@@ -8,6 +8,7 @@ import MetricCriteriaContext, {
   MetricComponent,
   MetricComponentValue,
 } from "../../../../Types/Monitor/MetricMonitor/MetricCriteriaContext";
+import MetricSeriesResult from "../../../../Types/Monitor/MetricMonitor/MetricSeriesResult";
 import MonitorStep from "../../../../Types/Monitor/MonitorStep";
 import { JSONObject } from "../../../../Types/JSON";
 import DataToProcess from "../DataToProcess";
@@ -23,6 +24,20 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
 import MetricUnitUtil from "../../../../Utils/MetricUnitUtil";
 import MetricFormulaEvaluator from "../../../../Utils/Metrics/MetricFormulaEvaluator";
 
+/**
+ * Result of evaluating a single criteria filter against a single metric
+ * series. `rootCause` is null when the filter did not match; otherwise
+ * it's the human-readable comparison message. `context` always reflects
+ * the metric identity for this series (used to render the metric
+ * details + breaching samples section of the incident root cause).
+ */
+export interface MetricSeriesEvaluationResult {
+  fingerprint: string | undefined;
+  labels: JSONObject;
+  rootCause: string | null;
+  context: MetricCriteriaContext;
+}
+
 export default class MetricMonitorCriteria {
   @CaptureSpan()
   public static async isMonitorInstanceCriteriaFilterMet(input: {
@@ -30,8 +45,50 @@ export default class MetricMonitorCriteria {
     criteriaFilter: CriteriaFilter;
     monitorStep: MonitorStep;
   }): Promise<string | null> {
-    // Metric Monitoring Check
+    const evaluations: Array<MetricSeriesEvaluationResult> =
+      MetricMonitorCriteria.evaluateAllSeries(input);
+
+    /*
+     * Backwards-compat: the scalar entrypoint collapses per-series
+     * evaluation down to the first matching series so existing callers
+     * (single-incident path) keep working. The per-series code path uses
+     * `evaluateAllSeries` directly.
+     */
+    const match: MetricSeriesEvaluationResult | undefined = evaluations.find(
+      (e: MetricSeriesEvaluationResult) => {
+        return e.rootCause !== null;
+      },
+    );
+
+    /*
+     * Always populate the legacy single-context field so the root-cause
+     * renderer can still read metric identity from the criteria filter
+     * even when nothing matched. Pick the first evaluation's context.
+     */
+    if (evaluations.length > 0) {
+      input.criteriaFilter.metricCriteriaContext = (
+        match || evaluations[0]!
+      ).context;
+    }
 
+    return match ? match.rootCause : null;
+  }
+
+  /**
+   * Evaluate a single criteria filter against every series produced by
+   * the monitor. For monitors without group-by, this returns a single
+   * evaluation covering all aggregated results (legacy behavior). For
+   * monitors with group-by attributes set, it returns one evaluation
+   * per unique series fingerprint — each with its own
+   * `MetricCriteriaContext` carrying that series' breaching samples
+   * and labels. The caller fans this out into one incident per
+   * breaching series.
+   */
+  public static evaluateAllSeries(input: {
+    dataToProcess: DataToProcess;
+    criteriaFilter: CriteriaFilter;
+    monitorStep: MonitorStep;
+  }): Array<MetricSeriesEvaluationResult> {
     if (
       input.criteriaFilter.metricMonitorOptions &&
       !input.criteriaFilter.metricMonitorOptions.metricAggregationType
@@ -41,20 +98,14 @@ export default class MetricMonitorCriteria {
     }
 
     if (input.criteriaFilter.checkOn !== CheckOn.MetricValue) {
-      return null;
+      return [];
     }
 
-    const rawThreshold: number | null = CompareCriteria.convertToNumber(
-      input.criteriaFilter.value,
-    );
-
-    const metricAlias: string =
-      input.criteriaFilter.metricMonitorOptions?.metricAlias || "";
-
     const metricResponse: MetricMonitorResponse =
       input.dataToProcess as MetricMonitorResponse;
-    const metricAggregatedResult: Array<AggregatedResult> =
-      metricResponse.metricResult || [];
+
+    const seriesBreakdown: Array<MetricSeriesResult> | undefined =
+      metricResponse.seriesBreakdown;
 
     const queryConfigs: Array<MetricQueryConfigData> =
       input.monitorStep.data?.metricMonitor?.metricViewConfig?.queryConfigs ||
@@ -63,6 +114,60 @@ export default class MetricMonitorCriteria {
       input.monitorStep.data?.metricMonitor?.metricViewConfig?.formulaConfigs ||
       [];
 
+    /*
+     * Series-less path: one synthetic "all-series" evaluation over the
+     * flat metricResult. Preserves the pre-group-by behavior exactly.
+     */
+    if (!seriesBreakdown || seriesBreakdown.length === 0) {
+      const result: MetricSeriesEvaluationResult =
+        MetricMonitorCriteria.evaluateOneSeries({
+          criteriaFilter: input.criteriaFilter,
+          aggregatedResults: metricResponse.metricResult || [],
+          queryConfigs,
+          formulaConfigs,
+          seriesFingerprint: undefined,
+          seriesLabels: {},
+        });
+      return [result];
+    }
+
+    return seriesBreakdown.map((series: MetricSeriesResult) => {
+      return MetricMonitorCriteria.evaluateOneSeries({
+        criteriaFilter: input.criteriaFilter,
+        aggregatedResults: series.aggregatedResults,
+        queryConfigs,
+        formulaConfigs,
+        seriesFingerprint: series.fingerprint,
+        seriesLabels: series.labels,
+      });
+    });
+  }
+
+  /**
+   * Core evaluation loop: compare the samples for one metric series
+   * against the criteria threshold. Builds the metric identity context,
+   * identifies breaching samples, and assembles the human-readable
+   * root-cause message. Factored out so `evaluateAllSeries` can invoke
+   * it once per series without duplicating logic.
+   */
+  private static evaluateOneSeries(input: {
+    criteriaFilter: CriteriaFilter;
+    aggregatedResults: Array<AggregatedResult>;
+    queryConfigs: Array<MetricQueryConfigData>;
+    formulaConfigs: Array<MetricFormulaConfigData>;
+    seriesFingerprint: string | undefined;
+    seriesLabels: JSONObject;
+  }): MetricSeriesEvaluationResult {
+    const rawThreshold: number | null = CompareCriteria.convertToNumber(
+      input.criteriaFilter.value,
+    );
+
+    const metricAlias: string =
+      input.criteriaFilter.metricMonitorOptions?.metricAlias || "";
+
+    const metricAggregatedResult: Array<AggregatedResult> =
+      input.aggregatedResults;
+
     /*
      * Resolve which query/formula the alias refers to. Use explicit index
      * checks (not `findIndex() || -1`, which incorrectly falls back to -1
@@ -73,25 +178,25 @@ export default class MetricMonitorCriteria {
     let aliasIndex: number = -1;
 
     if (metricAlias) {
-      const qIdx: number = queryConfigs.findIndex(
+      const qIdx: number = input.queryConfigs.findIndex(
         (q: MetricQueryConfigData) => {
           return q.metricAliasData?.metricVariable === metricAlias;
         },
       );
 
       if (qIdx >= 0) {
-        matchedQuery = queryConfigs[qIdx] || null;
+        matchedQuery = input.queryConfigs[qIdx] || null;
         aliasIndex = qIdx;
       } else {
-        const fIdx: number = formulaConfigs.findIndex(
+        const fIdx: number = input.formulaConfigs.findIndex(
           (f: MetricFormulaConfigData) => {
             return f.metricAliasData?.metricVariable === metricAlias;
           },
         );
 
         if (fIdx >= 0) {
-          matchedFormula = formulaConfigs[fIdx] || null;
-          aliasIndex = queryConfigs.length + fIdx;
+          matchedFormula = input.formulaConfigs[fIdx] || null;
+          aliasIndex = input.queryConfigs.length + fIdx;
         }
       }
     }
@@ -105,8 +210,8 @@ export default class MetricMonitorCriteria {
         ? metricAggregatedResult[aliasIndex]
         : metricAggregatedResult[0];
 
-    if (!matchedQuery && !matchedFormula && queryConfigs[0]) {
-      matchedQuery = queryConfigs[0];
+    if (!matchedQuery && !matchedFormula && input.queryConfigs[0]) {
+      matchedQuery = input.queryConfigs[0];
     }
 
     /*
@@ -119,14 +224,24 @@ export default class MetricMonitorCriteria {
         matchedFormula,
         metricAlias,
         criteriaFilter: input.criteriaFilter,
-        queryConfigs,
-        formulaConfigs,
+        queryConfigs: input.queryConfigs,
+        formulaConfigs: input.formulaConfigs,
       });
 
-    input.criteriaFilter.metricCriteriaContext = metricContext;
+    if (input.seriesFingerprint) {
+      metricContext.seriesFingerprint = input.seriesFingerprint;
+    }
+    if (input.seriesLabels && Object.keys(input.seriesLabels).length > 0) {
+      metricContext.seriesLabels = input.seriesLabels;
+    }
 
     if (rawThreshold === null) {
-      return null;
+      return {
+        fingerprint: input.seriesFingerprint,
+        labels: input.seriesLabels,
+        rootCause: null,
+        context: metricContext,
+      };
     }
 
     /*
@@ -180,11 +295,21 @@ export default class MetricMonitorCriteria {
         NoDataPolicy.Ignore;
 
       if (policy === NoDataPolicy.Ignore) {
-        return null;
+        return {
+          fingerprint: input.seriesFingerprint,
+          labels: input.seriesLabels,
+          rootCause: null,
+          context: metricContext,
+        };
       }
 
       if (policy === NoDataPolicy.Trigger) {
-        return `No data received for ${metricContext.metricName} in the evaluation window — triggering per no-data policy.`;
+        return {
+          fingerprint: input.seriesFingerprint,
+          labels: input.seriesLabels,
+          rootCause: `No data received for ${metricContext.metricName} in the evaluation window — triggering per no-data policy.`,
+          context: metricContext,
+        };
       }
 
       // TreatAsZero: fall through to the comparator with value 0.
@@ -206,7 +331,12 @@ export default class MetricMonitorCriteria {
       });
 
     if (!comparisonMessage) {
-      return null;
+      return {
+        fingerprint: input.seriesFingerprint,
+        labels: input.seriesLabels,
+        rootCause: null,
+        context: metricContext,
+      };
     }
 
     /*
@@ -232,8 +362,8 @@ export default class MetricMonitorCriteria {
       matchedFormula
         ? MetricMonitorCriteria.buildComponentValueLookup({
             components: metricContext.components || [],
-            queryConfigs,
-            formulaConfigs,
+            queryConfigs: input.queryConfigs,
+            formulaConfigs: input.formulaConfigs,
             metricAggregatedResult,
           })
         : null;
@@ -276,7 +406,12 @@ export default class MetricMonitorCriteria {
       metricContext.breachingSamples = breachingSamples;
     }
 
-    return comparisonMessage;
+    return {
+      fingerprint: input.seriesFingerprint,
+      labels: input.seriesLabels,
+      rootCause: comparisonMessage,
+      context: metricContext,
+    };
   }
 
   private static buildComponentValueLookup(input: {
@@ -438,6 +573,17 @@ export default class MetricMonitorCriteria {
       ? Object.keys(q.metricQueryData.groupBy as Record<string, unknown>)
       : [];
 
+    /*
+     * Include user-selected attribute keys as part of the groupBy view
+     * so the root-cause block shows "Grouped By: host.name" not just the
+     * raw columns ClickHouse was asked to partition on.
+     */
+    const groupByAttributeKeys: Array<string> =
+      q?.metricQueryData?.groupByAttributeKeys || [];
+    const allGroupBy: Array<string> = Array.from(
+      new Set([...groupBy, ...groupByAttributeKeys]),
+    );
+
     const components: Array<MetricComponent> | undefined = f
       ? MetricMonitorCriteria.buildFormulaComponents({
           formulaConfig: f,
@@ -454,7 +600,7 @@ export default class MetricMonitorCriteria {
       isFormula: Boolean(f),
       formulaExpression: f?.metricFormulaData?.metricFormula,
       filterAttributes,
-      groupBy,
+      groupBy: allGroupBy,
       timeWindowMinutes:
         input.criteriaFilter.evaluateOverTimeOptions?.timeValueInMinutes,
       ...(components && components.length > 0 ? { components } : {}),

package/Server/Utils/Monitor/Criteria/ServerMonitorCriteria.ts

@@ -208,6 +208,77 @@ export default class ServerMonitorCriteria {
       });
     }
 
+    if (
+      !(input.dataToProcess as ServerMonitorResponse)
+        .onlyCheckRequestReceivedAt &&
+      (input.criteriaFilter.checkOn === CheckOn.LoadAverage1Min ||
+        input.criteriaFilter.checkOn === CheckOn.LoadAverage5Min ||
+        input.criteriaFilter.checkOn === CheckOn.LoadAverage15Min)
+    ) {
+      threshold = CompareCriteria.convertToNumber(threshold);
+
+      const loadMetrics:
+        | { load1: number; load5: number; load15: number }
+        | undefined = (input.dataToProcess as ServerMonitorResponse)
+        .basicInfrastructureMetrics?.loadMetrics;
+
+      let currentLoad: number | undefined = undefined;
+      if (input.criteriaFilter.checkOn === CheckOn.LoadAverage1Min) {
+        currentLoad = loadMetrics?.load1;
+      } else if (input.criteriaFilter.checkOn === CheckOn.LoadAverage5Min) {
+        currentLoad = loadMetrics?.load5;
+      } else if (input.criteriaFilter.checkOn === CheckOn.LoadAverage15Min) {
+        currentLoad = loadMetrics?.load15;
+      }
+
+      const value: number | Array<number> =
+        (overTimeValue as Array<number>) || currentLoad || 0;
+
+      return CompareCriteria.compareCriteriaNumbers({
+        value: value,
+        threshold: threshold as number,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
+    if (
+      input.criteriaFilter.checkOn === CheckOn.SwapUsagePercent &&
+      !(input.dataToProcess as ServerMonitorResponse).onlyCheckRequestReceivedAt
+    ) {
+      threshold = CompareCriteria.convertToNumber(threshold);
+
+      const swapPercent: number | Array<number> =
+        (overTimeValue as Array<number>) ||
+        (input.dataToProcess as ServerMonitorResponse)
+          .basicInfrastructureMetrics?.memoryMetrics?.swapPercentUsed ||
+        0;
+
+      return CompareCriteria.compareCriteriaNumbers({
+        value: swapPercent,
+        threshold: threshold as number,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
+    if (
+      input.criteriaFilter.checkOn === CheckOn.CPUIoWaitPercent &&
+      !(input.dataToProcess as ServerMonitorResponse).onlyCheckRequestReceivedAt
+    ) {
+      threshold = CompareCriteria.convertToNumber(threshold);
+
+      const ioWaitPercent: number | Array<number> =
+        (overTimeValue as Array<number>) ||
+        (input.dataToProcess as ServerMonitorResponse)
+          .basicInfrastructureMetrics?.cpuMetrics?.timeIoWaitPercent ||
+        0;
+
+      return CompareCriteria.compareCriteriaNumbers({
+        value: ioWaitPercent,
+        threshold: threshold as number,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
     if (
       input.criteriaFilter.checkOn === CheckOn.ServerProcessName &&
       threshold &&

package/Server/Utils/Monitor/MonitorAlert.ts

@@ -26,6 +26,7 @@ import MonitorTemplateUtil from "./MonitorTemplateUtil";
 import { JSONObject } from "../../../Types/JSON";
 import OneUptimeDate from "../../../Types/Date";
 import MonitorEvaluationSummary from "../../../Types/Monitor/MonitorEvaluationSummary";
+import { PerSeriesCriteriaMatch } from "../../../Types/Probe/ProbeApiIngestResponse";
 
 export default class MonitorAlert {
   @CaptureSpan()
@@ -36,6 +37,7 @@ export default class MonitorAlert {
     criteriaInstance: MonitorCriteriaInstance | null;
     dataToProcess: DataToProcess;
     evaluationSummary?: MonitorEvaluationSummary | undefined;
+    breachingSeriesFingerprints?: Set<string> | undefined;
   }): Promise<Array<Alert>> {
     // check active alerts and if there are open alerts, do not cretae anothr alert.
     const openAlerts: Array<Alert> = await AlertService.findBy({
@@ -49,11 +51,14 @@ export default class MonitorAlert {
       limit: LIMIT_PER_PROJECT,
       select: {
         _id: true,
+        title: true,
         createdCriteriaId: true,
         projectId: true,
         alertNumber: true,
         alertNumberWithPrefix: true,
         currentAlertStateId: true,
+        seriesFingerprint: true,
+        seriesLabels: true,
       },
       props: {
         isRoot: true,
@@ -68,6 +73,7 @@ export default class MonitorAlert {
         autoResolveCriteriaInstanceIdAlertIdsDictionary:
           input.autoResolveCriteriaInstanceIdAlertIdsDictionary,
         criteriaInstance: input.criteriaInstance,
+        breachingSeriesFingerprints: input.breachingSeriesFingerprints,
       });
 
       if (shouldClose) {
@@ -106,6 +112,7 @@ export default class MonitorAlert {
     props: {
       telemetryQuery?: TelemetryQuery | undefined;
     };
+    matchesPerSeries?: Array<PerSeriesCriteriaMatch> | undefined;
   }): Promise<void> {
     const alertLogAttributes: LogAttributes = {
       projectId: input.monitor.projectId?.toString(),
@@ -116,6 +123,16 @@ export default class MonitorAlert {
       `${input.monitor.id?.toString()} - Check open alerts.`,
       alertLogAttributes,
     );
+
+    const breachingSeriesFingerprints: Set<string> | undefined =
+      input.matchesPerSeries
+        ? new Set<string>(
+            input.matchesPerSeries.map((m: PerSeriesCriteriaMatch) => {
+              return m.fingerprint;
+            }),
+          )
+        : undefined;
+
     // check active alerts and if there are open alerts, do not cretae anothr alert.
     const openAlerts: Array<Alert> =
       await this.checkOpenAlertsAndCloseIfResolved({
@@ -126,19 +143,31 @@ export default class MonitorAlert {
         criteriaInstance: input.criteriaInstance,
         dataToProcess: input.dataToProcess,
         evaluationSummary: input.evaluationSummary,
+        breachingSeriesFingerprints,
       });
 
-    if (input.criteriaInstance.data?.createAlerts) {
-      // create alerts
+    if (!input.criteriaInstance.data?.createAlerts) {
+      return;
+    }
 
-      for (const criteriaAlert of input.criteriaInstance.data?.alerts || []) {
-        // should create alert.
+    const seriesToProcess: Array<PerSeriesCriteriaMatch | undefined> =
+      input.matchesPerSeries && input.matchesPerSeries.length > 0
+        ? input.matchesPerSeries
+        : [undefined];
+
+    for (const criteriaAlert of input.criteriaInstance.data?.alerts || []) {
+      for (const seriesMatch of seriesToProcess) {
+        const seriesFingerprint: string | undefined = seriesMatch?.fingerprint;
+        const seriesLabels: JSONObject | undefined = seriesMatch?.labels;
+        const seriesRootCause: string =
+          seriesMatch?.rootCause || input.rootCause;
 
         const alreadyOpenAlert: Alert | undefined = openAlerts.find(
           (alert: Alert) => {
             return (
               alert.createdCriteriaId ===
-              input.criteriaInstance.data?.id.toString()
+                input.criteriaInstance.data?.id.toString() &&
+              (alert.seriesFingerprint || undefined) === seriesFingerprint
             );
           },
         );
@@ -156,9 +185,12 @@ export default class MonitorAlert {
         );
 
         if (hasAlreadyOpenAlert) {
+          const renderedAlertTitle: string =
+            alreadyOpenAlert?.title || criteriaAlert.title;
+
           input.evaluationSummary?.events.push({
             type: "alert-skipped",
-            title: `Alert already active: ${criteriaAlert.title}`,
+            title: `Alert already active: ${renderedAlertTitle}`,
             message:
               "Skipped creating a new alert because an active alert exists for this criteria.",
             relatedCriteriaId: input.criteriaInstance.data?.id,
@@ -183,6 +215,8 @@ export default class MonitorAlert {
           MonitorTemplateUtil.buildTemplateStorageMap({
             monitorType: input.monitor.monitorType!,
             dataToProcess: input.dataToProcess,
+            monitor: input.monitor,
+            seriesLabels,
           });
 
         alert.title = MonitorTemplateUtil.processTemplateString({
@@ -224,13 +258,20 @@ export default class MonitorAlert {
 
         alert.monitor = input.monitor;
         alert.projectId = input.monitor.projectId!;
-        alert.rootCause = input.rootCause;
+        alert.rootCause = seriesRootCause;
         alert.createdStateLog = JSON.parse(
           JSON.stringify(input.dataToProcess, null, 2),
         );
 
         alert.createdCriteriaId = input.criteriaInstance.data.id.toString();
 
+        if (seriesFingerprint) {
+          alert.seriesFingerprint = seriesFingerprint;
+        }
+        if (seriesLabels && Object.keys(seriesLabels).length > 0) {
+          alert.seriesLabels = seriesLabels;
+        }
+
         alert.onCallDutyPolicies =
           criteriaAlert.onCallPolicyIds?.map((id: ObjectID) => {
             const onCallPolicy: OnCallDutyPolicy = new OnCallDutyPolicy();
@@ -436,7 +477,50 @@ export default class MonitorAlert {
     openAlert: Alert;
     autoResolveCriteriaInstanceIdAlertIdsDictionary: Dictionary<Array<string>>;
     criteriaInstance: MonitorCriteriaInstance | null; // null if no criteia met.
+    breachingSeriesFingerprints?: Set<string> | undefined;
   }): boolean {
+    const openSeriesFingerprint: string | undefined =
+      input.openAlert.seriesFingerprint || undefined;
+
+    /*
+     * Per-series auto-resolve: when a breaching-series set is given and
+     * this alert has a fingerprint, resolve whenever the fingerprint is
+     * no longer in the set, independent of whether other series on the
+     * same monitor are still breaching.
+     */
+    if (
+      input.breachingSeriesFingerprints !== undefined &&
+      openSeriesFingerprint
+    ) {
+      const stillBreaching: boolean = input.breachingSeriesFingerprints.has(
+        openSeriesFingerprint,
+      );
+
+      if (stillBreaching) {
+        return false;
+      }
+
+      if (!input.openAlert.createdCriteriaId?.toString()) {
+        return false;
+      }
+
+      const autoResolveTemplates: Array<string> | undefined =
+        input.autoResolveCriteriaInstanceIdAlertIdsDictionary[
+          input.openAlert.createdCriteriaId.toString()
+        ];
+
+      /*
+       * Alert auto-resolve lists templates by criteria; presence of any
+       * template for this criteria means "this criteria's alerts are
+       * configured to auto-resolve", so resolve this series.
+       */
+      if (autoResolveTemplates && autoResolveTemplates.length > 0) {
+        return true;
+      }
+
+      return false;
+    }
+
     if (
       input.openAlert.createdCriteriaId?.toString() ===
       input.criteriaInstance?.data?.id.toString()