@oneuptime/common 10.0.71 → 10.0.73

package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts

@@ -9,7 +9,9 @@ import SSLMonitorCriteria from "./Criteria/SSLMonitorCriteria";
 import ServerMonitorCriteria from "./Criteria/ServerMonitorCriteria";
 import SyntheticMonitoringCriteria from "./Criteria/SyntheticMonitor";
 import LogMonitorCriteria from "./Criteria/LogMonitorCriteria";
-import MetricMonitorCriteria from "./Criteria/MetricMonitorCriteria";
+import MetricMonitorCriteria, {
+  MetricSeriesEvaluationResult,
+} from "./Criteria/MetricMonitorCriteria";
 import TraceMonitorCriteria from "./Criteria/TraceMonitorCriteria";
 import ExceptionMonitorCriteria from "./Criteria/ExceptionMonitorCriteria";
 import ProfileMonitorCriteria from "./Criteria/ProfileMonitorCriteria";
@@ -31,7 +33,9 @@ import MonitorEvaluationSummary, {
   MonitorEvaluationEvent,
   MonitorEvaluationFilterResult,
 } from "../../../Types/Monitor/MonitorEvaluationSummary";
-import ProbeApiIngestResponse from "../../../Types/Probe/ProbeApiIngestResponse";
+import ProbeApiIngestResponse, {
+  PerSeriesCriteriaMatch,
+} from "../../../Types/Probe/ProbeApiIngestResponse";
 import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
 import RequestFailedDetails from "../../../Types/Probe/RequestFailedDetails";
 import IncomingMonitorRequest from "../../../Types/Monitor/IncomingMonitor/IncomingMonitorRequest";
@@ -50,6 +54,7 @@ import MetricMonitorResponse, {
   KubernetesAffectedResource,
   KubernetesResourceBreakdown,
 } from "../../../Types/Monitor/MetricMonitor/MetricMonitorResponse";
+import MetricSeriesResult from "../../../Types/Monitor/MetricMonitor/MetricSeriesResult";
 import MetricCriteriaContext, {
   MetricBreachingSample,
   MetricComponent,
@@ -156,6 +161,26 @@ ${contextBlock}
 **Cause**: ${(input.dataToProcess as ProbeMonitorResponse).failureCause || ""}
 `;
         }
+
+        /*
+         * When this is a metric-style monitor with per-series results,
+         * compute the per-series match list so MonitorResource can fan
+         * out one incident per affected series. We do this *after* the
+         * scalar criteriaMetId/rootCause is populated so legacy readers
+         * still see a usable response if perSeriesMatches is ignored.
+         */
+        const perSeriesMatches: Array<PerSeriesCriteriaMatch> =
+          MonitorCriteriaEvaluator.collectPerSeriesMatches({
+            dataToProcess: input.dataToProcess,
+            monitor: input.monitor,
+            monitorStep: input.monitorStep,
+            criteriaInstance,
+          });
+
+        if (perSeriesMatches.length > 0) {
+          input.probeApiIngestResponse.perSeriesMatches = perSeriesMatches;
+        }
+
         break;
       }
     }
@@ -163,6 +188,150 @@ ${contextBlock}
     return input.probeApiIngestResponse;
   }
 
+  /**
+   * For metric-backed monitors (Metrics/Kubernetes/Docker) with
+   * per-series aggregated results, re-evaluate the matched criteria
+   * once per series and return one entry per series that breached.
+   * Returns an empty array when the monitor is not series-aware or
+   * no series matched — the caller falls back to the single-incident
+   * path in that case.
+   */
+  private static collectPerSeriesMatches(input: {
+    dataToProcess: DataToProcess;
+    monitor: Monitor;
+    monitorStep: MonitorStep;
+    criteriaInstance: MonitorCriteriaInstance;
+  }): Array<PerSeriesCriteriaMatch> {
+    if (
+      input.monitor.monitorType !== MonitorType.Metrics &&
+      input.monitor.monitorType !== MonitorType.Kubernetes &&
+      input.monitor.monitorType !== MonitorType.Docker
+    ) {
+      return [];
+    }
+
+    const metricResponse: MetricMonitorResponse =
+      input.dataToProcess as MetricMonitorResponse;
+
+    const seriesBreakdown: Array<MetricSeriesResult> | undefined =
+      metricResponse.seriesBreakdown;
+
+    if (!seriesBreakdown || seriesBreakdown.length === 0) {
+      return [];
+    }
+
+    const criteriaId: string | undefined = input.criteriaInstance.data?.id;
+    if (!criteriaId) {
+      return [];
+    }
+
+    const filterCondition: FilterCondition =
+      input.criteriaInstance.data?.filterCondition || FilterCondition.All;
+
+    const filters: Array<CriteriaFilter> =
+      input.criteriaInstance.data?.filters || [];
+
+    /*
+     * Evaluate every metric-value filter against every series. We only
+     * handle CheckOn.MetricValue for now — the criteria evaluator's
+     * other filter types don't carry series information.
+     */
+    const metricFilters: Array<CriteriaFilter> = filters.filter(
+      (f: CriteriaFilter) => {
+        return f.checkOn === CheckOn.MetricValue;
+      },
+    );
+
+    if (metricFilters.length === 0) {
+      return [];
+    }
+
+    /*
+     * Key: fingerprint. Value: results of each metric filter for that
+     * series, tracked so we can apply FilterCondition.All/Any per series.
+     */
+    const resultsByFingerprint: Map<
+      string,
+      {
+        labels: JSONObject;
+        filterResults: Array<MetricSeriesEvaluationResult>;
+      }
+    > = new Map();
+
+    for (const criteriaFilter of metricFilters) {
+      const evaluations: Array<MetricSeriesEvaluationResult> =
+        MetricMonitorCriteria.evaluateAllSeries({
+          dataToProcess: input.dataToProcess,
+          criteriaFilter,
+          monitorStep: input.monitorStep,
+        });
+
+      for (const evaluation of evaluations) {
+        const fingerprint: string | undefined = evaluation.fingerprint;
+        if (!fingerprint) {
+          continue;
+        }
+
+        const existing:
+          | {
+              labels: JSONObject;
+              filterResults: Array<MetricSeriesEvaluationResult>;
+            }
+          | undefined = resultsByFingerprint.get(fingerprint);
+
+        if (existing) {
+          existing.filterResults.push(evaluation);
+        } else {
+          resultsByFingerprint.set(fingerprint, {
+            labels: evaluation.labels,
+            filterResults: [evaluation],
+          });
+        }
+      }
+    }
+
+    const matches: Array<PerSeriesCriteriaMatch> = [];
+
+    for (const [fingerprint, entry] of resultsByFingerprint) {
+      // Were all/any of the metric filters met for this series?
+      const matched: Array<MetricSeriesEvaluationResult> =
+        entry.filterResults.filter((r: MetricSeriesEvaluationResult) => {
+          return r.rootCause !== null;
+        });
+
+      const seriesMatched: boolean =
+        filterCondition === FilterCondition.All
+          ? matched.length === entry.filterResults.length &&
+            entry.filterResults.length > 0
+          : matched.length > 0;
+
+      if (!seriesMatched) {
+        continue;
+      }
+
+      /*
+       * For the per-series rootCause, concatenate the matched filter
+       * messages so the incident message reflects exactly what
+       * breached on this specific series.
+       */
+      const rootCauseLines: Array<string> = matched.map(
+        (r: MetricSeriesEvaluationResult) => {
+          return `- ${r.rootCause}`;
+        },
+      );
+
+      matches.push({
+        criteriaMetId: criteriaId,
+        fingerprint,
+        labels: entry.labels,
+        rootCause: rootCauseLines.join("\n"),
+        metricContext: matched[0]?.context,
+      });
+    }
+
+    return matches;
+  }
+
   private static async processMonitorCriteriaInstance(input: {
     dataToProcess: DataToProcess;
     monitorStep: MonitorStep;

package/Server/Utils/Monitor/MonitorIncident.ts

@@ -29,6 +29,7 @@ import { JSONObject } from "../../../Types/JSON";
 import OneUptimeDate from "../../../Types/Date";
 import MonitorEvaluationSummary from "../../../Types/Monitor/MonitorEvaluationSummary";
 import { IncidentMemberRoleAssignment } from "../../../Types/Monitor/CriteriaIncident";
+import { PerSeriesCriteriaMatch } from "../../../Types/Probe/ProbeApiIngestResponse";
 
 export default class MonitorIncident {
   @CaptureSpan()
@@ -41,6 +42,15 @@ export default class MonitorIncident {
     criteriaInstance: MonitorCriteriaInstance | null;
     dataToProcess: DataToProcess;
     evaluationSummary?: MonitorEvaluationSummary | undefined;
+    /**
+     * When set, the fingerprint set of series still breaching on this
+     * tick. Any open per-series incident whose fingerprint is NOT in
+     * this set is auto-resolved — that's how a series returning to
+     * normal closes its incident independently of other series on the
+     * same monitor. Undefined means "legacy mode" and per-series
+     * incidents are treated like any other for dedupe/resolve.
+     */
+    breachingSeriesFingerprints?: Set<string> | undefined;
   }): Promise<Array<Incident>> {
     // check active incidents and if there are open incidents, do not cretae anothr incident.
     const openIncidents: Array<Incident> = await IncidentService.findBy({
@@ -54,12 +64,15 @@ export default class MonitorIncident {
       limit: LIMIT_PER_PROJECT,
       select: {
         _id: true,
+        title: true,
         createdCriteriaId: true,
         createdIncidentTemplateId: true,
         projectId: true,
         incidentNumber: true,
         incidentNumberWithPrefix: true,
         currentIncidentStateId: true,
+        seriesFingerprint: true,
+        seriesLabels: true,
       },
       props: {
         isRoot: true,
@@ -74,6 +87,7 @@ export default class MonitorIncident {
         autoResolveCriteriaInstanceIdIncidentIdsDictionary:
           input.autoResolveCriteriaInstanceIdIncidentIdsDictionary,
         criteriaInstance: input.criteriaInstance,
+        breachingSeriesFingerprints: input.breachingSeriesFingerprints,
       });
 
       if (shouldClose) {
@@ -115,6 +129,13 @@ export default class MonitorIncident {
     props: {
       telemetryQuery?: TelemetryQuery | undefined;
     };
+    /**
+     * When set, create one incident per series instead of one per
+     * monitor. Each entry gets its own rootCause, seriesFingerprint,
+     * and seriesLabels so the incident title + description can
+     * reference `{{host.name}}` etc. via the template engine.
+     */
+    matchesPerSeries?: Array<PerSeriesCriteriaMatch> | undefined;
   }): Promise<void> {
     const incidentLogAttributes: LogAttributes = {
       projectId: input.monitor.projectId?.toString(),
@@ -125,6 +146,21 @@ export default class MonitorIncident {
       `${input.monitor.id?.toString()} - Check open incidents.`,
       incidentLogAttributes,
     );
+
+    /*
+     * Per-series mode: close any open incident for a series that's no
+     * longer breaching *before* we look at the remaining open set, so
+     * dedupe decisions below match the post-resolve state.
+     */
+    const breachingSeriesFingerprints: Set<string> | undefined =
+      input.matchesPerSeries
+        ? new Set<string>(
+            input.matchesPerSeries.map((m: PerSeriesCriteriaMatch) => {
+              return m.fingerprint;
+            }),
+          )
+        : undefined;
+
     // check active incidents and if there are open incidents, do not cretae anothr incident.
     const openIncidents: Array<Incident> =
       await this.checkOpenIncidentsAndCloseIfResolved({
@@ -135,14 +171,29 @@ export default class MonitorIncident {
         criteriaInstance: input.criteriaInstance,
         dataToProcess: input.dataToProcess,
         evaluationSummary: input.evaluationSummary,
+        breachingSeriesFingerprints,
       });
 
-    if (input.criteriaInstance.data?.createIncidents) {
-      // create incidents
+    if (!input.criteriaInstance.data?.createIncidents) {
+      return;
+    }
 
-      for (const criteriaIncident of input.criteriaInstance.data?.incidents ||
-        []) {
-        // should create incident.
+    /*
+     * Series-less path: one incident per criteriaIncident template as
+     * before. Series-aware path: one incident per (series × template).
+     */
+    const seriesToProcess: Array<PerSeriesCriteriaMatch | undefined> =
+      input.matchesPerSeries && input.matchesPerSeries.length > 0
+        ? input.matchesPerSeries
+        : [undefined];
+
+    for (const criteriaIncident of input.criteriaInstance.data?.incidents ||
+      []) {
+      for (const seriesMatch of seriesToProcess) {
+        const seriesFingerprint: string | undefined = seriesMatch?.fingerprint;
+        const seriesLabels: JSONObject | undefined = seriesMatch?.labels;
+        const seriesRootCause: string =
+          seriesMatch?.rootCause || input.rootCause;
 
         const alreadyOpenIncident: Incident | undefined = openIncidents.find(
           (incident: Incident) => {
@@ -150,7 +201,8 @@ export default class MonitorIncident {
               incident.createdCriteriaId ===
                 input.criteriaInstance.data?.id.toString() &&
               incident.createdIncidentTemplateId ===
-                criteriaIncident.id.toString()
+                criteriaIncident.id.toString() &&
+              (incident.seriesFingerprint || undefined) === seriesFingerprint
             );
           },
         );
@@ -168,9 +220,20 @@ export default class MonitorIncident {
         );
 
         if (hasAlreadyOpenIncident) {
+          /*
+           * Use the open incident's already-rendered title when
+           * available — the template (`criteriaIncident.title`) still
+           * contains unresolved `{{…}}` placeholders because it's the
+           * criterion's template string, not the instance's rendered
+           * output. Falling back to the template only when the open
+           * incident somehow has no title.
+           */
+          const renderedTitle: string =
+            alreadyOpenIncident?.title || criteriaIncident.title;
+
           input.evaluationSummary?.events.push({
             type: "incident-skipped",
-            title: `Incident already active: ${criteriaIncident.title}`,
+            title: `Incident already active: ${renderedTitle}`,
             message:
               "Skipped creating a new incident because an active incident exists for this criteria.",
             relatedCriteriaId: input.criteriaInstance.data?.id,
@@ -193,6 +256,8 @@ export default class MonitorIncident {
           MonitorTemplateUtil.buildTemplateStorageMap({
             monitorType: input.monitor.monitorType!,
             dataToProcess: input.dataToProcess,
+            monitor: input.monitor,
+            seriesLabels,
           });
 
         incident.title = MonitorTemplateUtil.processTemplateString({
@@ -236,7 +301,7 @@ export default class MonitorIncident {
 
         incident.monitors = [input.monitor];
         incident.projectId = input.monitor.projectId!;
-        incident.rootCause = input.rootCause;
+        incident.rootCause = seriesRootCause;
         incident.createdStateLog = JSON.parse(
           JSON.stringify(input.dataToProcess, null, 2),
         );
@@ -245,6 +310,13 @@ export default class MonitorIncident {
 
         incident.createdIncidentTemplateId = criteriaIncident.id.toString();
 
+        if (seriesFingerprint) {
+          incident.seriesFingerprint = seriesFingerprint;
+        }
+        if (seriesLabels && Object.keys(seriesLabels).length > 0) {
+          incident.seriesLabels = seriesLabels;
+        }
+
         incident.onCallDutyPolicies =
           criteriaIncident.onCallPolicyIds?.map((id: ObjectID) => {
             const onCallPolicy: OnCallDutyPolicy = new OnCallDutyPolicy();
@@ -504,7 +576,60 @@ export default class MonitorIncident {
       Array<string>
     >;
     criteriaInstance: MonitorCriteriaInstance | null; // null if no criteia met.
+    breachingSeriesFingerprints?: Set<string> | undefined;
   }): boolean {
+    const openSeriesFingerprint: string | undefined =
+      input.openIncident.seriesFingerprint || undefined;
+
+    /*
+     * Per-series auto-resolve: when the monitor emits a breaching-
+     * series set and this open incident has a fingerprint, resolve
+     * whenever the fingerprint is no longer in the set — regardless
+     * of whether some *other* series is still breaching the same
+     * criteria. This is the whole point of per-host incidents.
+     */
+    if (
+      input.breachingSeriesFingerprints !== undefined &&
+      openSeriesFingerprint
+    ) {
+      const stillBreaching: boolean = input.breachingSeriesFingerprints.has(
+        openSeriesFingerprint,
+      );
+
+      if (stillBreaching) {
+        return false;
+      }
+
+      /*
+       * Series no longer breaching. Only auto-close if the criteria
+       * was configured to auto-resolve in the first place; otherwise
+       * stay open so a human can acknowledge.
+       */
+      if (!input.openIncident.createdCriteriaId?.toString()) {
+        return false;
+      }
+
+      if (!input.openIncident.createdIncidentTemplateId?.toString()) {
+        return false;
+      }
+
+      const autoResolveTemplates: Array<string> | undefined =
+        input.autoResolveCriteriaInstanceIdIncidentIdsDictionary[
+          input.openIncident.createdCriteriaId.toString()
+        ];
+
+      if (
+        autoResolveTemplates &&
+        autoResolveTemplates.includes(
+          input.openIncident.createdIncidentTemplateId.toString(),
+        )
+      ) {
+        return true;
+      }
+
+      return false;
+    }
+
     if (
       input.openIncident.createdCriteriaId?.toString() ===
       input.criteriaInstance?.data?.id.toString()