@oneuptime/common 10.0.70 → 10.0.72

package/Server/Utils/Monitor/MonitorIncident.ts

@@ -17,8 +17,10 @@ import { TelemetryQuery } from "../../../Types/Telemetry/TelemetryQuery";
 import { DisableAutomaticIncidentCreation } from "../../EnvironmentConfig";
 import IncidentService from "../../Services/IncidentService";
 import IncidentSeverityService from "../../Services/IncidentSeverityService";
+import IncidentStateService from "../../Services/IncidentStateService";
 import IncidentStateTimelineService from "../../Services/IncidentStateTimelineService";
 import IncidentMemberService from "../../Services/IncidentMemberService";
+import IncidentState from "../../../Models/DatabaseModels/IncidentState";
 import logger, { LogAttributes } from "../Logger";
 import CaptureSpan from "../Telemetry/CaptureSpan";
 import DataToProcess from "./DataToProcess";
@@ -27,6 +29,7 @@ import { JSONObject } from "../../../Types/JSON";
 import OneUptimeDate from "../../../Types/Date";
 import MonitorEvaluationSummary from "../../../Types/Monitor/MonitorEvaluationSummary";
 import { IncidentMemberRoleAssignment } from "../../../Types/Monitor/CriteriaIncident";
+import { PerSeriesCriteriaMatch } from "../../../Types/Probe/ProbeApiIngestResponse";
 
 export default class MonitorIncident {
   @CaptureSpan()
@@ -39,6 +42,15 @@ export default class MonitorIncident {
     criteriaInstance: MonitorCriteriaInstance | null;
     dataToProcess: DataToProcess;
     evaluationSummary?: MonitorEvaluationSummary | undefined;
+    /**
+     * When set, the fingerprint set of series still breaching on this
+     * tick. Any open per-series incident whose fingerprint is NOT in
+     * this set is auto-resolved — that's how a series returning to
+     * normal closes its incident independently of other series on the
+     * same monitor. Undefined means "legacy mode" and per-series
+     * incidents are treated like any other for dedupe/resolve.
+     */
+    breachingSeriesFingerprints?: Set<string> | undefined;
   }): Promise<Array<Incident>> {
     // check active incidents and if there are open incidents, do not cretae anothr incident.
     const openIncidents: Array<Incident> = await IncidentService.findBy({
@@ -52,11 +64,15 @@ export default class MonitorIncident {
       limit: LIMIT_PER_PROJECT,
       select: {
         _id: true,
+        title: true,
         createdCriteriaId: true,
         createdIncidentTemplateId: true,
         projectId: true,
         incidentNumber: true,
         incidentNumberWithPrefix: true,
+        currentIncidentStateId: true,
+        seriesFingerprint: true,
+        seriesLabels: true,
       },
       props: {
         isRoot: true,
@@ -71,6 +87,7 @@ export default class MonitorIncident {
         autoResolveCriteriaInstanceIdIncidentIdsDictionary:
           input.autoResolveCriteriaInstanceIdIncidentIdsDictionary,
         criteriaInstance: input.criteriaInstance,
+        breachingSeriesFingerprints: input.breachingSeriesFingerprints,
       });
 
       if (shouldClose) {
@@ -112,6 +129,13 @@ export default class MonitorIncident {
     props: {
       telemetryQuery?: TelemetryQuery | undefined;
     };
+    /**
+     * When set, create one incident per series instead of one per
+     * monitor. Each entry gets its own rootCause, seriesFingerprint,
+     * and seriesLabels so the incident title + description can
+     * reference `{{host.name}}` etc. via the template engine.
+     */
+    matchesPerSeries?: Array<PerSeriesCriteriaMatch> | undefined;
   }): Promise<void> {
     const incidentLogAttributes: LogAttributes = {
       projectId: input.monitor.projectId?.toString(),
@@ -122,6 +146,21 @@ export default class MonitorIncident {
       `${input.monitor.id?.toString()} - Check open incidents.`,
       incidentLogAttributes,
     );
+
+    /*
+     * Per-series mode: close any open incident for a series that's no
+     * longer breaching *before* we look at the remaining open set, so
+     * dedupe decisions below match the post-resolve state.
+     */
+    const breachingSeriesFingerprints: Set<string> | undefined =
+      input.matchesPerSeries
+        ? new Set<string>(
+            input.matchesPerSeries.map((m: PerSeriesCriteriaMatch) => {
+              return m.fingerprint;
+            }),
+          )
+        : undefined;
+
     // check active incidents and if there are open incidents, do not cretae anothr incident.
     const openIncidents: Array<Incident> =
       await this.checkOpenIncidentsAndCloseIfResolved({
@@ -132,14 +171,29 @@ export default class MonitorIncident {
         criteriaInstance: input.criteriaInstance,
         dataToProcess: input.dataToProcess,
         evaluationSummary: input.evaluationSummary,
+        breachingSeriesFingerprints,
       });
 
-    if (input.criteriaInstance.data?.createIncidents) {
-      // create incidents
+    if (!input.criteriaInstance.data?.createIncidents) {
+      return;
+    }
 
-      for (const criteriaIncident of input.criteriaInstance.data?.incidents ||
-        []) {
-        // should create incident.
+    /*
+     * Series-less path: one incident per criteriaIncident template as
+     * before. Series-aware path: one incident per (series × template).
+     */
+    const seriesToProcess: Array<PerSeriesCriteriaMatch | undefined> =
+      input.matchesPerSeries && input.matchesPerSeries.length > 0
+        ? input.matchesPerSeries
+        : [undefined];
+
+    for (const criteriaIncident of input.criteriaInstance.data?.incidents ||
+      []) {
+      for (const seriesMatch of seriesToProcess) {
+        const seriesFingerprint: string | undefined = seriesMatch?.fingerprint;
+        const seriesLabels: JSONObject | undefined = seriesMatch?.labels;
+        const seriesRootCause: string =
+          seriesMatch?.rootCause || input.rootCause;
 
         const alreadyOpenIncident: Incident | undefined = openIncidents.find(
           (incident: Incident) => {
@@ -147,7 +201,8 @@ export default class MonitorIncident {
               incident.createdCriteriaId ===
                 input.criteriaInstance.data?.id.toString() &&
               incident.createdIncidentTemplateId ===
-                criteriaIncident.id.toString()
+                criteriaIncident.id.toString() &&
+              (incident.seriesFingerprint || undefined) === seriesFingerprint
             );
           },
         );
@@ -165,9 +220,20 @@ export default class MonitorIncident {
         );
 
         if (hasAlreadyOpenIncident) {
+          /*
+           * Use the open incident's already-rendered title when
+           * available — the template (`criteriaIncident.title`) still
+           * contains unresolved `{{…}}` placeholders because it's the
+           * criterion's template string, not the instance's rendered
+           * output. Falling back to the template only when the open
+           * incident somehow has no title.
+           */
+          const renderedTitle: string =
+            alreadyOpenIncident?.title || criteriaIncident.title;
+
           input.evaluationSummary?.events.push({
             type: "incident-skipped",
-            title: `Incident already active: ${criteriaIncident.title}`,
+            title: `Incident already active: ${renderedTitle}`,
             message:
               "Skipped creating a new incident because an active incident exists for this criteria.",
             relatedCriteriaId: input.criteriaInstance.data?.id,
@@ -190,6 +256,8 @@ export default class MonitorIncident {
           MonitorTemplateUtil.buildTemplateStorageMap({
             monitorType: input.monitor.monitorType!,
             dataToProcess: input.dataToProcess,
+            monitor: input.monitor,
+            seriesLabels,
           });
 
         incident.title = MonitorTemplateUtil.processTemplateString({
@@ -233,7 +301,7 @@ export default class MonitorIncident {
 
         incident.monitors = [input.monitor];
         incident.projectId = input.monitor.projectId!;
-        incident.rootCause = input.rootCause;
+        incident.rootCause = seriesRootCause;
         incident.createdStateLog = JSON.parse(
           JSON.stringify(input.dataToProcess, null, 2),
         );
@@ -242,6 +310,13 @@ export default class MonitorIncident {
 
         incident.createdIncidentTemplateId = criteriaIncident.id.toString();
 
+        if (seriesFingerprint) {
+          incident.seriesFingerprint = seriesFingerprint;
+        }
+        if (seriesLabels && Object.keys(seriesLabels).length > 0) {
+          incident.seriesLabels = seriesLabels;
+        }
+
         incident.onCallDutyPolicies =
           criteriaIncident.onCallPolicyIds?.map((id: ObjectID) => {
             const onCallPolicy: OnCallDutyPolicy = new OnCallDutyPolicy();
@@ -393,6 +468,82 @@ export default class MonitorIncident {
         input.openIncident.projectId!,
       );
 
+    /*
+     * Skip the Resolved insert if the incident's timeline is already at or
+     * past the Resolved state in the project's workflow order. Two cases:
+     *  1. Latest timeline state is Resolved but Incident.currentIncidentStateId
+     *     is stuck on an earlier state (partial-failure from a prior resolve).
+     *     Re-inserting Resolved would throw "state cannot be same as previous"
+     *     from IncidentStateTimelineService.onBeforeCreate.
+     *  2. The project defines a custom state after Resolved (e.g. Closed) and
+     *     the incident has moved into it. Inserting Resolved would throw
+     *     "cannot transition to Resolved from Closed because Resolved is
+     *     before Closed in the order of incident states."
+     * Either failure bubbles up through ingest workers and causes monitors to
+     * flap. Reconcile Incident.currentIncidentStateId if out of sync with the
+     * timeline, then return.
+     */
+    const [resolvedState, latestTimeline]: [
+      IncidentState | null,
+      IncidentStateTimeline | null,
+    ] = await Promise.all([
+      IncidentStateService.findOneBy({
+        query: {
+          _id: resolvedStateId.toString(),
+        },
+        select: {
+          order: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      }),
+      IncidentStateTimelineService.findOneBy({
+        query: {
+          incidentId: input.openIncident.id!,
+        },
+        sort: {
+          startsAt: SortOrder.Descending,
+        },
+        select: {
+          incidentStateId: true,
+          incidentState: {
+            order: true,
+          },
+        },
+        props: {
+          isRoot: true,
+        },
+      }),
+    ]);
+
+    const latestOrder: number | undefined | null =
+      latestTimeline?.incidentState?.order;
+    const resolvedOrder: number | undefined | null = resolvedState?.order;
+
+    if (
+      latestTimeline?.incidentStateId &&
+      typeof latestOrder === "number" &&
+      typeof resolvedOrder === "number" &&
+      latestOrder >= resolvedOrder
+    ) {
+      if (
+        input.openIncident.currentIncidentStateId?.toString() !==
+        latestTimeline.incidentStateId.toString()
+      ) {
+        await IncidentService.updateOneById({
+          id: input.openIncident.id!,
+          data: {
+            currentIncidentStateId: latestTimeline.incidentStateId,
+          },
+          props: {
+            isRoot: true,
+          },
+        });
+      }
+      return;
+    }
+
     const incidentStateTimeline: IncidentStateTimeline =
       new IncidentStateTimeline();
     incidentStateTimeline.incidentId = input.openIncident.id!;
@@ -425,7 +576,60 @@ export default class MonitorIncident {
       Array<string>
     >;
     criteriaInstance: MonitorCriteriaInstance | null; // null if no criteia met.
+    breachingSeriesFingerprints?: Set<string> | undefined;
   }): boolean {
+    const openSeriesFingerprint: string | undefined =
+      input.openIncident.seriesFingerprint || undefined;
+
+    /*
+     * Per-series auto-resolve: when the monitor emits a breaching-
+     * series set and this open incident has a fingerprint, resolve
+     * whenever the fingerprint is no longer in the set — regardless
+     * of whether some *other* series is still breaching the same
+     * criteria. This is the whole point of per-host incidents.
+     */
+    if (
+      input.breachingSeriesFingerprints !== undefined &&
+      openSeriesFingerprint
+    ) {
+      const stillBreaching: boolean = input.breachingSeriesFingerprints.has(
+        openSeriesFingerprint,
+      );
+
+      if (stillBreaching) {
+        return false;
+      }
+
+      /*
+       * Series no longer breaching. Only auto-close if the criteria
+       * was configured to auto-resolve in the first place; otherwise
+       * stay open so a human can acknowledge.
+       */
+      if (!input.openIncident.createdCriteriaId?.toString()) {
+        return false;
+      }
+
+      if (!input.openIncident.createdIncidentTemplateId?.toString()) {
+        return false;
+      }
+
+      const autoResolveTemplates: Array<string> | undefined =
+        input.autoResolveCriteriaInstanceIdIncidentIdsDictionary[
+          input.openIncident.createdCriteriaId.toString()
+        ];
+
+      if (
+        autoResolveTemplates &&
+        autoResolveTemplates.includes(
+          input.openIncident.createdIncidentTemplateId.toString(),
+        )
+      ) {
+        return true;
+      }
+
+      return false;
+    }
+
     if (
       input.openIncident.createdCriteriaId?.toString() ===
       input.criteriaInstance?.data?.id.toString()