@oneuptime/common 11.3.5 → 11.3.6

package/Server/Utils/Monitor/MonitorIncident.ts

@@ -72,6 +72,14 @@ export default class MonitorIncident {
      * incidents are treated like any other for dedupe/resolve.
      */
     breachingSeriesFingerprints?: Set<string> | undefined;
+    /**
+     * Event-driven (incoming-request / webhook) mode. When true, an open
+     * incident carrying a seriesFingerprint is never auto-resolved here —
+     * webhooks resolve per-key only via resolveSeriesIncidentsByFingerprint,
+     * never by absence. Must be passed on BOTH the criteria-met and the
+     * no-criteria-met code paths for grouped incoming-request monitors.
+     */
+    disableSeriesAbsenceResolution?: boolean | undefined;
   }): Promise<Array<Incident>> {
     // check active incidents and if there are open incidents, do not create another incident.
     const openIncidents: Array<Incident> = await IncidentService.findBy({
@@ -109,6 +117,7 @@ export default class MonitorIncident {
           input.autoResolveCriteriaInstanceIdIncidentIdsDictionary,
         criteriaInstance: input.criteriaInstance,
         breachingSeriesFingerprints: input.breachingSeriesFingerprints,
+        disableSeriesAbsenceResolution: input.disableSeriesAbsenceResolution,
       });
 
       if (shouldClose) {
@@ -137,6 +146,107 @@ export default class MonitorIncident {
     return openIncidents;
   }
 
+  /**
+   * Event-driven (incoming-request / webhook) resolution: resolve the
+   * open incidents for the given payload-derived fingerprints — and only
+   * those — when the criteria that created them has auto-resolve enabled.
+   *
+   * Unlike the metric snapshot model, this never resolves an incident by
+   * absence: a webhook describes only the keys in its payload, so a
+   * missing key is not a recovery signal. The caller passes exactly the
+   * fingerprints the payload explicitly classified as resolved (see
+   * IncomingRequestIncidentGrouping.collectResolvedFingerprints).
+   */
+  @CaptureSpan()
+  public static async resolveSeriesIncidentsByFingerprint(input: {
+    monitor: Monitor;
+    fingerprints: Array<string>;
+    rootCause: string;
+    dataToProcess: DataToProcess;
+    autoResolveCriteriaInstanceIdIncidentIdsDictionary: Dictionary<
+      Array<string>
+    >;
+    evaluationSummary?: MonitorEvaluationSummary | undefined;
+  }): Promise<void> {
+    if (!input.fingerprints || input.fingerprints.length === 0) {
+      return;
+    }
+
+    const fingerprintSet: Set<string> = new Set<string>(input.fingerprints);
+
+    const openIncidents: Array<Incident> = await IncidentService.findBy({
+      query: {
+        monitors: [input.monitor.id!],
+        currentIncidentState: {
+          isResolvedState: false,
+        },
+      },
+      skip: 0,
+      limit: LIMIT_PER_PROJECT,
+      select: {
+        _id: true,
+        title: true,
+        createdCriteriaId: true,
+        createdIncidentTemplateId: true,
+        projectId: true,
+        incidentNumber: true,
+        incidentNumberWithPrefix: true,
+        seriesFingerprint: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    for (const openIncident of openIncidents) {
+      const fingerprint: string | undefined =
+        openIncident.seriesFingerprint || undefined;
+
+      if (!fingerprint || !fingerprintSet.has(fingerprint)) {
+        continue;
+      }
+
+      const createdCriteriaId: string | undefined =
+        openIncident.createdCriteriaId?.toString();
+      const createdIncidentTemplateId: string | undefined =
+        openIncident.createdIncidentTemplateId?.toString();
+
+      // Only auto-resolve when the creating criteria opted into it.
+      if (!createdCriteriaId || !createdIncidentTemplateId) {
+        continue;
+      }
+
+      const autoResolveTemplates: Array<string> | undefined =
+        input.autoResolveCriteriaInstanceIdIncidentIdsDictionary[
+          createdCriteriaId
+        ];
+
+      if (
+        !autoResolveTemplates ||
+        !autoResolveTemplates.includes(createdIncidentTemplateId)
+      ) {
+        continue;
+      }
+
+      await this.resolveOpenIncident({
+        openIncident: openIncident,
+        rootCause: input.rootCause,
+        dataToProcess: input.dataToProcess,
+      });
+
+      input.evaluationSummary?.events.push({
+        type: "incident-resolved",
+        title: `Incident resolved: ${openIncident.id?.toString()}`,
+        message:
+          "Incident auto-resolved because the incoming payload reported this key as resolved.",
+        relatedIncidentId: openIncident.id?.toString(),
+        relatedIncidentNumber: openIncident.incidentNumber,
+        relatedIncidentNumberWithPrefix: openIncident.incidentNumberWithPrefix,
+        at: OneUptimeDate.getCurrentDate(),
+      });
+    }
+  }
+
   @CaptureSpan()
   public static async criteriaMetCreateIncidentsAndUpdateMonitorStatus(input: {
     criteriaInstance: MonitorCriteriaInstance;
@@ -165,6 +275,17 @@ export default class MonitorIncident {
      * are suppressed at creation time. See MonitorMaintenanceSuppression.
      */
     suppressedSeriesFingerprints?: Set<string> | undefined;
+    /**
+     * Event-driven monitors (incoming-request / webhook fan-out) must not
+     * use the metric snapshot model where a series absent from this tick's
+     * breaching set is auto-resolved — a single webhook only describes the
+     * keys in that payload, not the full firing set, so absence is not
+     * recovery. When true, the per-series absence-resolve pass is skipped;
+     * those incidents are resolved explicitly elsewhere (see
+     * IncomingRequestIncidentGrouping + resolveSeriesIncidentsByFingerprint).
+     * Per-key create + dedupe still happen via matchesPerSeries.
+     */
+    disableSeriesAbsenceResolution?: boolean | undefined;
   }): Promise<void> {
     const incidentLogAttributes: LogAttributes = {
       projectId: input.monitor.projectId?.toString(),
@@ -182,7 +303,7 @@ export default class MonitorIncident {
      * dedupe decisions below match the post-resolve state.
      */
     const breachingSeriesFingerprints: Set<string> | undefined =
-      input.matchesPerSeries
+      input.matchesPerSeries && !input.disableSeriesAbsenceResolution
         ? new Set<string>(
             input.matchesPerSeries.map((m: PerSeriesCriteriaMatch) => {
               return m.fingerprint;
@@ -201,6 +322,7 @@ export default class MonitorIncident {
         dataToProcess: input.dataToProcess,
         evaluationSummary: input.evaluationSummary,
         breachingSeriesFingerprints,
+        disableSeriesAbsenceResolution: input.disableSeriesAbsenceResolution,
       });
 
     if (!input.criteriaInstance.data?.createIncidents) {
@@ -224,8 +346,15 @@ export default class MonitorIncident {
      * Series-less path: one incident per criteriaIncident template as
      * before. Series-aware path: one incident per (series × template).
      */
+    /*
+     * `undefined` matchesPerSeries → legacy single-incident path. A
+     * defined (even empty) array → per-series mode: iterate exactly the
+     * matches. An empty array therefore creates nothing — used by grouped
+     * incoming-request criteria on a payload with no firing key so they
+     * don't fall back to a single whole-monitor incident.
+     */
     const seriesToProcess: Array<PerSeriesCriteriaMatch | undefined> =
-      input.matchesPerSeries && input.matchesPerSeries.length > 0
+      input.matchesPerSeries !== undefined
         ? input.matchesPerSeries
         : [undefined];
 
@@ -981,10 +1110,26 @@ export default class MonitorIncident {
     >;
     criteriaInstance: MonitorCriteriaInstance | null; // null if no criteia met.
     breachingSeriesFingerprints?: Set<string> | undefined;
+    disableSeriesAbsenceResolution?: boolean | undefined;
   }): boolean {
     const openSeriesFingerprint: string | undefined =
       input.openIncident.seriesFingerprint || undefined;
 
+    /*
+     * Event-driven (incoming-request / webhook) per-key incidents must
+     * NEVER be resolved by absence — only explicitly, via
+     * resolveSeriesIncidentsByFingerprint, when the payload reports the
+     * key as recovered. A single webhook describes only the keys in its
+     * own payload, so neither the per-series breaching-set path nor the
+     * legacy cross-criteria path below may close a series incident here.
+     * Without this guard, a heartbeat-timeout cron tick or a webhook that
+     * the grouping criteria rejects would bulk-resolve all open per-key
+     * incidents by absence.
+     */
+    if (input.disableSeriesAbsenceResolution && openSeriesFingerprint) {
+      return false;
+    }
+
     /*
      * Per-series auto-resolve: when the monitor emits a breaching-
      * series set and this open incident has a fingerprint, resolve

package/Server/Utils/Monitor/MonitorResource.ts

@@ -35,6 +35,7 @@ import ExceptionMonitorResponse from "../../../Types/Monitor/ExceptionMonitor/Ex
 import { TelemetryQuery } from "../../../Types/Telemetry/TelemetryQuery";
 import MonitorIncident from "./MonitorIncident";
 import MonitorAlert from "./MonitorAlert";
+import IncomingRequestIncidentGrouping from "./IncomingRequestIncidentGrouping";
 import MonitorMaintenanceSuppression from "./MonitorMaintenanceSuppression";
 import MonitorStatusTimelineUtil from "./MonitorStatusTimeline";
 import CaptureSpan from "../Telemetry/CaptureSpan";
@@ -659,6 +660,49 @@ export default class MonitorResourceUtil {
         }
       }
 
+      /*
+       * Incoming Request / webhook grouped resolution (event-driven). A
+       * payload can explicitly mark keys as resolved (e.g. Grafana
+       * status=resolved). Resolve exactly those keys' incidents — never by
+       * absence. Runs regardless of whether a firing criteria matched, so a
+       * pure "resolved" webhook still closes the right incident. No-op
+       * unless a criteria has incidentGrouping configured.
+       */
+      if (
+        monitor.monitorType === MonitorType.IncomingRequest &&
+        criteriaInstances.some((criteriaInstance: MonitorCriteriaInstance) => {
+          return IncomingRequestIncidentGrouping.isGroupingConfigured(
+            criteriaInstance,
+          );
+        })
+      ) {
+        const resolvedFingerprints: Array<string> =
+          IncomingRequestIncidentGrouping.collectResolvedFingerprints({
+            dataToProcess: dataToProcess,
+            criteriaInstances: criteriaInstances,
+          });
+
+        if (resolvedFingerprints.length > 0) {
+          await MonitorIncident.resolveSeriesIncidentsByFingerprint({
+            monitor: monitor,
+            fingerprints: resolvedFingerprints,
+            rootCause: "Incoming request reported this key as resolved.",
+            dataToProcess: dataToProcess,
+            autoResolveCriteriaInstanceIdIncidentIdsDictionary,
+            evaluationSummary: evaluationSummary,
+          });
+
+          await MonitorAlert.resolveSeriesAlertsByFingerprint({
+            monitor: monitor,
+            fingerprints: resolvedFingerprints,
+            rootCause: "Incoming request reported this key as resolved.",
+            dataToProcess: dataToProcess,
+            autoResolveCriteriaInstanceIdAlertIdsDictionary,
+            evaluationSummary: evaluationSummary,
+          });
+        }
+      }
+
       if (response.criteriaMetId && response.rootCause) {
         logger.debug(
           `${dataToProcess.monitorId.toString()} - Criteria met: ${
@@ -796,6 +840,15 @@ export default class MonitorResourceUtil {
           },
           matchesPerSeries: response.perSeriesMatches,
           suppressedSeriesFingerprints,
+          /*
+           * Incoming-request grouping is event-driven: a webhook describes
+           * only the keys in its payload, so absence from this tick is not
+           * recovery. Skip the snapshot absence-resolve pass; grouped
+           * incidents are resolved explicitly via the resolution block
+           * above. Per-key create + dedupe still run from matchesPerSeries.
+           */
+          disableSeriesAbsenceResolution:
+            monitor.monitorType === MonitorType.IncomingRequest,
         });
 
         await MonitorAlert.criteriaMetCreateAlertsAndUpdateMonitorStatus({
@@ -810,6 +863,14 @@ export default class MonitorResourceUtil {
           },
           matchesPerSeries: response.perSeriesMatches,
           suppressedSeriesFingerprints,
+          /*
+           * Incoming-request grouping is event-driven (see the incident
+           * create call above): skip the snapshot absence-resolve pass for
+           * webhooks; grouped alerts are resolved explicitly via the
+           * resolution block above. Per-key create + dedupe still run.
+           */
+          disableSeriesAbsenceResolution:
+            monitor.monitorType === MonitorType.IncomingRequest,
         });
       } else if (
         !response.criteriaMetId &&
@@ -828,6 +889,17 @@ export default class MonitorResourceUtil {
           criteriaInstance: null, // no criteria met!
           dataToProcess: dataToProcess,
           evaluationSummary: evaluationSummary,
+          /*
+           * Event-driven grouping: for incoming-request monitors, never
+           * absence-resolve per-key (seriesFingerprint) incidents on the
+           * no-criteria-met path — a heartbeat tick or a rejected webhook
+           * must not bulk-close grouped incidents. They resolve only via
+           * the explicit resolution block above. Non-grouped incoming
+           * incidents have no seriesFingerprint, so they still resolve
+           * normally; non-incoming-request types are unaffected (flag false).
+           */
+          disableSeriesAbsenceResolution:
+            monitor.monitorType === MonitorType.IncomingRequest,
         });
 
         await MonitorAlert.checkOpenAlertsAndCloseIfResolved({
@@ -837,6 +909,8 @@ export default class MonitorResourceUtil {
           criteriaInstance: null, // no criteria met!
           dataToProcess: dataToProcess,
           evaluationSummary: evaluationSummary,
+          disableSeriesAbsenceResolution:
+            monitor.monitorType === MonitorType.IncomingRequest,
         });
 
         // get last monitor status timeline.

package/Tests/Server/Utils/AnalyticsDatabase/ClusterAwareSchema.test.ts

@@ -256,7 +256,8 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
       expect(dropColumn).toContain("SpanItemV3Local");
       expect(dropColumn).toContain("ON CLUSTER 'oneuptime'");
 
-      const dropIndex: string = spanGen.toDropSkipIndexStatement("idx_trace_id");
+      const dropIndex: string =
+        spanGen.toDropSkipIndexStatement("idx_trace_id");
       expect(dropIndex).toContain("SpanItemV3Local");
       expect(dropIndex).toContain("ON CLUSTER 'oneuptime'");
     });

package/Tests/Server/Utils/AnalyticsDatabase/StatementGenerator.test.ts

@@ -19,6 +19,7 @@ import IsNull from "../../../../Types/BaseDatabase/IsNull";
 import NotNull from "../../../../Types/BaseDatabase/NotNull";
 import GreaterThan from "../../../../Types/BaseDatabase/GreaterThan";
 import Includes from "../../../../Types/BaseDatabase/Includes";
+import IncludesNone from "../../../../Types/BaseDatabase/IncludesNone";
 import Search from "../../../../Types/BaseDatabase/Search";
 import StartsWith from "../../../../Types/BaseDatabase/StartsWith";
 
@@ -363,6 +364,37 @@ describe("StatementGenerator", () => {
         expect(statement.query).toBe("");
         expect(statement.query_params).toStrictEqual({});
       });
+
+      test("emits direct map subscript NOT IN(...) for IncludesNone wrapper", () => {
+        const statement: Statement = mapGenerator.toWhereStatement({
+          attributes: {
+            "k8s.cluster.name": new IncludesNone(["prod-east", "prod-west"]),
+          },
+        } as any);
+        /*
+         * "is none of" emits the same O(1) Map subscript fast path as
+         * IN, negated. Missing keys (subscript returns '') pass NOT IN,
+         * matching NotEqual's map semantics.
+         */
+        expect(statement.query).toBe(
+          "AND {p0:Identifier}[{p1:String}] NOT IN {p2:Array(String)}",
+        );
+        expect(statement.query_params).toStrictEqual({
+          p0: "attributes",
+          p1: "k8s.cluster.name",
+          p2: ["prod-east", "prod-west"],
+        });
+      });
+
+      test("drops empty IncludesNone wrapper instead of producing NOT IN ()", () => {
+        const statement: Statement = mapGenerator.toWhereStatement({
+          attributes: {
+            "k8s.cluster.name": new IncludesNone([]),
+          },
+        } as any);
+        expect(statement.query).toBe("");
+        expect(statement.query_params).toStrictEqual({});
+      });
     });
 
     describe("ArrayText columns", () => {