@oneuptime/common 11.3.5 → 11.3.6

package/Server/Utils/Monitor/IncomingRequestIncidentGrouping.ts

@@ -0,0 +1,428 @@
+import logger from "../Logger";
+import VMUtil from "../VM/VMAPI";
+import MetricSeriesFingerprint from "../../../Utils/Metrics/MetricSeriesFingerprint";
+import DataToProcess from "./DataToProcess";
+import IncomingMonitorRequest from "../../../Types/Monitor/IncomingMonitor/IncomingMonitorRequest";
+import IncidentGroupingConfig from "../../../Types/Monitor/IncomingMonitor/IncidentGroupingConfig";
+import MonitorCriteriaInstance from "../../../Types/Monitor/MonitorCriteriaInstance";
+import { PerSeriesCriteriaMatch } from "../../../Types/Probe/ProbeApiIngestResponse";
+import { JSONObject, JSONValue } from "../../../Types/JSON";
+import Typeof from "../../../Types/Typeof";
+
+/**
+ * One key extracted from an incoming webhook payload by an
+ * {@link IncidentGroupingConfig}.
+ */
+export interface IncomingRequestGroupingItem {
+  /** Stable dedupe key derived from the extracted value. */
+  fingerprint: string;
+  /** Label map exposed to incident title/description templates. */
+  labels: JSONObject;
+  /** The raw extracted value (e.g. the Grafana alert name). */
+  keyValue: string;
+  /** True when the payload classifies this key as resolved/recovered. */
+  isResolved: boolean;
+}
+
+const DEFAULT_MAX_KEYS_PER_PAYLOAD: number = 100;
+const ARRAY_WILDCARD: string = "[*]";
+
+/**
+ * Turns an incoming-request / webhook payload into per-key incident
+ * matches, so a single monitor can hold multiple concurrent incidents —
+ * one per distinct value (Grafana alert name, fingerprint, etc.). This
+ * reuses the same `seriesFingerprint` mechanism metric monitors use for
+ * per-series incidents; see {@link IncidentGroupingConfig}.
+ */
+export default class IncomingRequestIncidentGrouping {
+  /**
+   * Parse the request body into a JSON object. Bodies arrive either
+   * pre-parsed (JSONObject) or as a raw JSON string depending on the
+   * ingestion path; both are handled. Non-JSON / non-object bodies
+   * (e.g. a bare string heartbeat) return null — grouping is a no-op.
+   */
+  public static getRequestBodyObject(
+    dataToProcess: DataToProcess,
+  ): JSONObject | null {
+    const body: string | JSONObject | undefined = (
+      dataToProcess as IncomingMonitorRequest
+    ).requestBody;
+
+    if (!body) {
+      return null;
+    }
+
+    if (typeof body === Typeof.String) {
+      try {
+        const parsed: unknown = JSON.parse(body as string);
+        return parsed &&
+          typeof parsed === Typeof.Object &&
+          !Array.isArray(parsed)
+          ? (parsed as JSONObject)
+          : null;
+      } catch {
+        return null;
+      }
+    }
+
+    if (typeof body === Typeof.Object && !Array.isArray(body)) {
+      return body as JSONObject;
+    }
+
+    return null;
+  }
+
+  /**
+   * Firing matches for one criteria — used by the criteria evaluator to
+   * fan a matched incoming-request criteria into one incident per key.
+   * Returns an empty array when grouping is not configured (preserving
+   * the legacy single-incident behaviour) or when there is no usable
+   * payload (e.g. the heartbeat-timeout re-evaluation cron, which carries
+   * no fresh body).
+   */
+  public static collectFiringMatches(input: {
+    dataToProcess: DataToProcess;
+    criteriaInstance: MonitorCriteriaInstance;
+    rootCause?: string | undefined;
+  }): Array<PerSeriesCriteriaMatch> {
+    const grouping: IncidentGroupingConfig | undefined =
+      input.criteriaInstance.data?.incidentGrouping;
+
+    if (!grouping?.groupByJSONPath) {
+      return [];
+    }
+
+    if (
+      (input.dataToProcess as IncomingMonitorRequest)
+        .onlyCheckForIncomingRequestReceivedAt
+    ) {
+      return [];
+    }
+
+    const criteriaId: string | undefined = input.criteriaInstance.data?.id;
+    if (!criteriaId) {
+      return [];
+    }
+
+    const body: JSONObject | null = this.getRequestBodyObject(
+      input.dataToProcess,
+    );
+    if (!body) {
+      return [];
+    }
+
+    const matches: Array<PerSeriesCriteriaMatch> = [];
+
+    for (const item of this.extractItems({ requestBody: body, grouping })) {
+      if (item.isResolved) {
+        // A resolved event never opens an incident.
+        continue;
+      }
+
+      matches.push({
+        criteriaMetId: criteriaId,
+        fingerprint: item.fingerprint,
+        labels: item.labels,
+        rootCause:
+          input.rootCause ||
+          `Incoming request matched grouping key \`${item.keyValue}\`.`,
+        metricContext: undefined,
+      });
+    }
+
+    return matches;
+  }
+
+  /**
+   * Fingerprints the payload explicitly classifies as resolved, across
+   * every grouping-enabled criteria on the monitor's step. The caller
+   * resolves the matching open incidents (event-driven resolution: only
+   * keys the payload says recovered are closed — never by absence).
+   */
+  public static collectResolvedFingerprints(input: {
+    dataToProcess: DataToProcess;
+    criteriaInstances: Array<MonitorCriteriaInstance>;
+  }): Array<string> {
+    if (
+      (input.dataToProcess as IncomingMonitorRequest)
+        .onlyCheckForIncomingRequestReceivedAt
+    ) {
+      return [];
+    }
+
+    const body: JSONObject | null = this.getRequestBodyObject(
+      input.dataToProcess,
+    );
+    if (!body) {
+      return [];
+    }
+
+    const fingerprints: Set<string> = new Set<string>();
+
+    for (const criteriaInstance of input.criteriaInstances) {
+      const grouping: IncidentGroupingConfig | undefined =
+        criteriaInstance.data?.incidentGrouping;
+
+      if (!grouping?.groupByJSONPath) {
+        continue;
+      }
+
+      for (const item of this.extractItems({ requestBody: body, grouping })) {
+        if (item.isResolved) {
+          fingerprints.add(item.fingerprint);
+        }
+      }
+    }
+
+    return Array.from(fingerprints);
+  }
+
+  /**
+   * Whether any criteria on the step has grouping configured. Used to
+   * force per-series mode so a grouped criteria never falls back to a
+   * single whole-monitor incident on a payload that yields no firing key.
+   */
+  public static isGroupingConfigured(
+    criteriaInstance: MonitorCriteriaInstance | undefined,
+  ): boolean {
+    return Boolean(criteriaInstance?.data?.incidentGrouping?.groupByJSONPath);
+  }
+
+  /**
+   * Extract all distinct keys from a payload for one grouping config,
+   * classifying each as firing or resolved. Deduplicates by fingerprint
+   * and caps the count at `maxKeysPerPayload` to bound cardinality.
+   */
+  public static extractItems(input: {
+    requestBody: JSONObject;
+    grouping: IncidentGroupingConfig;
+  }): Array<IncomingRequestGroupingItem> {
+    const { requestBody, grouping } = input;
+
+    const groupByPath: string = this.normalizePath(grouping.groupByJSONPath);
+    if (!groupByPath) {
+      return [];
+    }
+
+    const labelKey: string = this.labelKeyFromPath(groupByPath);
+    const maxKeys: number =
+      grouping.maxKeysPerPayload && grouping.maxKeysPerPayload > 0
+        ? grouping.maxKeysPerPayload
+        : DEFAULT_MAX_KEYS_PER_PAYLOAD;
+
+    const items: Array<IncomingRequestGroupingItem> = [];
+    const seenFingerprints: Set<string> = new Set<string>();
+
+    const addItem: (keyValue: string, isResolved: boolean) => void = (
+      keyValue: string,
+      isResolved: boolean,
+    ): void => {
+      const labels: JSONObject = { [labelKey]: keyValue };
+      const fingerprint: string =
+        MetricSeriesFingerprint.computeFingerprint(labels);
+
+      if (seenFingerprints.has(fingerprint)) {
+        // Same key twice in one payload — keep the first occurrence.
+        return;
+      }
+
+      seenFingerprints.add(fingerprint);
+      items.push({ fingerprint, labels, keyValue, isResolved });
+    };
+
+    const wildcardIndex: number = groupByPath.indexOf(ARRAY_WILDCARD);
+
+    // Simple (non-array) path: a single key for the whole payload.
+    if (wildcardIndex === -1) {
+      const keyValue: string | null = this.toScalarString(
+        VMUtil.deepFind(requestBody, groupByPath),
+      );
+
+      if (keyValue === null) {
+        return [];
+      }
+
+      addItem(
+        keyValue,
+        this.isResolvedForScope(requestBody, requestBody, grouping, false),
+      );
+      return items;
+    }
+
+    // Array fan-out: one key per element of the array at the `[*]` prefix.
+    const prefixPath: string = groupByPath
+      .slice(0, wildcardIndex)
+      .replace(/\.$/, "");
+    const elementSuffix: string = this.stripLeadingDot(
+      groupByPath.slice(wildcardIndex + ARRAY_WILDCARD.length),
+    );
+
+    const arrayValue: JSONValue | undefined = prefixPath
+      ? VMUtil.deepFind(requestBody, prefixPath)
+      : (requestBody as unknown as JSONValue);
+
+    if (!Array.isArray(arrayValue)) {
+      return [];
+    }
+
+    for (const element of arrayValue) {
+      if (items.length >= maxKeys) {
+        logger.warn(
+          `IncomingRequestIncidentGrouping: payload produced more than ${maxKeys} keys for path "${groupByPath}"; remaining keys ignored.`,
+        );
+        break;
+      }
+
+      const isObjectElement: boolean =
+        Boolean(element) && typeof element === Typeof.Object;
+
+      let keyValue: string | null;
+
+      if (elementSuffix) {
+        // `prefix[*].sub.path` — element must be an object to dig into.
+        if (!isObjectElement) {
+          continue;
+        }
+        keyValue = this.toScalarString(
+          VMUtil.deepFind(element as JSONObject, elementSuffix),
+        );
+      } else {
+        // `prefix[*]` — the array element itself is the key (array of scalars).
+        keyValue = this.toScalarString(element as JSONValue);
+      }
+
+      if (keyValue === null) {
+        continue;
+      }
+
+      /*
+       * Per-element resolve classification only applies when the element
+       * is an object (so `[*].status` can be read off it). For scalar
+       * elements there is no per-element field, so fall back to root scope.
+       */
+      addItem(
+        keyValue,
+        this.isResolvedForScope(
+          isObjectElement ? (element as JSONObject) : requestBody,
+          requestBody,
+          grouping,
+          isObjectElement,
+        ),
+      );
+    }
+
+    return items;
+  }
+
+  /**
+   * Classify whether the configured resolve condition holds. When the
+   * resolve path uses `[*]` AND we are evaluating a specific array element
+   * (`isElementContext`), it is read off that element (`elementScope`).
+   * Otherwise it is evaluated against the whole payload (`rootScope`).
+   *
+   * If the resolve path uses `[*]` but the group-by path does NOT (so we
+   * are not in element context — a self-contradictory config), the path is
+   * evaluated verbatim against the root: `deepFind` does not interpret
+   * `[*]`, so it returns undefined (not resolved) rather than silently
+   * rebinding to a same-named top-level field.
+   */
+  private static isResolvedForScope(
+    elementScope: JSONObject,
+    rootScope: JSONObject,
+    grouping: IncidentGroupingConfig,
+    isElementContext: boolean,
+  ): boolean {
+    if (
+      !grouping.resolvedWhenJSONPath ||
+      grouping.resolvedWhenValue === undefined
+    ) {
+      return false;
+    }
+
+    const resolvedPath: string = this.normalizePath(
+      grouping.resolvedWhenJSONPath,
+    );
+    const wildcardIndex: number = resolvedPath.indexOf(ARRAY_WILDCARD);
+
+    let scope: JSONObject = rootScope;
+    let path: string = resolvedPath;
+
+    if (wildcardIndex !== -1 && isElementContext) {
+      scope = elementScope;
+      path = this.stripLeadingDot(
+        resolvedPath.slice(wildcardIndex + ARRAY_WILDCARD.length),
+      );
+    }
+
+    if (!path) {
+      return false;
+    }
+
+    const value: string | null = this.toScalarString(
+      VMUtil.deepFind(scope, path),
+    );
+
+    return value !== null && value === grouping.resolvedWhenValue;
+  }
+
+  /**
+   * Derive a human-friendly label key from a JSON path — the last named
+   * segment, with array/wildcard syntax stripped. `alerts[*].labels.alertname`
+   * → `alertname`. Used as the template variable name (so titles can say
+   * `{{alertname}}`) and as the fingerprint label key.
+   */
+  private static labelKeyFromPath(path: string): string {
+    const cleaned: string = path
+      .replace(/\[\*\]/g, "")
+      .replace(/\[[^\]]*\]/g, "");
+    const parts: Array<string> = cleaned.split(".").filter(Boolean);
+    return parts.length > 0 ? parts[parts.length - 1]! : "key";
+  }
+
+  private static stripLeadingDot(path: string): string {
+    return path.startsWith(".") ? path.slice(1) : path;
+  }
+
+  /**
+   * Resolve the `requestBody`-rooted path users write — the same convention
+   * incident templates and JavaScript-expression filters use — into a path
+   * relative to the body (the actual scope `deepFind` resolves against).
+   * Both of these forms are accepted and resolve identically:
+   *
+   *   `{{requestBody.alerts[*].labels.alertname}}`   (template wrapper)
+   *   `requestBody.alerts[*].labels.alertname`
+   *
+   * The `requestBody` root is REQUIRED — a path not rooted there resolves
+   * to nothing, so the field has exactly one convention.
+   */
+  private static normalizePath(rawPath: string): string {
+    let path: string = rawPath.trim();
+
+    if (path.startsWith("{{") && path.endsWith("}}")) {
+      path = path.slice(2, -2).trim();
+    }
+
+    if (path.startsWith("requestBody.")) {
+      return path.slice("requestBody.".length);
+    }
+
+    return "";
+  }
+
+  /**
+   * Coerce an extracted value to a non-empty scalar string. Objects and
+   * arrays cannot be a grouping key and return null (the item is skipped).
+   */
+  private static toScalarString(value: JSONValue | undefined): string | null {
+    if (value === undefined || value === null) {
+      return null;
+    }
+
+    if (typeof value === Typeof.Object) {
+      return null;
+    }
+
+    const stringValue: string = String(value);
+    return stringValue.length > 0 ? stringValue : null;
+  }
+}

package/Server/Utils/Monitor/MonitorAlert.ts

@@ -47,6 +47,14 @@ export default class MonitorAlert {
     dataToProcess: DataToProcess;
     evaluationSummary?: MonitorEvaluationSummary | undefined;
     breachingSeriesFingerprints?: Set<string> | undefined;
+    /**
+     * Event-driven (incoming-request / webhook) mode. When true, an open
+     * alert carrying a seriesFingerprint is never auto-resolved here —
+     * webhooks resolve per-key only via resolveSeriesAlertsByFingerprint,
+     * never by absence. Passed on BOTH the criteria-met and no-criteria-met
+     * code paths for grouped incoming-request monitors.
+     */
+    disableSeriesAbsenceResolution?: boolean | undefined;
   }): Promise<Array<Alert>> {
     // check active alerts and if there are open alerts, do not create another alert.
     const openAlerts: Array<Alert> = await AlertService.findBy({
@@ -83,6 +91,7 @@ export default class MonitorAlert {
           input.autoResolveCriteriaInstanceIdAlertIdsDictionary,
         criteriaInstance: input.criteriaInstance,
         breachingSeriesFingerprints: input.breachingSeriesFingerprints,
+        disableSeriesAbsenceResolution: input.disableSeriesAbsenceResolution,
       });
 
       if (shouldClose) {
@@ -110,6 +119,95 @@ export default class MonitorAlert {
     return openAlerts;
   }
 
+  /**
+   * Event-driven (incoming-request / webhook) resolution: resolve the open
+   * alerts for the given payload-derived fingerprints — and only those —
+   * when the criteria that created them has auto-resolve enabled. Mirrors
+   * MonitorIncident.resolveSeriesIncidentsByFingerprint; never resolves by
+   * absence (a webhook describes only the keys in its payload).
+   */
+  @CaptureSpan()
+  public static async resolveSeriesAlertsByFingerprint(input: {
+    monitor: Monitor;
+    fingerprints: Array<string>;
+    rootCause: string;
+    dataToProcess: DataToProcess;
+    autoResolveCriteriaInstanceIdAlertIdsDictionary: Dictionary<Array<string>>;
+    evaluationSummary?: MonitorEvaluationSummary | undefined;
+  }): Promise<void> {
+    if (!input.fingerprints || input.fingerprints.length === 0) {
+      return;
+    }
+
+    const fingerprintSet: Set<string> = new Set<string>(input.fingerprints);
+
+    const openAlerts: Array<Alert> = await AlertService.findBy({
+      query: {
+        monitor: input.monitor.id!,
+        currentAlertState: {
+          isResolvedState: false,
+        },
+      },
+      skip: 0,
+      limit: LIMIT_PER_PROJECT,
+      select: {
+        _id: true,
+        title: true,
+        createdCriteriaId: true,
+        projectId: true,
+        alertNumber: true,
+        alertNumberWithPrefix: true,
+        seriesFingerprint: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    for (const openAlert of openAlerts) {
+      const fingerprint: string | undefined =
+        openAlert.seriesFingerprint || undefined;
+
+      if (!fingerprint || !fingerprintSet.has(fingerprint)) {
+        continue;
+      }
+
+      const createdCriteriaId: string | undefined =
+        openAlert.createdCriteriaId?.toString();
+
+      if (!createdCriteriaId) {
+        continue;
+      }
+
+      // Only auto-resolve when the creating criteria opted into it.
+      const autoResolveTemplates: Array<string> | undefined =
+        input.autoResolveCriteriaInstanceIdAlertIdsDictionary[
+          createdCriteriaId
+        ];
+
+      if (!autoResolveTemplates || autoResolveTemplates.length === 0) {
+        continue;
+      }
+
+      await this.resolveOpenAlert({
+        openAlert: openAlert,
+        rootCause: input.rootCause,
+        dataToProcess: input.dataToProcess,
+      });
+
+      input.evaluationSummary?.events.push({
+        type: "alert-resolved",
+        title: `Alert resolved: ${openAlert.id?.toString()}`,
+        message:
+          "Alert auto-resolved because the incoming payload reported this key as resolved.",
+        relatedAlertId: openAlert.id?.toString(),
+        relatedAlertNumber: openAlert.alertNumber,
+        relatedAlertNumberWithPrefix: openAlert.alertNumberWithPrefix,
+        at: OneUptimeDate.getCurrentDate(),
+      });
+    }
+  }
+
   @CaptureSpan()
   public static async criteriaMetCreateAlertsAndUpdateMonitorStatus(input: {
     criteriaInstance: MonitorCriteriaInstance;
@@ -129,6 +227,16 @@ export default class MonitorAlert {
      * evaluating. See MonitorMaintenanceSuppression.
      */
     suppressedSeriesFingerprints?: Set<string> | undefined;
+    /**
+     * Event-driven monitors (incoming-request / webhook fan-out) must not
+     * use the metric snapshot model where a series absent from this tick's
+     * breaching set is auto-resolved — a single webhook only describes the
+     * keys in that payload, so absence is not recovery. When true, the
+     * per-series absence-resolve pass is skipped; those alerts are resolved
+     * explicitly elsewhere (see IncomingRequestIncidentGrouping +
+     * resolveSeriesAlertsByFingerprint). Per-key create + dedupe still run.
+     */
+    disableSeriesAbsenceResolution?: boolean | undefined;
   }): Promise<void> {
     const alertLogAttributes: LogAttributes = {
       projectId: input.monitor.projectId?.toString(),
@@ -141,7 +249,7 @@ export default class MonitorAlert {
     );
 
     const breachingSeriesFingerprints: Set<string> | undefined =
-      input.matchesPerSeries
+      input.matchesPerSeries && !input.disableSeriesAbsenceResolution
         ? new Set<string>(
             input.matchesPerSeries.map((m: PerSeriesCriteriaMatch) => {
               return m.fingerprint;
@@ -160,6 +268,7 @@ export default class MonitorAlert {
         dataToProcess: input.dataToProcess,
         evaluationSummary: input.evaluationSummary,
         breachingSeriesFingerprints,
+        disableSeriesAbsenceResolution: input.disableSeriesAbsenceResolution,
       });
 
     if (!input.criteriaInstance.data?.createAlerts) {
@@ -179,8 +288,15 @@ export default class MonitorAlert {
         monitor: input.monitor,
       });
 
+    /*
+     * `undefined` matchesPerSeries → legacy single-alert path. A defined
+     * (even empty) array → per-series mode: iterate exactly the matches.
+     * An empty array therefore creates nothing — used by grouped
+     * incoming-request criteria on a payload with no firing key so they
+     * don't fall back to a single whole-monitor alert.
+     */
     const seriesToProcess: Array<PerSeriesCriteriaMatch | undefined> =
-      input.matchesPerSeries && input.matchesPerSeries.length > 0
+      input.matchesPerSeries !== undefined
         ? input.matchesPerSeries
         : [undefined];
 
@@ -572,12 +688,33 @@ export default class MonitorAlert {
       );
     }
 
-    await AlertStateTimelineService.create({
-      data: alertStateTimeline,
-      props: {
-        isRoot: true,
-      },
-    });
+    try {
+      await AlertStateTimelineService.create({
+        data: alertStateTimeline,
+        props: {
+          isRoot: true,
+        },
+      });
+    } catch (err) {
+      /*
+       * Idempotent concurrency race: two evaluations for the same monitor
+       * (e.g. the explicit per-key resolveSeriesAlertsByFingerprint path and
+       * the checkOpenAlertsAndCloseIfResolved path) can both decide to resolve
+       * the same open alert near-simultaneously. The loser's onBeforeCreate
+       * dedupe throws this exact BadDataException. Treat as a no-op at debug
+       * level instead of failing the queue job. Mirrors resolveOpenIncident.
+       */
+      if (
+        err instanceof BadDataException &&
+        err.message === "Alert state cannot be same as previous state."
+      ) {
+        logger.debug(
+          `${input.openAlert.id?.toString()} - Alert already in resolved state; skipping duplicate state timeline (concurrent race).`,
+        );
+      } else {
+        throw err;
+      }
+    }
   }
 
   private static shouldCloseAlert(input: {
@@ -585,10 +722,23 @@ export default class MonitorAlert {
     autoResolveCriteriaInstanceIdAlertIdsDictionary: Dictionary<Array<string>>;
     criteriaInstance: MonitorCriteriaInstance | null; // null if no criteia met.
     breachingSeriesFingerprints?: Set<string> | undefined;
+    disableSeriesAbsenceResolution?: boolean | undefined;
   }): boolean {
     const openSeriesFingerprint: string | undefined =
       input.openAlert.seriesFingerprint || undefined;
 
+    /*
+     * Event-driven (incoming-request / webhook) per-key alerts must NEVER
+     * be resolved by absence — only explicitly, via
+     * resolveSeriesAlertsByFingerprint, when the payload reports the key as
+     * recovered. Mirrors MonitorIncident.shouldCloseIncident. Without this
+     * guard, a heartbeat-timeout cron tick or a rejected webhook would
+     * bulk-resolve all open per-key alerts by absence.
+     */
+    if (input.disableSeriesAbsenceResolution && openSeriesFingerprint) {
+      return false;
+    }
+
     /*
      * Per-series auto-resolve: when a breaching-series set is given and
      * this alert has a fingerprint, resolve whenever the fingerprint is

package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts

@@ -5,6 +5,7 @@ import APIRequestCriteria from "./Criteria/APIRequestCriteria";
 import CustomCodeMonitoringCriteria from "./Criteria/CustomCodeMonitorCriteria";
 import IncomingEmailCriteria from "./Criteria/IncomingEmailCriteria";
 import IncomingRequestCriteria from "./Criteria/IncomingRequestCriteria";
+import IncomingRequestIncidentGrouping from "./IncomingRequestIncidentGrouping";
 import SSLMonitorCriteria from "./Criteria/SSLMonitorCriteria";
 import ServerMonitorCriteria from "./Criteria/ServerMonitorCriteria";
 import SyntheticMonitoringCriteria from "./Criteria/SyntheticMonitor";
@@ -215,6 +216,17 @@ ${contextBlock}
 
         if (perSeriesMatches.length > 0) {
           input.probeApiIngestResponse.perSeriesMatches = perSeriesMatches;
+        } else if (
+          input.monitor.monitorType === MonitorType.IncomingRequest &&
+          IncomingRequestIncidentGrouping.isGroupingConfigured(criteriaInstance)
+        ) {
+          /*
+           * Grouped incoming-request criteria that produced no firing key
+           * (e.g. a pure "resolved" webhook). Force per-series mode with an
+           * empty set so the create path does NOT fall back to opening a
+           * single whole-monitor incident.
+           */
+          input.probeApiIngestResponse.perSeriesMatches = [];
         }
 
         break;
@@ -238,6 +250,20 @@ ${contextBlock}
     monitorStep: MonitorStep;
     criteriaInstance: MonitorCriteriaInstance;
   }): Promise<Array<PerSeriesCriteriaMatch>> {
+    /*
+     * Incoming Request / webhook monitors fan out per payload-derived key
+     * (e.g. one incident per Grafana alert name) when incidentGrouping is
+     * configured. No-op (returns []) when grouping is not configured, so
+     * existing incoming-request monitors keep their single-incident
+     * behaviour.
+     */
+    if (input.monitor.monitorType === MonitorType.IncomingRequest) {
+      return IncomingRequestIncidentGrouping.collectFiringMatches({
+        dataToProcess: input.dataToProcess,
+        criteriaInstance: input.criteriaInstance,
+      });
+    }
+
     if (
       input.monitor.monitorType !== MonitorType.Metrics &&
       input.monitor.monitorType !== MonitorType.Kubernetes &&