@oneuptime/common 11.3.4 → 11.3.6

package/Tests/Server/Utils/Monitor/IncomingRequestIncidentGrouping.test.ts

@@ -0,0 +1,389 @@
+import IncomingRequestIncidentGrouping from "../../../../Server/Utils/Monitor/IncomingRequestIncidentGrouping";
+import DataToProcess from "../../../../Server/Utils/Monitor/DataToProcess";
+import IncidentGroupingConfig from "../../../../Types/Monitor/IncomingMonitor/IncidentGroupingConfig";
+import IncomingMonitorRequest from "../../../../Types/Monitor/IncomingMonitor/IncomingMonitorRequest";
+import MonitorCriteriaInstance from "../../../../Types/Monitor/MonitorCriteriaInstance";
+import { JSONObject } from "../../../../Types/JSON";
+import ObjectID from "../../../../Types/ObjectID";
+import { PerSeriesCriteriaMatch } from "../../../../Types/Probe/ProbeApiIngestResponse";
+
+function makeRequest(
+  requestBody: JSONObject | string | undefined,
+  overrides?: Partial<IncomingMonitorRequest>,
+): DataToProcess {
+  const request: IncomingMonitorRequest = {
+    projectId: ObjectID.generate(),
+    monitorId: ObjectID.generate(),
+    requestBody: requestBody,
+    incomingRequestReceivedAt: new Date(0),
+    checkedAt: new Date(0),
+    ...overrides,
+  };
+
+  return request as unknown as DataToProcess;
+}
+
+function makeCriteria(
+  grouping: IncidentGroupingConfig | undefined,
+  id: string = "criteria-1",
+): MonitorCriteriaInstance {
+  const criteria: MonitorCriteriaInstance = new MonitorCriteriaInstance();
+  criteria.data!.id = id;
+  criteria.data!.incidentGrouping = grouping;
+  return criteria;
+}
+
+// Grafana / Alertmanager-shaped payloads.
+function firingAlertmanagerPayload(): JSONObject {
+  return {
+    status: "firing",
+    alerts: [
+      { status: "firing", labels: { alertname: "High RAM Usage" } },
+      { status: "firing", labels: { alertname: "High CPU Usage" } },
+      { status: "firing", labels: { alertname: "High Disk Usage" } },
+    ],
+  };
+}
+
+describe("IncomingRequestIncidentGrouping", () => {
+  describe("collectFiringMatches", () => {
+    it("returns one firing match per array element (Grafana alerts[*])", () => {
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest(firingAlertmanagerPayload()),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "requestBody.alerts[*].labels.alertname",
+          }),
+        });
+
+      expect(matches).toHaveLength(3);
+      expect(
+        matches.map((m: PerSeriesCriteriaMatch) => {
+          return m.labels;
+        }),
+      ).toEqual([
+        { alertname: "High RAM Usage" },
+        { alertname: "High CPU Usage" },
+        { alertname: "High Disk Usage" },
+      ]);
+      // Distinct keys produce distinct fingerprints (=> concurrent incidents).
+      const fingerprints: Set<string> = new Set<string>(
+        matches.map((m: PerSeriesCriteriaMatch) => {
+          return m.fingerprint;
+        }),
+      );
+      expect(fingerprints.size).toBe(3);
+      expect(
+        matches.every((m: PerSeriesCriteriaMatch) => {
+          return m.criteriaMetId === "criteria-1";
+        }),
+      ).toBe(true);
+    });
+
+    it("supports a simple (non-array) group-by path", () => {
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest({
+            commonLabels: { alertname: "Disk Full" },
+          }),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "requestBody.commonLabels.alertname",
+          }),
+        });
+
+      expect(matches).toHaveLength(1);
+      expect(matches[0]!.labels).toEqual({ alertname: "Disk Full" });
+    });
+
+    it("returns [] when grouping is not configured (legacy behaviour)", () => {
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest(firingAlertmanagerPayload()),
+          criteriaInstance: makeCriteria(undefined),
+        });
+
+      expect(matches).toEqual([]);
+    });
+
+    it("returns [] on the heartbeat-timeout re-evaluation (no fresh body)", () => {
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest(firingAlertmanagerPayload(), {
+            onlyCheckForIncomingRequestReceivedAt: true,
+          }),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "requestBody.alerts[*].labels.alertname",
+          }),
+        });
+
+      expect(matches).toEqual([]);
+    });
+
+    it("parses a raw JSON string body", () => {
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest(
+            JSON.stringify({ commonLabels: { alertname: "Stringified" } }),
+          ),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "requestBody.commonLabels.alertname",
+          }),
+        });
+
+      expect(matches).toHaveLength(1);
+      expect(matches[0]!.labels).toEqual({ alertname: "Stringified" });
+    });
+
+    it("excludes elements classified as resolved (per-element status)", () => {
+      const payload: JSONObject = {
+        status: "firing",
+        alerts: [
+          { status: "firing", labels: { alertname: "Still Firing" } },
+          { status: "resolved", labels: { alertname: "Recovered" } },
+        ],
+      };
+
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest(payload),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "requestBody.alerts[*].labels.alertname",
+            resolvedWhenJSONPath: "requestBody.alerts[*].status",
+            resolvedWhenValue: "resolved",
+          }),
+        });
+
+      expect(matches).toHaveLength(1);
+      expect(matches[0]!.labels).toEqual({ alertname: "Still Firing" });
+    });
+
+    it("excludes everything when the whole payload is resolved (payload-level status)", () => {
+      const payload: JSONObject = {
+        status: "resolved",
+        commonLabels: { alertname: "High CPU Usage" },
+      };
+
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest(payload),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "requestBody.commonLabels.alertname",
+            resolvedWhenJSONPath: "requestBody.status",
+            resolvedWhenValue: "resolved",
+          }),
+        });
+
+      expect(matches).toEqual([]);
+    });
+
+    it("dedupes the same key appearing twice in one payload", () => {
+      const payload: JSONObject = {
+        alerts: [
+          { labels: { alertname: "Dup" } },
+          { labels: { alertname: "Dup" } },
+        ],
+      };
+
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest(payload),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "requestBody.alerts[*].labels.alertname",
+          }),
+        });
+
+      expect(matches).toHaveLength(1);
+    });
+
+    it("caps the number of keys at maxKeysPerPayload", () => {
+      const payload: JSONObject = {
+        alerts: Array.from({ length: 10 }, (_unused: unknown, i: number) => {
+          return { labels: { alertname: `alert-${i}` } };
+        }),
+      };
+
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest(payload),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "requestBody.alerts[*].labels.alertname",
+            maxKeysPerPayload: 3,
+          }),
+        });
+
+      expect(matches).toHaveLength(3);
+    });
+  });
+
+  describe("collectResolvedFingerprints", () => {
+    it("returns the fingerprints the payload marks resolved", () => {
+      const payload: JSONObject = {
+        alerts: [
+          { status: "firing", labels: { alertname: "Still Firing" } },
+          { status: "resolved", labels: { alertname: "Recovered" } },
+        ],
+      };
+
+      const grouping: IncidentGroupingConfig = {
+        groupByJSONPath: "requestBody.alerts[*].labels.alertname",
+        resolvedWhenJSONPath: "requestBody.alerts[*].status",
+        resolvedWhenValue: "resolved",
+      };
+
+      const resolved: Array<string> =
+        IncomingRequestIncidentGrouping.collectResolvedFingerprints({
+          dataToProcess: makeRequest(payload),
+          criteriaInstances: [makeCriteria(grouping)],
+        });
+
+      expect(resolved).toHaveLength(1);
+
+      /*
+       * The resolved fingerprint must equal the firing fingerprint for the
+       * same key — otherwise resolution would target the wrong incident.
+       */
+      const firingForRecovered: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest({
+            alerts: [{ status: "firing", labels: { alertname: "Recovered" } }],
+          }),
+          criteriaInstance: makeCriteria(grouping),
+        });
+
+      expect(resolved[0]).toBe(firingForRecovered[0]!.fingerprint);
+    });
+
+    it("returns [] when no resolve classifier is configured", () => {
+      const resolved: Array<string> =
+        IncomingRequestIncidentGrouping.collectResolvedFingerprints({
+          dataToProcess: makeRequest({ status: "resolved" }),
+          criteriaInstances: [
+            makeCriteria({
+              groupByJSONPath: "requestBody.commonLabels.alertname",
+            }),
+          ],
+        });
+
+      expect(resolved).toEqual([]);
+    });
+  });
+
+  describe("isGroupingConfigured", () => {
+    it("is true only when a group-by path is set", () => {
+      expect(
+        IncomingRequestIncidentGrouping.isGroupingConfigured(
+          makeCriteria({
+            groupByJSONPath: "requestBody.commonLabels.alertname",
+          }),
+        ),
+      ).toBe(true);
+      expect(
+        IncomingRequestIncidentGrouping.isGroupingConfigured(
+          makeCriteria(undefined),
+        ),
+      ).toBe(false);
+    });
+  });
+
+  describe("array-of-scalars fan-out (`prefix[*]`)", () => {
+    it("fans out one match per scalar array element", () => {
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest({ tags: ["RAM", "CPU", "Disk"] }),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "requestBody.tags[*]",
+          }),
+        });
+
+      expect(matches).toHaveLength(3);
+      expect(
+        matches.map((m: PerSeriesCriteriaMatch) => {
+          return m.labels;
+        }),
+      ).toEqual([{ tags: "RAM" }, { tags: "CPU" }, { tags: "Disk" }]);
+    });
+  });
+
+  describe("requestBody-rooted convention (matches incident templates)", () => {
+    it("accepts the `{{requestBody. … }}` template wrapper", () => {
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest({
+            commonLabels: { alertname: "Disk Full" },
+          }),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "{{requestBody.commonLabels.alertname}}",
+          }),
+        });
+
+      expect(matches).toHaveLength(1);
+      expect(matches[0]!.labels).toEqual({ alertname: "Disk Full" });
+    });
+
+    it("ignores a bare path that is not rooted at requestBody (not supported)", () => {
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest(firingAlertmanagerPayload()),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "alerts[*].labels.alertname",
+          }),
+        });
+
+      expect(matches).toEqual([]);
+    });
+
+    it("the plain and `{{ }}`-wrapped forms produce identical fingerprints", () => {
+      const plain: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest(firingAlertmanagerPayload()),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "requestBody.alerts[*].labels.alertname",
+          }),
+        });
+
+      const wrapped: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest(firingAlertmanagerPayload()),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "{{requestBody.alerts[*].labels.alertname}}",
+          }),
+        });
+
+      expect(
+        wrapped.map((m: PerSeriesCriteriaMatch) => {
+          return m.fingerprint;
+        }),
+      ).toEqual(
+        plain.map((m: PerSeriesCriteriaMatch) => {
+          return m.fingerprint;
+        }),
+      );
+    });
+  });
+
+  describe("mixed-mode resolve config (resolve has [*], group-by does not)", () => {
+    it("does not silently mis-scope a [*] resolve path onto a top-level field", () => {
+      /*
+       * Self-contradictory config: group-by is non-array but resolve uses [*].
+       * The top-level `status: resolved` must NOT be treated as a per-element
+       * resolve — otherwise this single key would be wrongly classified.
+       */
+      const matches: Array<PerSeriesCriteriaMatch> =
+        IncomingRequestIncidentGrouping.collectFiringMatches({
+          dataToProcess: makeRequest({
+            status: "resolved",
+            commonLabels: { alertname: "High CPU Usage" },
+          }),
+          criteriaInstance: makeCriteria({
+            groupByJSONPath: "requestBody.commonLabels.alertname",
+            resolvedWhenJSONPath: "requestBody.alerts[*].status",
+            resolvedWhenValue: "resolved",
+          }),
+        });
+
+      // Treated as firing (not mis-scoped to top-level status) ⇒ stays open.
+      expect(matches).toHaveLength(1);
+      expect(matches[0]!.labels).toEqual({ alertname: "High CPU Usage" });
+    });
+  });
+});

package/Tests/Server/Utils/Monitor/SeriesAbsenceResolutionGuard.test.ts

@@ -0,0 +1,177 @@
+import MonitorIncident from "../../../../Server/Utils/Monitor/MonitorIncident";
+import MonitorAlert from "../../../../Server/Utils/Monitor/MonitorAlert";
+import Incident from "../../../../Models/DatabaseModels/Incident";
+import Alert from "../../../../Models/DatabaseModels/Alert";
+import MonitorCriteriaInstance from "../../../../Types/Monitor/MonitorCriteriaInstance";
+import Dictionary from "../../../../Types/Dictionary";
+
+/*
+ * Regression tests for the event-driven (incoming-request / webhook)
+ * absence-resolution guard. A per-key incident/alert (one carrying a
+ * seriesFingerprint, created by a grouped incoming-request criteria) must
+ * NEVER be auto-resolved by absence — only explicitly via
+ * resolveSeries{Incidents,Alerts}ByFingerprint when the payload reports the
+ * key as recovered. shouldClose{Incident,Alert} is private; we exercise it
+ * via the documented `as any` escape hatch since it is the single decision
+ * point both the criteria-met and no-criteria-met resolve paths funnel
+ * through.
+ */
+
+function incident(fingerprint: string | undefined): Incident {
+  const i: Incident = new Incident();
+  i.createdCriteriaId = "criteria-A";
+  i.createdIncidentTemplateId = "template-1";
+  if (fingerprint) {
+    i.seriesFingerprint = fingerprint;
+  }
+  return i;
+}
+
+function alert(fingerprint: string | undefined): Alert {
+  const a: Alert = new Alert();
+  a.createdCriteriaId = "criteria-A";
+  if (fingerprint) {
+    a.seriesFingerprint = fingerprint;
+  }
+  return a;
+}
+
+function criteria(id: string): MonitorCriteriaInstance {
+  const c: MonitorCriteriaInstance = new MonitorCriteriaInstance();
+  c.data!.id = id;
+  return c;
+}
+
+const INCIDENT_AUTO_RESOLVE: Dictionary<Array<string>> = {
+  "criteria-A": ["template-1"],
+};
+const ALERT_AUTO_RESOLVE: Dictionary<Array<string>> = {
+  "criteria-A": ["template-1"],
+};
+
+function shouldCloseIncident(input: Record<string, unknown>): boolean {
+  return (MonitorIncident as any).shouldCloseIncident(input);
+}
+function shouldCloseAlert(input: Record<string, unknown>): boolean {
+  return (MonitorAlert as any).shouldCloseAlert(input);
+}
+
+describe("Series absence-resolution guard (incoming-request grouping)", () => {
+  describe("MonitorIncident.shouldCloseIncident", () => {
+    it("does NOT absence-resolve a per-key incident on the no-criteria-met path when the guard is on (HIGH)", () => {
+      expect(
+        shouldCloseIncident({
+          openIncident: incident("fp-RAM"),
+          autoResolveCriteriaInstanceIdIncidentIdsDictionary:
+            INCIDENT_AUTO_RESOLVE,
+          criteriaInstance: null, // no criteria met (e.g. heartbeat cron tick)
+          breachingSeriesFingerprints: undefined,
+          disableSeriesAbsenceResolution: true,
+        }),
+      ).toBe(false);
+    });
+
+    it("WOULD absence-resolve it without the guard — proving the guard is load-bearing", () => {
+      expect(
+        shouldCloseIncident({
+          openIncident: incident("fp-RAM"),
+          autoResolveCriteriaInstanceIdIncidentIdsDictionary:
+            INCIDENT_AUTO_RESOLVE,
+          criteriaInstance: null,
+          breachingSeriesFingerprints: undefined,
+          disableSeriesAbsenceResolution: false,
+        }),
+      ).toBe(true);
+    });
+
+    it("does NOT cross-criteria absence-resolve a per-key incident when the guard is on (MEDIUM)", () => {
+      expect(
+        shouldCloseIncident({
+          openIncident: incident("fp-RAM"), // created by criteria-A
+          autoResolveCriteriaInstanceIdIncidentIdsDictionary:
+            INCIDENT_AUTO_RESOLVE,
+          criteriaInstance: criteria("criteria-B"), // a DIFFERENT criteria fired
+          breachingSeriesFingerprints: undefined,
+          disableSeriesAbsenceResolution: true,
+        }),
+      ).toBe(false);
+    });
+
+    it("still resolves a NON-grouped incoming incident (no seriesFingerprint) — guard is inert", () => {
+      expect(
+        shouldCloseIncident({
+          openIncident: incident(undefined),
+          autoResolveCriteriaInstanceIdIncidentIdsDictionary:
+            INCIDENT_AUTO_RESOLVE,
+          criteriaInstance: null,
+          breachingSeriesFingerprints: undefined,
+          disableSeriesAbsenceResolution: true,
+        }),
+      ).toBe(true);
+    });
+
+    it("does not change metric per-series behaviour (guard off, snapshot absence-resolve still works)", () => {
+      // Fingerprint not in the still-breaching set + auto-resolve configured ⇒ resolve.
+      expect(
+        shouldCloseIncident({
+          openIncident: incident("fp-host-2"),
+          autoResolveCriteriaInstanceIdIncidentIdsDictionary:
+            INCIDENT_AUTO_RESOLVE,
+          criteriaInstance: criteria("criteria-A"),
+          breachingSeriesFingerprints: new Set<string>(["fp-host-1"]),
+          disableSeriesAbsenceResolution: false,
+        }),
+      ).toBe(true);
+    });
+  });
+
+  describe("MonitorAlert.shouldCloseAlert", () => {
+    it("does NOT absence-resolve a per-key alert on the no-criteria-met path when the guard is on", () => {
+      expect(
+        shouldCloseAlert({
+          openAlert: alert("fp-RAM"),
+          autoResolveCriteriaInstanceIdAlertIdsDictionary: ALERT_AUTO_RESOLVE,
+          criteriaInstance: null,
+          breachingSeriesFingerprints: undefined,
+          disableSeriesAbsenceResolution: true,
+        }),
+      ).toBe(false);
+    });
+
+    it("WOULD absence-resolve it without the guard", () => {
+      expect(
+        shouldCloseAlert({
+          openAlert: alert("fp-RAM"),
+          autoResolveCriteriaInstanceIdAlertIdsDictionary: ALERT_AUTO_RESOLVE,
+          criteriaInstance: null,
+          breachingSeriesFingerprints: undefined,
+          disableSeriesAbsenceResolution: false,
+        }),
+      ).toBe(true);
+    });
+
+    it("does NOT cross-criteria absence-resolve a per-key alert when the guard is on", () => {
+      expect(
+        shouldCloseAlert({
+          openAlert: alert("fp-RAM"),
+          autoResolveCriteriaInstanceIdAlertIdsDictionary: ALERT_AUTO_RESOLVE,
+          criteriaInstance: criteria("criteria-B"),
+          breachingSeriesFingerprints: undefined,
+          disableSeriesAbsenceResolution: true,
+        }),
+      ).toBe(false);
+    });
+
+    it("still resolves a NON-grouped incoming alert (no seriesFingerprint) — guard is inert", () => {
+      expect(
+        shouldCloseAlert({
+          openAlert: alert(undefined),
+          autoResolveCriteriaInstanceIdAlertIdsDictionary: ALERT_AUTO_RESOLVE,
+          criteriaInstance: null,
+          breachingSeriesFingerprints: undefined,
+          disableSeriesAbsenceResolution: true,
+        }),
+      ).toBe(true);
+    });
+  });
+});