@oneuptime/common 12.0.32 → 12.0.34

package/Tests/Types/Monitor/Recommendation/MonitorRecommendationAlertDebuggability.test.ts

@@ -0,0 +1,363 @@
+import { JSONObject } from "../../../../Types/JSON";
+import MonitorCriteriaInstance from "../../../../Types/Monitor/MonitorCriteriaInstance";
+import MonitorStep from "../../../../Types/Monitor/MonitorStep";
+import MonitorType from "../../../../Types/Monitor/MonitorType";
+import ObjectID from "../../../../Types/ObjectID";
+import MonitorRecommendationCatalog from "../../../../Types/Monitor/Recommendation/MonitorRecommendationCatalog";
+import {
+  MonitorRecommendation,
+  MonitorRecommendationArgs,
+} from "../../../../Types/Monitor/Recommendation/MonitorRecommendationTypes";
+import SeriesDebugHints from "../../../../Types/Monitor/SeriesContext/SeriesDebugHints";
+import SeriesLabelDisplay, {
+  DisplaySeriesLabel,
+} from "../../../../Types/Monitor/SeriesContext/SeriesLabelDisplay";
+import SeriesContextEnricher from "../../../../Server/Utils/Monitor/SeriesContextEnricher";
+
+/*
+ * End-to-end guard on the thing the recommendations exist for: an alert
+ * an SRE can act on without opening anything else.
+ *
+ * Every monitor on the reporting project's alert list was created from a
+ * recommendation, and all forty-nine of its alerts read
+ *
+ *   "[K8s] Pod CPU Saturating Container Limit (>90%) - oneuptime-test -
+ *    Pod CPU Saturating Container Limit"
+ *
+ * with no pod, no container and no node anywhere in the title. Three
+ * separate things had to be true for that, and this file asserts all
+ * three across EVERY shipped template rather than for one of them:
+ *
+ *   1. The template groups by enough to identify the thing that broke.
+ *   2. Every key it groups by is one the display layer can name, so the
+ *      alert says "Pod", not "resource.k8s.pod.name".
+ *   3. Rendering the template's own title through the alert-creation path
+ *      actually produces a title carrying that identity.
+ *
+ * It runs against the CATALOG rather than the individual template
+ * modules, so a new resource type is covered the day it is registered.
+ */
+
+function buildArgs(): MonitorRecommendationArgs {
+  return {
+    resourceIdentifier: "test-resource",
+    onlineMonitorStatusId: ObjectID.generate(),
+    offlineMonitorStatusId: ObjectID.generate(),
+    defaultIncidentSeverityId: ObjectID.generate(),
+    defaultAlertSeverityId: ObjectID.generate(),
+    monitorName: "Test Monitor",
+  };
+}
+
+interface TemplateCase {
+  recommendationId: string;
+  monitorType: MonitorType;
+  step: MonitorStep;
+  groupByKeys: Array<string>;
+}
+
+const ALL_TEMPLATE_CASES: Array<TemplateCase> =
+  MonitorRecommendationCatalog.getAllRecommendations().map(
+    (recommendation: MonitorRecommendation): TemplateCase => {
+      const step: MonitorStep = recommendation.getMonitorStep(buildArgs());
+
+      return {
+        recommendationId: recommendation.recommendationId,
+        monitorType: recommendation.monitorType,
+        step,
+        groupByKeys: MonitorStep.getGroupByAttributeKeys(step),
+      };
+    },
+  );
+
+const GROUPED_TEMPLATE_CASES: Array<TemplateCase> = ALL_TEMPLATE_CASES.filter(
+  (testCase: TemplateCase) => {
+    return testCase.groupByKeys.length > 0;
+  },
+);
+
+/*
+ * Stand-in values for a series. Real label values are opaque strings, so
+ * a per-key synthetic value is enough to prove the identity reaches the
+ * title - and using the key itself as the seed makes a failure message
+ * say exactly which label went missing.
+ */
+function buildSeriesLabels(groupByKeys: Array<string>): JSONObject {
+  const labels: JSONObject = {};
+
+  for (const key of groupByKeys) {
+    labels[key] = `value-for-${SeriesLabelDisplay.normalizeKey(key)}`;
+  }
+
+  return labels;
+}
+
+function getCriteriaInstances(
+  step: MonitorStep,
+): Array<MonitorCriteriaInstance> {
+  return step.data?.monitorCriteria?.data?.monitorCriteriaInstanceArray || [];
+}
+
+describe("Recommendation templates produce debuggable alerts", () => {
+  test("the catalog actually has templates to check", () => {
+    // A silently-empty catalog would make every test.each below vacuous.
+    expect(ALL_TEMPLATE_CASES.length).toBeGreaterThan(50);
+    expect(GROUPED_TEMPLATE_CASES.length).toBeGreaterThan(30);
+  });
+
+  /*
+   * Every Service metric template thresholds a property of ONE process — a
+   * heap, an event loop, a thread pool. Ungrouped, the ClickHouse aggregation
+   * collapses every replica into a single number: an Avg template on three
+   * JVMs at 95/20/20 reads 45 and stays green while one OOM-kills, and a Max
+   * template fires without being able to name the replica that broke. That
+   * nameless alert is the exact failure this file exists for.
+   *
+   * The two request-latency percentiles are the only legitimate exemptions:
+   * a p95 is the SLO number for the service as a whole, and splitting it per
+   * instance makes it a different alert.
+   */
+  test("service metric templates identify the replica, not just the service", () => {
+    const serviceMetricCases: Array<TemplateCase> = ALL_TEMPLATE_CASES.filter(
+      (testCase: TemplateCase) => {
+        return (
+          testCase.recommendationId.startsWith("Service:") &&
+          testCase.monitorType === MonitorType.Metrics
+        );
+      },
+    );
+
+    expect(serviceMetricCases.length).toBeGreaterThan(20);
+
+    expect(
+      serviceMetricCases
+        .filter((testCase: TemplateCase) => {
+          return testCase.groupByKeys.length === 0;
+        })
+        .map((testCase: TemplateCase) => {
+          return testCase.recommendationId;
+        })
+        .sort(),
+    ).toEqual(["Service:service-latency-p95", "Service:service-latency-p99"]);
+
+    for (const testCase of serviceMetricCases) {
+      if (testCase.groupByKeys.length === 0) {
+        continue;
+      }
+
+      expect(testCase.groupByKeys).toEqual(["resource.service.instance.id"]);
+    }
+  });
+
+  describe("every group-by key is a key the alert can name", () => {
+    test.each(GROUPED_TEMPLATE_CASES)(
+      "$recommendationId",
+      (testCase: TemplateCase) => {
+        for (const key of testCase.groupByKeys) {
+          /*
+           * `getFriendlyLabelName` never fails - it falls back to a
+           * prettified form of the raw key. That fallback is the right
+           * behaviour for a user's own custom attribute, but for a
+           * SHIPPED template it means nobody registered the key, and the
+           * alert reads "K8s Pod Name: web-1" instead of "Pod: web-1".
+           */
+          expect(SeriesLabelDisplay.isKnownLabelKey(key)).toBe(true);
+
+          // ...and the registered name is actually usable in a sentence.
+          expect(
+            SeriesLabelDisplay.getFriendlyLabelName(key).length,
+          ).toBeGreaterThan(0);
+        }
+      },
+    );
+  });
+
+  describe("every grouped template's alert title names the resource", () => {
+    test.each(GROUPED_TEMPLATE_CASES)(
+      "$recommendationId",
+      (testCase: TemplateCase) => {
+        const seriesLabels: JSONObject = buildSeriesLabels(
+          testCase.groupByKeys,
+        );
+
+        const criteriaInstances: Array<MonitorCriteriaInstance> =
+          getCriteriaInstances(testCase.step);
+
+        const alertTitles: Array<string> = criteriaInstances.flatMap(
+          (instance: MonitorCriteriaInstance) => {
+            return (instance.data?.alerts || []).map(
+              (alert: { title: string }) => {
+                return alert.title;
+              },
+            );
+          },
+        );
+
+        expect(alertTitles.length).toBeGreaterThan(0);
+
+        for (const title of alertTitles) {
+          const enriched: string = SeriesContextEnricher.enrichTitle({
+            title,
+            seriesLabels,
+          });
+
+          /*
+           * The single most-identifying label must be in the title. The
+           * rest are in the description block and the Affected Resource
+           * table; the title only has room for the top few.
+           */
+          const topLabel: DisplaySeriesLabel =
+            SeriesLabelDisplay.getDisplayLabels(seriesLabels)[0]!;
+
+          expect(enriched).toContain(topLabel.value);
+          expect(enriched).not.toContain("{{");
+        }
+      },
+    );
+  });
+
+  describe("every grouped template's alert description carries the full identity", () => {
+    test.each(GROUPED_TEMPLATE_CASES)(
+      "$recommendationId",
+      (testCase: TemplateCase) => {
+        const seriesLabels: JSONObject = buildSeriesLabels(
+          testCase.groupByKeys,
+        );
+
+        const descriptions: Array<string> = getCriteriaInstances(
+          testCase.step,
+        ).flatMap((instance: MonitorCriteriaInstance) => {
+          return (instance.data?.alerts || []).map(
+            (alert: { description: string }) => {
+              return alert.description;
+            },
+          );
+        });
+
+        for (const description of descriptions) {
+          const enriched: string = SeriesContextEnricher.enrichDescription({
+            description,
+            seriesLabels,
+            monitorType: testCase.monitorType,
+          });
+
+          // EVERY label, not just the ones that fit in a title.
+          for (const label of SeriesLabelDisplay.getDisplayLabels(
+            seriesLabels,
+          )) {
+            expect(enriched).toContain(label.value);
+          }
+        }
+      },
+    );
+  });
+
+  describe("incident titles get the same treatment as alert titles", () => {
+    test.each(GROUPED_TEMPLATE_CASES)(
+      "$recommendationId",
+      (testCase: TemplateCase) => {
+        const seriesLabels: JSONObject = buildSeriesLabels(
+          testCase.groupByKeys,
+        );
+
+        const incidentTitles: Array<string> = getCriteriaInstances(
+          testCase.step,
+        ).flatMap((instance: MonitorCriteriaInstance) => {
+          return (instance.data?.incidents || []).map(
+            (incident: { title: string }) => {
+              return incident.title;
+            },
+          );
+        });
+
+        const topLabel: DisplaySeriesLabel =
+          SeriesLabelDisplay.getDisplayLabels(seriesLabels)[0]!;
+
+        for (const title of incidentTitles) {
+          expect(
+            SeriesContextEnricher.enrichTitle({ title, seriesLabels }),
+          ).toContain(topLabel.value);
+        }
+      },
+    );
+  });
+
+  describe("infrastructure templates hand the engineer a first command", () => {
+    /*
+     * The monitor types where a read-only inspection command is both
+     * universally correct and genuinely the next thing to run. Metrics /
+     * IoT / RUM / Service series identify an application-level entity
+     * with no such command, and are deliberately excluded rather than
+     * given a guess.
+     */
+    const TYPES_WITH_COMMANDS: Array<MonitorType> = [
+      MonitorType.Kubernetes,
+      MonitorType.Docker,
+      MonitorType.Podman,
+      MonitorType.DockerSwarm,
+      MonitorType.Host,
+      MonitorType.Proxmox,
+      MonitorType.Ceph,
+    ];
+
+    const CASES: Array<TemplateCase> = GROUPED_TEMPLATE_CASES.filter(
+      (testCase: TemplateCase) => {
+        return TYPES_WITH_COMMANDS.includes(testCase.monitorType);
+      },
+    );
+
+    test("there are infrastructure templates to check", () => {
+      expect(CASES.length).toBeGreaterThan(20);
+    });
+
+    test.each(CASES)("$recommendationId", (testCase: TemplateCase) => {
+      const commands: Array<string> = SeriesDebugHints.getDebugCommands({
+        monitorType: testCase.monitorType,
+        seriesLabels: buildSeriesLabels(testCase.groupByKeys),
+      }).map((command: { command: string }) => {
+        return command.command;
+      });
+
+      expect(commands.length).toBeGreaterThan(0);
+
+      for (const command of commands) {
+        // No half-filled interpolation reaches a copy-paste button.
+        expect(command).not.toMatch(/\s\s/);
+        expect(command.trim()).toBe(command);
+      }
+    });
+  });
+
+  describe("ungrouped templates stay untouched", () => {
+    const UNGROUPED: Array<TemplateCase> = ALL_TEMPLATE_CASES.filter(
+      (testCase: TemplateCase) => {
+        return testCase.groupByKeys.length === 0;
+      },
+    );
+
+    test("some templates are legitimately whole-resource", () => {
+      /*
+       * etcd leadership, API server throttling, scheduler backlog: one
+       * value for the cluster. If this ever hits zero, something has
+       * started inventing series that do not exist.
+       */
+      expect(UNGROUPED.length).toBeGreaterThan(0);
+    });
+
+    test.each(UNGROUPED)(
+      "$recommendationId keeps its title verbatim",
+      (testCase: TemplateCase) => {
+        for (const instance of getCriteriaInstances(testCase.step)) {
+          for (const alert of instance.data?.alerts || []) {
+            expect(
+              SeriesContextEnricher.enrichTitle({
+                title: alert.title,
+                seriesLabels: {},
+              }),
+            ).toBe(alert.title);
+          }
+        }
+      },
+    );
+  });
+});

package/Tests/Types/Monitor/Recommendation/MonitorRecommendationNotificationMode.test.ts

@@ -92,8 +92,9 @@ function getRecoveryCriteriaInstances(
 
 /*
  * Every criteria instance that ships something to create — the "Unhealthy"
- * ones. Ceph's Monitor Disk template ships two of them (a CRIT tier and a LOW
- * tier), so this is deliberately a list and not a single instance.
+ * ones. Deliberately a list and not a single instance: nothing in
+ * `applyNotificationSettingsToMonitorStep` limits a step to one, and a monitor
+ * a user has edited by hand can carry several.
  */
 function getBreachCriteriaInstances(
   monitorStep: MonitorStep,
@@ -116,6 +117,50 @@ function applyMode(data: {
   });
 }
 
+/*
+ * A step carrying TWO unhealthy criteria, assembled by hand.
+ *
+ * No shipped template has two any more — Ceph's monitor-disk template used to
+ * tier MON_DISK_CRIT and MON_DISK_LOW into one monitor, and that was the bug
+ * the split fixed. The guarantee that the mode is applied to EVERY unhealthy
+ * criteria is still real: `applyNotificationModeToCriteriaInstance` runs per
+ * instance, and a monitor a user has since edited can hold any number. So it
+ * is exercised against a synthetic step rather than dropped, and an
+ * implementation that only touched the first instance still fails.
+ */
+function buildTwoUnhealthyCriteriaStep(
+  recommendation: MonitorRecommendation,
+): MonitorStep {
+  const monitorStep: MonitorStep = recommendation.getMonitorStep(buildArgs());
+
+  const breachInstance: MonitorCriteriaInstance =
+    getBreachCriteriaInstances(monitorStep)[0]!;
+
+  const secondTier: MonitorCriteriaInstance = new MonitorCriteriaInstance();
+
+  secondTier.data = {
+    ...breachInstance.data!,
+    id: ObjectID.generate().toString(),
+    name: "Second Unhealthy Tier",
+    incidents: (breachInstance.data!.incidents || []).map(
+      (incident: CriteriaIncident) => {
+        return { ...incident, id: ObjectID.generate().toString() };
+      },
+    ),
+    alerts: (breachInstance.data!.alerts || []).map((alert: CriteriaAlert) => {
+      return { ...alert, id: ObjectID.generate().toString() };
+    }),
+  };
+
+  monitorStep.data!.monitorCriteria!.data!.monitorCriteriaInstanceArray = [
+    breachInstance,
+    secondTier,
+    ...getRecoveryCriteriaInstances(monitorStep),
+  ];
+
+  return monitorStep;
+}
+
 const ALL_RECOMMENDATIONS: Array<MonitorRecommendation> =
   MonitorRecommendationCatalog.getAllRecommendations();
 
@@ -191,24 +236,27 @@ describe("MonitorRecommendationNotificationMode", () => {
       }
     });
 
-    it("ships at least one recommendation with more than one unhealthy criteria", () => {
+    it("ships exactly one unhealthy criteria per recommendation", () => {
       /*
-       * Ceph's monitor-disk template tiers CRIT and LOW into two unhealthy
-       * criteria on one monitor. Without a multi-criteria template in the
-       * catalog, "the mode is applied to EVERY unhealthy criteria" below could
-       * never fail — a buggy implementation that only touched the first
-       * criteria instance would pass every other test in this file.
+       * `applyNotificationSettingsToMonitorStep` stamps ONE severity — the
+       * recommendation's own — onto every criteria instance in the step, so a
+       * second, milder tier inside a single template opens incidents at the
+       * template's severity and pages that severity's on-call policy. Ceph's
+       * monitor-disk template did exactly that: its MON_DISK_LOW (Warning)
+       * tier opened Critical incidents while the card promised Warning. Two
+       * severities need two templates (`ceph-mon-disk-space` and
+       * `ceph-mon-disk-low`).
+       *
+       * A future template that genuinely wants two tiers at the SAME severity
+       * may relax this — but it has to say so here, deliberately, rather than
+       * re-introduce the mismatch by accident.
        */
-      const multiCriteriaRecommendations: Array<MonitorRecommendation> =
-        ALL_RECOMMENDATIONS.filter((recommendation: MonitorRecommendation) => {
-          return (
-            getBreachCriteriaInstances(
-              recommendation.getMonitorStep(buildArgs()),
-            ).length > 1
-          );
-        });
-
-      expect(multiCriteriaRecommendations.length).toBeGreaterThan(0);
+      for (const recommendation of ALL_RECOMMENDATIONS) {
+        expect(
+          getBreachCriteriaInstances(recommendation.getMonitorStep(buildArgs()))
+            .length,
+        ).toBe(1);
+      }
     });
   });
 
@@ -261,6 +309,33 @@ describe("MonitorRecommendationNotificationMode", () => {
       }
     });
 
+    it("applies the mode to EVERY unhealthy criteria, not just the first", () => {
+      for (const notificationMode of ALL_MODES) {
+        const recommendation: MonitorRecommendation = ALL_RECOMMENDATIONS[0]!;
+        const monitorStep: MonitorStep =
+          buildTwoUnhealthyCriteriaStep(recommendation);
+
+        expect(getBreachCriteriaInstances(monitorStep).length).toBe(2);
+
+        MonitorRecommendationUtil.applyNotificationSettingsToMonitorStep({
+          monitorStep: monitorStep,
+          notificationSettings: { notificationMode: notificationMode },
+          severity: recommendation.severity,
+        });
+
+        for (const criteriaInstance of getBreachCriteriaInstances(
+          monitorStep,
+        )) {
+          expect(criteriaInstance.data?.createIncidents).toBe(
+            notificationMode !== MonitorRecommendationNotificationMode.Alert,
+          );
+          expect(criteriaInstance.data?.createAlerts).toBe(
+            notificationMode !== MonitorRecommendationNotificationMode.Incident,
+          );
+        }
+      }
+    });
+
     it("an absent mode leaves the flags exactly as the template shipped them", () => {
       /*
        * The backwards-compatible path. `notificationMode` is optional, and any