@oneuptime/common 12.0.33 → 13.0.0

package/Tests/Types/Monitor/ServiceAlertTemplates.test.ts

@@ -18,8 +18,10 @@ import MetricFormulaConfigData from "../../../Types/Metrics/MetricFormulaConfigD
 import {
   CheckOn,
   CriteriaFilter,
+  EvaluateOverTimeType,
   FilterType,
 } from "../../../Types/Monitor/CriteriaFilter";
+import SeriesLabelDisplay from "../../../Types/Monitor/SeriesContext/SeriesLabelDisplay";
 import { ServiceLanguage } from "../../../Types/Service/ServiceLanguage";
 import ObjectID from "../../../Types/ObjectID";
 
@@ -593,6 +595,215 @@ describe("ServiceAlertTemplates", () => {
         expect(new Set<string>(aliases).size).toBe(aliases.length);
       }
     });
+
+    /*
+     * `MetricMonitorCriteria.evaluateAllSeries` writes its `AnyValue` default
+     * back THROUGH this object when `metricAggregationType` is unset, and the
+     * criteria edit form spreads and re-assigns it. Sharing one object between
+     * the two filters makes either of those flip the healthy criteria as a
+     * side effect of touching the unhealthy one — which is how you get both
+     * criteria reporting Met on different value sets in one evaluation.
+     */
+    it("gives each criteria filter its own metricMonitorOptions object", () => {
+      for (const template of ALL_TEMPLATES) {
+        const filters: Array<CriteriaFilter> = getFilters(buildStep(template));
+
+        expect(filters.length).toBe(2);
+
+        const unhealthy: CriteriaFilter = filters[0]!;
+        const healthy: CriteriaFilter = filters[1]!;
+
+        if (!unhealthy.metricMonitorOptions) {
+          expect(healthy.metricMonitorOptions).toBeUndefined();
+          continue;
+        }
+
+        // Same content...
+        expect(healthy.metricMonitorOptions).toEqual(
+          unhealthy.metricMonitorOptions,
+        );
+        // ...different object.
+        expect(healthy.metricMonitorOptions).not.toBe(
+          unhealthy.metricMonitorOptions,
+        );
+      }
+    });
+
+    /*
+     * Guards the in-place write at `MetricMonitorCriteria.evaluateAllSeries`
+     * from ever being reachable: an unset aggregation is what triggers it.
+     */
+    it("keeps both filters on the sustained aggregation", () => {
+      let checked: number = 0;
+
+      for (const template of ALL_TEMPLATES) {
+        for (const filter of getFilters(buildStep(template))) {
+          if (!filter.metricMonitorOptions) {
+            continue;
+          }
+
+          expect(filter.metricMonitorOptions.metricAggregationType).toBe(
+            EvaluateOverTimeType.AllValues,
+          );
+          checked++;
+        }
+      }
+
+      expect(checked).toBeGreaterThan(0);
+    });
+  });
+
+  /*
+   * Without a group-by every replica collapses into one number: an Avg heap
+   * template on three JVMs at 95/20/20 reads 45 and stays green while one
+   * OOM-kills, and a Max template fires without being able to name the
+   * replica. The two latency percentiles are service-level on purpose and are
+   * the only exemptions.
+   */
+  describe("per-instance grouping", () => {
+    const INSTANCE_KEY: string = "resource.service.instance.id";
+
+    const SERVICE_LEVEL_METRIC_TEMPLATE_IDS: Array<string> = [
+      "service-latency-p95",
+      "service-latency-p99",
+    ];
+
+    function getQueryConfigs(
+      template: ServiceAlertTemplate,
+    ): Array<MetricQueryConfigData> {
+      return (
+        buildStep(template).data?.metricMonitor?.metricViewConfig
+          ?.queryConfigs || []
+      );
+    }
+
+    it("groups every per-process metric template by the service instance", () => {
+      let grouped: number = 0;
+
+      for (const template of ALL_TEMPLATES) {
+        if (template.monitorType !== MonitorType.Metrics) {
+          continue;
+        }
+
+        const expected: Array<string> =
+          SERVICE_LEVEL_METRIC_TEMPLATE_IDS.includes(template.id)
+            ? []
+            : [INSTANCE_KEY];
+
+        const queryConfigs: Array<MetricQueryConfigData> =
+          getQueryConfigs(template);
+
+        expect(queryConfigs.length).toBeGreaterThan(0);
+
+        for (const queryConfig of queryConfigs) {
+          expect(
+            queryConfig.metricQueryData?.groupByAttributeKeys || [],
+          ).toEqual(expected);
+        }
+
+        if (expected.length > 0) {
+          grouped++;
+        }
+      }
+
+      // The canary: if grouping is reverted wholesale this drops to zero.
+      expect(grouped).toBeGreaterThan(20);
+    });
+
+    /*
+     * The two SLO percentiles opt out by passing `[]`, and `buildQueryConfig`
+     * must then omit the field rather than serialize an empty array — so an
+     * ungrouped template's stored payload is byte-identical to what it was
+     * before grouping existed.
+     */
+    it.each(SERVICE_LEVEL_METRIC_TEMPLATE_IDS)(
+      "%s stays service-level, with no group-by key written at all",
+      (templateId: string) => {
+        for (const queryConfig of getQueryConfigs(
+          getServiceAlertTemplateById(templateId)!,
+        )) {
+          expect(
+            queryConfig.metricQueryData?.groupByAttributeKeys,
+          ).toBeUndefined();
+        }
+
+        expect(
+          MonitorStep.getGroupByAttributeKeys(
+            buildStep(getServiceAlertTemplateById(templateId)!),
+          ),
+        ).toEqual([]);
+      },
+    );
+
+    /*
+     * The query config is only half of it: the worker reads the keys back
+     * through `MonitorStep.getGroupByAttributeKeys`, and that is what decides
+     * whether the evaluation fans out per series at all.
+     */
+    it("surfaces the instance key through the accessor the worker reads", () => {
+      for (const template of ALL_TEMPLATES) {
+        if (
+          template.monitorType !== MonitorType.Metrics ||
+          SERVICE_LEVEL_METRIC_TEMPLATE_IDS.includes(template.id)
+        ) {
+          continue;
+        }
+
+        expect(
+          MonitorStep.getGroupByAttributeKeys(buildStep(template)),
+        ).toEqual([INSTANCE_KEY]);
+      }
+    });
+
+    /*
+     * A key nobody registered renders as "Service Instance Id: ..." from the
+     * prettified-key fallback instead of "Instance: ...", and the
+     * catalog-level debuggability suite fails on it.
+     */
+    it("groups only by keys SeriesLabelDisplay can name", () => {
+      for (const template of ALL_TEMPLATES) {
+        for (const key of MonitorStep.getGroupByAttributeKeys(
+          buildStep(template),
+        )) {
+          expect(SeriesLabelDisplay.isKnownLabelKey(key)).toBe(true);
+        }
+      }
+    });
+
+    it("names the instance key 'Instance', prefixed or not", () => {
+      expect(SeriesLabelDisplay.getFriendlyLabelName(INSTANCE_KEY)).toBe(
+        "Instance",
+      );
+      expect(
+        SeriesLabelDisplay.getFriendlyLabelName("service.instance.id"),
+      ).toBe("Instance");
+    });
+
+    it("ranks the instance above the service it belongs to", () => {
+      // The monitor name already says which service; the replica does not.
+      expect(SeriesLabelDisplay.getLabelPriority(INSTANCE_KEY)).toBeLessThan(
+        SeriesLabelDisplay.getLabelPriority("service.name"),
+      );
+    });
+
+    /*
+     * `service.instance.id` is optional in the resource semantic conventions.
+     * An SDK that omits it produces one series whose label value is the empty
+     * string, and the display layer must drop it rather than render
+     * "Instance: " — which is what makes grouping safe to default on.
+     */
+    it("renders nothing for a service that does not report an instance", () => {
+      expect(
+        SeriesLabelDisplay.getDisplayLabels({ [INSTANCE_KEY]: "" }),
+      ).toEqual([]);
+      expect(
+        SeriesLabelDisplay.getDisplayLabels({ [INSTANCE_KEY]: "worker-7" }).map(
+          (label: { name: string; value: string }) => {
+            return `${label.name}: ${label.value}`;
+          },
+        ),
+      ).toEqual(["Instance: worker-7"]);
+    });
   });
 
   describe("the two ratio templates", () => {
@@ -651,6 +862,29 @@ describe("ServiceAlertTemplates", () => {
         }
       },
     );
+
+    /*
+     * The formula divides one query by the other. Both sides must carry the
+     * same group-by key, or the "Grouped By" root-cause block — which
+     * `MetricMonitorCriteria.buildContext` reads off a SINGLE query config —
+     * renders empty for whichever side it happened to read.
+     */
+    it.each(RATIO_TEMPLATE_IDS)(
+      "%s groups both queries on the same key, so the formula divides one instance's usage by its own limit",
+      (templateId: string) => {
+        const queryConfigs: Array<MetricQueryConfigData> =
+          buildStep(getServiceAlertTemplateById(templateId)!).data
+            ?.metricMonitor?.metricViewConfig?.queryConfigs || [];
+
+        expect(queryConfigs.length).toBe(2);
+        expect(queryConfigs[0]!.metricQueryData?.groupByAttributeKeys).toEqual([
+          "resource.service.instance.id",
+        ]);
+        expect(queryConfigs[1]!.metricQueryData?.groupByAttributeKeys).toEqual(
+          queryConfigs[0]!.metricQueryData?.groupByAttributeKeys,
+        );
+      },
+    );
   });
 
   describe("the metric names each template reads", () => {
@@ -694,6 +928,42 @@ describe("ServiceAlertTemplates", () => {
       "service-go-scheduler-latency-p99": ["go.schedule.duration"],
     };
 
+    /*
+     * Constraint 3 in the module header: a monitor holds exactly ONE metric
+     * name, so a template pointed at a pre-stabilization name must at least
+     * SAY so — otherwise a team on a current SDK gets a monitor that is green
+     * forever and indistinguishable from a healthy service.
+     */
+    it("names the metric it reads whenever that metric is a pre-stabilization name", () => {
+      let checked: number = 0;
+
+      for (const template of ALL_TEMPLATES) {
+        if (template.monitorType !== MonitorType.Metrics) {
+          continue;
+        }
+
+        const queryConfigs: Array<MetricQueryConfigData> =
+          buildStep(template).data?.metricMonitor?.metricViewConfig
+            ?.queryConfigs || [];
+
+        for (const queryConfig of queryConfigs) {
+          const metricName: string = String(
+            queryConfig.metricQueryData?.filterData?.["metricName"] || "",
+          );
+
+          if (!metricName.startsWith("process.runtime.")) {
+            continue;
+          }
+
+          expect(template.description).toContain(metricName);
+          checked++;
+        }
+      }
+
+      // Both Python templates read a `process.runtime.cpython.*` name today.
+      expect(checked).toBe(2);
+    });
+
     it("covers every metric template", () => {
       const metricTemplateIds: Array<string> = ALL_TEMPLATES.filter(
         (template: ServiceAlertTemplate) => {

package/Tests/Types/Monitor/TemplateGroupByKeys.test.ts

@@ -50,8 +50,9 @@ import ObjectID from "../../../Types/ObjectID";
  *     nothing. Filesystem utilization is per-mountpoint, so it groups by the
  *     unprefixed `mountpoint` datapoint label the hostmetrics receiver sets
  *     (read the same way in App/FeatureSet/Dashboard/src/Pages/Host/View/
- *     Overview.tsx). Host templates for per-device disk I/O or network
- *     interfaces do not exist yet; when one is added it groups by `device`.
+ *     Overview.tsx), plus the `device` label the same datapoint carries.
+ *     Host templates for per-device disk I/O or network interfaces do not
+ *     exist yet; when one is added it groups by `device`.
  *
  * Each table is exhaustive both ways — a new template with no row here fails
  * loudly, which forces the group-by decision to be made deliberately.
@@ -86,6 +87,14 @@ const EXPECTED_DOCKER_GROUP_BY: Array<GroupByExpectation> = [
   },
 ];
 
+/*
+ * `podman-cpu-throttling` was REMOVED from the catalog, not from this list by
+ * oversight. It thresholded `container.cpu.throttling.throttled_time` — a
+ * LIFETIME cumulative counter — at `> 0`, so any container that had ever been
+ * throttled since it started pinned the monitor unhealthy forever, and the
+ * healthy criteria could never match. Nothing in the ingest or criteria path
+ * deltas a counter, so there was no threshold that would have made it work.
+ */
 const EXPECTED_PODMAN_GROUP_BY: Array<GroupByExpectation> = [
   { id: "podman-high-cpu", groupByAttributeKeys: [CONTAINER_NAME_ATTRIBUTE] },
   {
@@ -96,10 +105,6 @@ const EXPECTED_PODMAN_GROUP_BY: Array<GroupByExpectation> = [
     id: "podman-restart-loop",
     groupByAttributeKeys: [CONTAINER_NAME_ATTRIBUTE],
   },
-  {
-    id: "podman-cpu-throttling",
-    groupByAttributeKeys: [CONTAINER_NAME_ATTRIBUTE],
-  },
   { id: "podman-high-pids", groupByAttributeKeys: [CONTAINER_NAME_ATTRIBUTE] },
   {
     id: "podman-container-down",
@@ -113,8 +118,17 @@ const EXPECTED_HOST_GROUP_BY: Array<GroupByExpectation> = [
   { id: "host-high-memory", groupByAttributeKeys: [] },
   { id: "host-high-load-average", groupByAttributeKeys: [] },
   { id: "host-high-processes", groupByAttributeKeys: [] },
-  // Per-entity within the host: one series (and one incident) per mount.
-  { id: "host-high-filesystem", groupByAttributeKeys: ["mountpoint"] },
+  /*
+   * Per-entity within the host: one series (and one incident) per mount.
+   * `device` rides along as identity the alert can NAME - it is constant
+   * for a mount so it adds no series, and "/var is 95% full on
+   * /dev/nvme0n1p2" is a materially different page from "/var is 95%
+   * full" when / sits on the same device.
+   */
+  {
+    id: "host-high-filesystem",
+    groupByAttributeKeys: ["mountpoint", "device"],
+  },
 ];
 
 function buildDockerArgs(): DockerAlertTemplateArgs {
@@ -280,7 +294,7 @@ describe("Alert template group-by keys", () => {
     }
   });
 
-  test("the host filesystem template is grouped per mountpoint", () => {
+  test("the host filesystem template is grouped per mountpoint, and names its device", () => {
     const filesystemTemplate: HostAlertTemplate | undefined =
       HOST_TEMPLATES.find((template: HostAlertTemplate) => {
         return template.id === "host-high-filesystem";
@@ -291,6 +305,20 @@ describe("Alert template group-by keys", () => {
     const step: MonitorStep =
       filesystemTemplate!.getMonitorStep(buildHostArgs());
 
-    expect(MonitorStep.getGroupByAttributeKeys(step)).toEqual(["mountpoint"]);
+    const keys: Array<string> = MonitorStep.getGroupByAttributeKeys(step);
+
+    /*
+     * `mountpoint` is the identity - one incident per mount, which is
+     * what stops a second mount filling up from being silenced behind
+     * the first one's open incident.
+     *
+     * `device` is identity the ALERT CAN NAME. The hostmetrics receiver
+     * puts it on the same datapoint and it is constant for a given
+     * mount, so it costs no extra series, and "/var is 95% full on
+     * /dev/nvme0n1p2" is a materially different page from "/var is 95%
+     * full" when / sits on the same device. See SeriesLabelDisplay,
+     * which renders both onto the alert title and description.
+     */
+    expect(keys).toEqual(["mountpoint", "device"]);
   });
 });

package/Tests/Types/Monitor/Utils/RecommendationCriteriaAssertions.ts

@@ -0,0 +1,102 @@
+import { FilterType } from "../../../../Types/Monitor/CriteriaFilter";
+import { getRecoveryThreshold } from "../../../../Types/Monitor/Recommendation/RecommendationCriteriaBuilder";
+
+/*
+ * The invariant every recommendation template's fire/recover pair must hold.
+ *
+ * These suites used to assert the OPPOSITE — that the two thresholds were
+ * the identical number and exactly partitioned the range ("disjoint
+ * complement"). That is a textbook flapping configuration, and it was
+ * load-bearing in the tests, so the bug could not be fixed without them
+ * failing. A metric parked at the threshold satisfied "> 90" on one
+ * evaluation and "<= 90" on the next, forever; one customer cluster
+ * produced 39 alert emails from a single monitor in under two hours.
+ *
+ * The invariant is now a DEAD BAND: the recover comparison is still the
+ * complement of the fire comparison, but its threshold sits strictly
+ * inside the fire threshold, so a metric has to move a real distance back
+ * before the monitor is called healthy.
+ */
+
+export interface CriteriaThreshold {
+  filterType: FilterType;
+  value: number;
+}
+
+export function getComplementFilterType(
+  filterType: FilterType,
+): FilterType | null {
+  switch (filterType) {
+    case FilterType.GreaterThan:
+      return FilterType.LessThanOrEqualTo;
+    case FilterType.GreaterThanOrEqualTo:
+      return FilterType.LessThan;
+    case FilterType.LessThan:
+      return FilterType.GreaterThanOrEqualTo;
+    case FilterType.LessThanOrEqualTo:
+      return FilterType.GreaterThan;
+    default:
+      return null;
+  }
+}
+
+/*
+ * Whether a fire/recover pair leaves a gap the metric must cross.
+ *
+ * A threshold of exactly 0 gets no dead band — 10% of 0 is 0, and a
+ * count-based criterion ("> 0 failed jobs") recovers correctly at exactly
+ * 0 — so an equal-valued pair is accepted there and only there. The same
+ * applies to the `> 0` / `= 0` pair Ceph uses for health-check series.
+ */
+export function hasRecoveryDeadBand(
+  fire: CriteriaThreshold,
+  recover: CriteriaThreshold,
+): boolean {
+  const expectedComplement: FilterType | null = getComplementFilterType(
+    fire.filterType,
+  );
+
+  const isComplement: boolean =
+    recover.filterType === expectedComplement ||
+    // Ceph health-check series: fire when the count is > 0, recover at = 0.
+    (fire.value === 0 &&
+      fire.filterType === FilterType.GreaterThan &&
+      recover.filterType === FilterType.EqualTo) ||
+    // Docker Swarm task-down: fire when uptime == 0, recover when uptime > 0.
+    (fire.value === 0 &&
+      fire.filterType === FilterType.EqualTo &&
+      recover.filterType === FilterType.GreaterThan);
+
+  if (!isComplement) {
+    return false;
+  }
+
+  const expectedRecoveryValue: number | undefined = getRecoveryThreshold({
+    filterType: fire.filterType,
+    value: fire.value,
+  });
+
+  if (expectedRecoveryValue === undefined) {
+    // No meaningful dead band exists; the thresholds must match exactly.
+    return recover.value === fire.value;
+  }
+
+  if (recover.value !== expectedRecoveryValue) {
+    return false;
+  }
+
+  /*
+   * And the band must point the right way: a ceiling recovers BELOW its
+   * firing threshold, a floor recovers ABOVE it. Checked independently of
+   * getRecoveryThreshold so a sign error in that function cannot make this
+   * assertion vacuously true.
+   */
+  if (
+    fire.filterType === FilterType.GreaterThan ||
+    fire.filterType === FilterType.GreaterThanOrEqualTo
+  ) {
+    return recover.value < fire.value;
+  }
+
+  return recover.value > fire.value;
+}