@oneuptime/common 12.0.33 → 12.0.34

package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts

@@ -53,7 +53,11 @@ import DatabaseMonitorResponse, {
   DatabaseMetricGroupStatus,
 } from "../../../Types/Monitor/DatabaseMonitor/DatabaseMonitorResponse";
 import MonitorType from "../../../Types/Monitor/MonitorType";
-import { CheckOn, CriteriaFilter } from "../../../Types/Monitor/CriteriaFilter";
+import {
+  CheckOn,
+  CriteriaFilter,
+  FilterType,
+} from "../../../Types/Monitor/CriteriaFilter";
 import OneUptimeDate from "../../../Types/Date";
 import { JSONObject } from "../../../Types/JSON";
 import Dictionary from "../../../Types/Dictionary";
@@ -2677,10 +2681,73 @@ ${contextBlock}
     return sections.length > 0 ? sections.join("\n") : null;
   }
 
+  /*
+   * Which rows of the Ceph affected-resources table actually BREACHED.
+   *
+   * The blanket `metricValue > 0` below is right for every count- or
+   * level-style criteria ("> 0 active health checks", "> 85 % used"), but
+   * it is exactly backwards for the availability signals: the OSD Down /
+   * OSD Out / Quorum Degraded criteria fire on `< 1` over ceph_osd_up /
+   * ceph_osd_in / ceph_mon_quorum_status, so the ZERO rows are the down
+   * daemons. Dropping them rendered a table of the HEALTHY daemons under
+   * an incident that tells the reader to "check the root cause for the
+   * affected ceph_daemon label".
+   *
+   * Invert only for a criteria that fires when the metric FALLS.
+   * Everything else — every `> 0` health check, PG count, latency and
+   * capacity ratio, and every `= 0` recovery criteria — keeps the
+   * existing `> 0`, worst-highest-first table byte for byte.
+   */
+  private static getCephBreachPredicate(
+    criteriaInstance?: MonitorCriteriaInstance | undefined,
+  ): { matches: (value: number) => boolean; worstIsLowest: boolean } {
+    const metricFilters: Array<CriteriaFilter> = (
+      criteriaInstance?.data?.filters || []
+    ).filter((f: CriteriaFilter) => {
+      return f.checkOn === CheckOn.MetricValue && typeof f.value === "number";
+    });
+
+    const firesWhenMetricFalls: boolean =
+      metricFilters.length > 0 &&
+      metricFilters.every((f: CriteriaFilter) => {
+        return (
+          f.filterType === FilterType.LessThan ||
+          f.filterType === FilterType.LessThanOrEqualTo
+        );
+      });
+
+    if (!firesWhenMetricFalls) {
+      return {
+        matches: (value: number): boolean => {
+          return value > 0;
+        },
+        worstIsLowest: false,
+      };
+    }
+
+    return {
+      matches: (value: number): boolean => {
+        return metricFilters.some((f: CriteriaFilter) => {
+          const threshold: number = f.value as number;
+
+          return f.filterType === FilterType.LessThan
+            ? value < threshold
+            : value <= threshold;
+        });
+      },
+      /*
+       * A criteria that fires when the metric falls ranks the smallest
+       * value worst, so "worst first" is ascending here.
+       */
+      worstIsLowest: true,
+    };
+  }
+
   private static buildCephRootCauseContext(input: {
     dataToProcess: DataToProcess;
     monitorStep: MonitorStep;
     monitor: Monitor;
+    criteriaInstance?: MonitorCriteriaInstance | undefined;
   }): string | null {
     const metricResponse: MetricMonitorResponse =
       input.dataToProcess as MetricMonitorResponse;
@@ -2736,21 +2803,28 @@ ${contextBlock}
 
     if (breakdown && breakdown.affectedResources.length > 0) {
       /*
-       * K8s-parity render: drop zero-value rows, worst (highest)
-       * first, top 10. Note that for availability metrics
-       * (ceph_osd_up / ceph_osd_in / ceph_mon_quorum_status)
-       * zero-valued rows ARE the down resources — those still drive
-       * alerting through the per-series criteria evaluation; this
-       * table is supplementary context only.
+       * Keep the rows that satisfy the criteria that just matched, worst
+       * first, top 10 — NOT simply the non-zero rows. For ceph_osd_up /
+       * ceph_osd_in / ceph_mon_quorum_status the criteria is `< 1`, so
+       * the zero rows are the whole point of the incident.
        */
+      const breach: {
+        matches: (value: number) => boolean;
+        worstIsLowest: boolean;
+      } = MonitorCriteriaEvaluator.getCephBreachPredicate(
+        input.criteriaInstance,
+      );
+
       const sortedResources: Array<CephAffectedResource> = [
         ...breakdown.affectedResources,
       ]
         .filter((r: CephAffectedResource) => {
-          return r.metricValue > 0;
+          return breach.matches(r.metricValue);
         })
         .sort((a: CephAffectedResource, b: CephAffectedResource) => {
-          return b.metricValue - a.metricValue;
+          return breach.worstIsLowest
+            ? a.metricValue - b.metricValue
+            : b.metricValue - a.metricValue;
         });
 
       /*
@@ -2858,10 +2932,17 @@ ${contextBlock}
       lines.push(
         `Recommended actions: Check container logs with \`kubectl logs ${topResource.podName || "<pod-name>"} -c ${topResource.containerName || "<container>"} --previous\` and inspect events with \`kubectl describe pod ${topResource.podName || "<pod-name>"}\`.`,
       );
-    } else if (
-      metricName === "k8s.pod.phase" &&
-      breakdown.attributes["k8s.pod.phase"] === "Pending"
-    ) {
+    } else if (metricName === "k8s.pod.phase") {
+      /*
+       * The second half of this condition used to be
+       * `breakdown.attributes["k8s.pod.phase"] === "Pending"`, which could
+       * never be true. `breakdown.attributes` is the QUERY's attribute map
+       * (MonitorTelemetryMonitor stamps `resource.`-prefixed keys into it),
+       * and no receiver or processor in the shipped agent emits a pod
+       * phase attribute at all — the phase is the metric's VALUE
+       * (1 = Pending ... 5 = Unknown). So this branch never ran and every
+       * pod-phase alert fell through to the generic Kubernetes text.
+       */
       lines.push(`Pods are stuck in Pending phase and unable to be scheduled.`);
       lines.push(
         `Common causes: insufficient CPU/memory resources on nodes, node affinity/taint restrictions preventing scheduling, PersistentVolumeClaim pending, or resource quota exceeded.`,
@@ -2889,7 +2970,18 @@ ${contextBlock}
       );
     } else if (
       metricName === "k8s.node.cpu.utilization" ||
-      (metricName.includes("cpu") && metricName.includes("utilization"))
+      /*
+       * Node-scoped ONLY. The bare substring test used to swallow every
+       * pod- and container-scoped CPU metric — `k8s.pod.cpu.utilization`,
+       * and now `k8s.pod.cpu_limit_utilization` — and answer them with
+       * "Node X is at N% CPU utilization... consider scaling the cluster",
+       * directly contradicting the pod-limit template's own description,
+       * which tells the reader NOT to add replicas, and naming a node for
+       * a pod-scoped alert.
+       */
+      (metricName.startsWith("k8s.node.") &&
+        metricName.includes("cpu") &&
+        metricName.includes("utilization"))
     ) {
       lines.push(`Node CPU utilization has exceeded the configured threshold.`);
       if (topResource.nodeName) {
@@ -2905,7 +2997,10 @@ ${contextBlock}
       );
     } else if (
       metricName === "k8s.node.memory.usage" ||
-      (metricName.includes("memory") && metricName.includes("usage"))
+      // Node-scoped only — see the CPU branch above.
+      (metricName.startsWith("k8s.node.") &&
+        metricName.includes("memory") &&
+        metricName.includes("usage"))
     ) {
       lines.push(
         `Node memory utilization has exceeded the configured threshold.`,

package/Server/Utils/Monitor/MonitorIncident.ts

@@ -27,6 +27,7 @@ import logger, { LogAttributes } from "../Logger";
 import CaptureSpan from "../Telemetry/CaptureSpan";
 import DataToProcess from "./DataToProcess";
 import MonitorTemplateUtil from "./MonitorTemplateUtil";
+import SeriesContextEnricher from "./SeriesContextEnricher";
 import MonitorDependencySuppression, {
   DependencySuppressionResult,
 } from "./MonitorDependencySuppression";
@@ -597,13 +598,28 @@ export default class MonitorIncident {
               seriesLabels,
             });
 
-          incident.title = MonitorTemplateUtil.processTemplateString({
-            value: criteriaIncident.title,
-            storageMap,
+          /*
+           * Render the criteria's template, then make it say WHICH
+           * series it is about. Mirrors MonitorAlert exactly - the two
+           * must agree, or the same breach reads differently depending
+           * on whether the criteria was configured to raise an alert or
+           * an incident. See SeriesContextEnricher for why this is not
+           * done inside the template itself.
+           */
+          incident.title = SeriesContextEnricher.enrichTitle({
+            title: MonitorTemplateUtil.processTemplateString({
+              value: criteriaIncident.title,
+              storageMap,
+            }),
+            seriesLabels,
           });
-          incident.description = MonitorTemplateUtil.processTemplateString({
-            value: criteriaIncident.description,
-            storageMap,
+          incident.description = SeriesContextEnricher.enrichDescription({
+            description: MonitorTemplateUtil.processTemplateString({
+              value: criteriaIncident.description,
+              storageMap,
+            }),
+            seriesLabels,
+            monitorType: input.monitor.monitorType,
           });
 
           /*

package/Server/Utils/Monitor/MonitorTemplateUtil.ts

@@ -32,6 +32,8 @@ import ExternalStatusPageMonitorResponse, {
 } from "../../../Types/Monitor/ExternalStatusPageMonitor/ExternalStatusPageMonitorResponse";
 import MetricMonitorResponse from "../../../Types/Monitor/MetricMonitor/MetricMonitorResponse";
 import Typeof from "../../../Types/Typeof";
+import SeriesDebugHints from "../../../Types/Monitor/SeriesContext/SeriesDebugHints";
+import SeriesLabelDisplay from "../../../Types/Monitor/SeriesContext/SeriesLabelDisplay";
 import VMUtil from "../VM/VMAPI";
 import DataToProcess from "./DataToProcess";
 import logger from "../Logger";
@@ -627,6 +629,33 @@ export default class MonitorTemplateUtil {
       storageMap["seriesLabels"] = data.seriesLabels;
     }
 
+    /*
+     * Ready-made renderings of the series identity.
+     *
+     * These are set unconditionally - to "" when the monitor is not
+     * grouped, or when its labels carry no usable value - and that is
+     * the whole point. `VMUtil.replaceValueInPlace` leaves a placeholder
+     * it cannot resolve in the output verbatim, so a title written as
+     * `"Pod CPU high{{seriesResourceSuffix}}"` would otherwise render
+     * with the braces still in it on any monitor without a group-by.
+     * Always defining them makes the variables safe to use in a shipped
+     * template that has to work for grouped and ungrouped monitors
+     * alike.
+     */
+    storageMap["seriesResourceSuffix"] = SeriesLabelDisplay.buildTitleSuffix(
+      data.seriesLabels,
+    );
+    storageMap["seriesResourceSummary"] = SeriesLabelDisplay.buildInlineSummary(
+      data.seriesLabels,
+    );
+    storageMap["seriesResourceBlock"] = SeriesLabelDisplay.buildMarkdownBlock(
+      data.seriesLabels,
+    );
+    storageMap["seriesDebugCommands"] = SeriesDebugHints.buildMarkdownBlock({
+      monitorType: data.monitorType,
+      seriesLabels: data.seriesLabels,
+    });
+
     /*
      * Monitor identity fields. Always exposed (when a monitor is provided),
      * independent of monitorType, so templates like `{{monitorName}}` work

package/Server/Utils/Monitor/SeriesContextEnricher.ts

@@ -0,0 +1,304 @@
+import ColumnLength from "../../../Types/Database/ColumnLength";
+import { JSONObject } from "../../../Types/JSON";
+import MonitorType from "../../../Types/Monitor/MonitorType";
+import SeriesDebugHints from "../../../Types/Monitor/SeriesContext/SeriesDebugHints";
+import SeriesLabelDisplay, {
+  DisplaySeriesLabel,
+} from "../../../Types/Monitor/SeriesContext/SeriesLabelDisplay";
+
+/*
+ * Make a per-series alert or incident say WHICH pod, container, node or
+ * mount it is about.
+ *
+ * A grouped metric monitor raises one alert per breaching series and
+ * stores that series' identity in `seriesLabels`. Until now the identity
+ * stopped there: the title came from the criteria template, which is one
+ * fixed string shared by every series, so a cluster with fifty saturated
+ * pods produced fifty alerts whose titles were character-for-character
+ * identical. The identity was reachable - in a table, at the bottom of
+ * the alert detail page - but the alert list, the Slack message, the
+ * email and the phone notification all showed the same undifferentiated
+ * line.
+ *
+ * This enricher runs at creation time, after the criteria template has
+ * been rendered, and appends the series identity to the title and a
+ * resource + first-commands block to the description.
+ *
+ * Two properties matter and are covered by tests:
+ *
+ *   - It never fights the user's own template. A title that already
+ *     names the series (because the user wrote `{{resource.k8s.pod.name}}`
+ *     into it, or because the new `{{seriesResourceSuffix}}` variable
+ *     expanded there) is left exactly as the user wrote it. The check is
+ *     on the VALUES, not on the placeholders, so it works the same
+ *     whether the identity arrived from a template variable or was typed
+ *     literally.
+ *
+ *   - It is a no-op for whole-monitor alerts. A monitor with no group-by
+ *     has no series identity, and nothing is appended.
+ */
+
+/*
+ * Hoisted rather than written inline at its single use site: two lint
+ * rules disagree about parenthesising a regex literal in an expression
+ * and `--fix` oscillates between them (ESLintCircularFixesWarning). A
+ * named constant sidesteps the argument and reads better anyway.
+ */
+const WordCharacterPattern: RegExp = /[A-Za-z0-9]/;
+
+export default class SeriesContextEnricher {
+  /**
+   * The `title` column's own limit, taken from the schema rather than
+   * restated as a number - `Alert.title` and `Incident.title` are both
+   * `ColumnType.LongText`.
+   */
+  private static readonly MaxTitleLength: number = ColumnLength.LongText;
+
+  /**
+   * Shortest label value that may be matched as a bare substring.
+   *
+   * Ceph pool ids and Proxmox vmids are one or two characters, and
+   * "does the title contain 3?" is answered `true` by any title with a
+   * threshold in it ("Restarts > 3"). Below this length only this
+   * module's own `Name: value` rendering counts as a mention, which
+   * keeps idempotency without letting a coincidence swallow the
+   * identity.
+   */
+  private static readonly MinimumMatchableValueLength: number = 4;
+
+  /**
+   * What counts as "part of the same word" when deciding whether a
+   * value found in the text is really a mention of it.
+   *
+   * Deliberately narrower than `\w`: `-`, `.`, `/` and `_` all appear
+   * inside the values themselves (pod names, mountpoints, Ceph daemon
+   * ids), so treating them as word characters would make a value that
+   * IS present look like part of a longer token.
+   */
+  private static isWordCharacter(character: string): boolean {
+    if (character === "") {
+      return false;
+    }
+
+    return WordCharacterPattern.test(character);
+  }
+
+  /**
+   * Whether `text` already names this label's value.
+   *
+   * Two ways to be already-named, and both matter:
+   *
+   *   - This module's own rendering (`"Pod: web-1"`) is matched exactly.
+   *     That is what makes the enricher idempotent, whatever the value
+   *     looks like.
+   *
+   *   - A distinctive value appearing anywhere on a token boundary. That
+   *     is the user's own template - `{{resource.k8s.pod.name}}` has
+   *     already expanded by the time this runs, so the check has to be
+   *     on the VALUE, not on the placeholder. The boundary requirement
+   *     stops a pod called `web` from counting as mentioned because the
+   *     title happens to contain the word "website".
+   */
+  private static isValueMentioned(input: {
+    text: string;
+    label: DisplaySeriesLabel;
+  }): boolean {
+    const { text, label } = input;
+
+    if (text.includes(`${label.name}: ${label.value}`)) {
+      return true;
+    }
+
+    if (
+      label.value.length < SeriesContextEnricher.MinimumMatchableValueLength
+    ) {
+      return false;
+    }
+
+    let searchFrom: number = 0;
+
+    for (;;) {
+      const index: number = text.indexOf(label.value, searchFrom);
+
+      if (index === -1) {
+        return false;
+      }
+
+      const before: string = index > 0 ? text.charAt(index - 1) : "";
+      const after: string = text.charAt(index + label.value.length);
+
+      if (
+        !SeriesContextEnricher.isWordCharacter(before) &&
+        !SeriesContextEnricher.isWordCharacter(after)
+      ) {
+        return true;
+      }
+
+      searchFrom = index + 1;
+    }
+  }
+
+  /*
+   * The identity labels not already present in `text`.
+   *
+   * Deliberately per-label rather than all-or-nothing: a title that
+   * mentions the pod by name but not the namespace still gets the
+   * namespace appended, because dropping the rest whenever a user's
+   * template happened to mention one identifier would lose exactly the
+   * context this exists to add.
+   */
+  private static getUnmentionedLabels(input: {
+    text: string;
+    seriesLabels: JSONObject | undefined;
+  }): Array<DisplaySeriesLabel> {
+    const labels: Array<DisplaySeriesLabel> =
+      SeriesLabelDisplay.getDisplayLabels(input.seriesLabels);
+
+    if (labels.length === 0) {
+      return [];
+    }
+
+    const text: string = input.text || "";
+
+    return labels.filter((label: DisplaySeriesLabel) => {
+      return !SeriesContextEnricher.isValueMentioned({ text, label });
+    });
+  }
+
+  /*
+   * `title` with the series identity appended, or `title` unchanged when
+   * there is no identity to add or the title already carries it.
+   */
+  public static enrichTitle(input: {
+    title: string;
+    seriesLabels: JSONObject | undefined;
+  }): string {
+    const title: string = input.title || "";
+
+    const unmentioned: Array<DisplaySeriesLabel> =
+      SeriesContextEnricher.getUnmentionedLabels({
+        text: title,
+        seriesLabels: input.seriesLabels,
+      });
+
+    if (unmentioned.length === 0) {
+      return title;
+    }
+
+    /*
+     * Rebuild a label map from just the unmentioned labels so the
+     * shared formatter still applies its own ordering and title cap
+     * rather than this module reimplementing either.
+     */
+    const remaining: JSONObject = {};
+
+    for (const label of unmentioned) {
+      remaining[label.key] = label.value;
+    }
+
+    const suffix: string = SeriesLabelDisplay.buildTitleSuffix(remaining);
+
+    if (!suffix) {
+      return title;
+    }
+
+    /*
+     * `Alert.title` and `Incident.title` are LongText, i.e. 500
+     * characters. Overflowing the column does not produce a shortened
+     * title - it fails the INSERT, and the alert does not exist at all.
+     * That would turn a labelling improvement into dropped pages, so the
+     * appended identity is bounded here as well as per-value inside
+     * SeriesLabelDisplay.
+     *
+     * A base title that is ALREADY at or over the limit is returned
+     * untouched: it was going to fail with or without this enricher, and
+     * silently truncating what the user wrote is not this module's call
+     * to make.
+     */
+    if (title.length >= SeriesContextEnricher.MaxTitleLength) {
+      return title;
+    }
+
+    const room: number = SeriesContextEnricher.MaxTitleLength - title.length;
+
+    if (suffix.length <= room) {
+      return `${title}${suffix}`;
+    }
+
+    return `${title}${suffix.slice(0, room)}`;
+  }
+
+  /*
+   * `description` with an "Affected resource" block and the read-only
+   * commands worth running first appended.
+   *
+   * The description is what reaches Slack, email and the mobile push, so
+   * this is where the full identity goes - the title only has room for
+   * the top few labels.
+   */
+  public static enrichDescription(input: {
+    description: string;
+    seriesLabels: JSONObject | undefined;
+    monitorType: MonitorType | undefined;
+  }): string {
+    const description: string = input.description || "";
+
+    const labels: Array<DisplaySeriesLabel> =
+      SeriesLabelDisplay.getDisplayLabels(input.seriesLabels);
+
+    if (labels.length === 0) {
+      return description;
+    }
+
+    const sections: Array<string> = [];
+
+    /*
+     * Only skip the resource block when the description already names
+     * EVERY label. A description mentioning the pod but not the node is
+     * still missing the thing the engineer needs.
+     */
+    const unmentioned: Array<DisplaySeriesLabel> =
+      SeriesContextEnricher.getUnmentionedLabels({
+        text: description,
+        seriesLabels: input.seriesLabels,
+      });
+
+    const block: string = SeriesLabelDisplay.buildMarkdownBlock(
+      input.seriesLabels,
+    );
+
+    /*
+     * The exact-block check is the idempotency guarantee. Per-label
+     * matching alone cannot provide it: a one-character label value
+     * (a Ceph pool id, a Proxmox vmid) is deliberately not matched as a
+     * bare substring, so a second pass would append the block again.
+     */
+    if (unmentioned.length > 0 && block && !description.includes(block)) {
+      sections.push(block);
+    }
+
+    const commandsBlock: string = SeriesDebugHints.buildMarkdownBlock({
+      monitorType: input.monitorType,
+      seriesLabels: input.seriesLabels,
+    });
+
+    /*
+     * The commands embed the label values, so a description that
+     * already carries them would otherwise get a near-duplicate block
+     * appended on top of a user's own runbook snippet.
+     */
+    if (commandsBlock && !description.includes(commandsBlock)) {
+      sections.push(commandsBlock);
+    }
+
+    if (sections.length === 0) {
+      return description;
+    }
+
+    if (!description.trim()) {
+      return sections.join("\n\n");
+    }
+
+    return `${description}\n\n${sections.join("\n\n")}`;
+  }
+}