@oneuptime/common 14.0.0 → 14.0.1

package/Server/Utils/Monitor/AffectedResourceList.ts

@@ -1,4 +1,8 @@
 import { escapeMarkdownInline } from "../../../Utils/Markdown/MarkdownEscape";
+import RootCauseList, {
+  RootCauseListDetail,
+  RootCauseListItem,
+} from "./RootCauseList";
 
 /*
  * The "Affected Resources" block of a platform monitor's root cause.
@@ -30,10 +34,8 @@ import { escapeMarkdownInline } from "../../../Utils/Markdown/MarkdownEscape";
  * and the rest of its identity sits underneath as labelled bullets. A
  * missing attribute is simply left out, where the table had to print "-".
  *
- * Plain markdown, deliberately: marked (email), react-markdown (dashboard),
- * slackify-markdown (Slack), the dashboard editor's WYSIWYG converters and
- * Markdown.convertToPlainText all understand a nested list, so nothing
- * downstream needs to know this block exists.
+ * The list itself is a RootCauseList, the shape the Breaching Samples block
+ * of a metric monitor's root cause uses too.
  */
 
 export interface AffectedResourceListDetail {
@@ -67,46 +69,36 @@ export default class AffectedResourceList {
     // The resources to show, already sorted worst first.
     entries: Array<AffectedResourceListEntry>;
   }): string {
-    const lines: Array<string> = [];
-
-    input.entries.forEach(
-      (entry: AffectedResourceListEntry, index: number): void => {
-        const marker: string = `${index + 1}.`;
-
-        /*
-         * A nested bullet belongs to its item only when it is indented to
-         * the item's content column, which is one past the marker — three
-         * spaces under "1.", four under "10.". A fixed three-space indent
-         * would push item 10's details out into a separate top-level list.
-         */
-        const indent: string = " ".repeat(marker.length + 1);
-
+    const items: Array<RootCauseListItem> = input.entries.map(
+      (entry: AffectedResourceListEntry): RootCauseListItem => {
         /*
          * The kind is bold so each item's first line stands out from the
          * detail bullets under it — the value at the end of the line is
          * bold too, so the eye lands on what it is and how bad it is.
          */
         const kind: string = escapeMarkdownInline(entry.kind).trim();
-        const title: string = [kind ? `**${kind}**` : "", entry.name.trim()]
-          .filter((part: string): boolean => {
-            return part.length > 0;
-          })
-          .join(" ");
 
-        lines.push(`${marker} ${title} — ${entry.value}`);
-
-        for (const detail of entry.details) {
-          if (!detail.value || detail.value.trim().length === 0) {
-            continue;
-          }
-
-          lines.push(
-            `${indent}- ${escapeMarkdownInline(detail.label)}: ${detail.value}`,
-          );
-        }
+        return {
+          title: [kind ? `**${kind}**` : "", entry.name.trim()]
+            .filter((part: string): boolean => {
+              return part.length > 0;
+            })
+            .join(" "),
+          value: entry.value,
+          details: entry.details.map(
+            (detail: AffectedResourceListDetail): RootCauseListDetail => {
+              return {
+                label: escapeMarkdownInline(detail.label),
+                value: detail.value,
+              };
+            },
+          ),
+        };
       },
     );
 
+    const lines: Array<string> = [RootCauseList.render(items)];
+
     const hiddenCount: number = input.totalCount - input.entries.length;
 
     if (hiddenCount > 0) {
@@ -125,47 +117,13 @@ export default class AffectedResourceList {
   }
 
   /*
-   * A resource identifier as an inline code span.
-   *
-   * The identifiers come from telemetry attributes, and a VMware VM or a
-   * Proxmox guest can be named anything. Wrapping a name that contains a
-   * backtick in single backticks closes the span early and spills the rest
-   * of the name — and whatever markdown it contains — into the list. So the
-   * fence is always one backtick longer than the longest run inside the
-   * value, and padded with a space on each side when the value has any
-   * backticks at all (CommonMark strips exactly one from each side, so the
-   * padding never shows). Line breaks become spaces: a newline inside a
-   * list item would end the item.
+   * A resource identifier as an inline code span. The identifiers come from
+   * telemetry attributes, and a VMware VM or a Proxmox guest can be named
+   * anything — RootCauseList.code keeps a name with backticks or line breaks
+   * inside its span.
    */
   public static code(value: string | undefined | null): string {
-    if (value === undefined || value === null) {
-      return "";
-    }
-
-    const text: string = String(value)
-      .replace(/\s*[\r\n]+\s*/g, " ")
-      .trim();
-
-    if (text.length === 0) {
-      return "";
-    }
-
-    const backtickRuns: Array<string> = text.match(/`+/g) || [];
-
-    const longestRun: number = backtickRuns.reduce(
-      (longest: number, run: string): number => {
-        return Math.max(longest, run.length);
-      },
-      0,
-    );
-
-    if (longestRun === 0) {
-      return `\`${text}\``;
-    }
-
-    const fence: string = "`".repeat(longestRun + 1);
-
-    return `${fence} ${text} ${fence}`;
+    return RootCauseList.code(value);
   }
 
   /*

package/Server/Utils/Monitor/Criteria/CompareCriteria.ts

@@ -984,7 +984,7 @@ export default class CompareCriteria {
        * reads naturally ("is 42, 55, 60"). For larger arrays, summarize
        * — dumping 30+ numbers on one line makes the root cause unreadable
        * and the detailed breakdown is shown in the Breaching Samples
-       * table below.
+       * list below.
        */
       const MAX_INLINE: number = 5;
 

package/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.ts

@@ -1004,7 +1004,7 @@ export default class MetricMonitorCriteria {
     const sigmaAbs: number = Math.abs(firstBreach.sigma);
 
     /*
-     * Same rendering as the breaching-samples table and the comparison
+     * Same rendering as the breaching-samples list and the comparison
      * sentence — an anomaly on a memory metric reads "1.07 GB", not
      * "1073741824.000 By". σ is a count of standard deviations, not a
      * quantity in the metric's unit, so it stays a bare number; the

package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts

@@ -34,6 +34,10 @@ import AffectedResourceList, {
   AffectedResourceListDetail,
   AffectedResourceListEntry,
 } from "./AffectedResourceList";
+import RootCauseList, {
+  RootCauseListDetail,
+  RootCauseListItem,
+} from "./RootCauseList";
 import DataToProcess from "./DataToProcess";
 import Monitor from "../../../Models/DatabaseModels/Monitor";
 import MonitorCriteria from "../../../Types/Monitor/MonitorCriteria";
@@ -124,6 +128,15 @@ export interface TelemetryExplorerDeepLink {
   dropped: Array<string>;
 }
 
+/**
+ * Which affected-resources rows breached the matched criteria, and which
+ * end of the value range is the worst.
+ */
+interface AffectedResourceBreachPredicate {
+  matches: (value: number) => boolean;
+  worstIsLowest: boolean;
+}
+
 export default class MonitorCriteriaEvaluator {
   public static async processMonitorStep(input: {
     dataToProcess: DataToProcess;
@@ -1556,9 +1569,7 @@ ${contextBlock}
           samples: breachingSamples,
           totalSamples: ctx.totalSamplesInWindow,
           unit: ctx.unit,
-          metricName: ctx.metricName,
           unitHeuristicMetricName: unitHeuristicMetricName,
-          alias: ctx.alias,
           components: ctx.components || [],
         })}`,
       );
@@ -1614,11 +1625,25 @@ ${contextBlock}
   }
 
   /**
-   * Build the **Breaching Samples** markdown section — a table of
-   * timestamps, values, and any group-by attributes. Caps the row count
-   * so a 30-minute window at 1-second granularity doesn't dump thousands
-   * of lines onto the root-cause page; the caller can always drill in
-   * via the metric explorer link.
+   * Build the **Breaching Samples** markdown section — a numbered list of
+   * the samples that breached, oldest first. Caps the list so a 30-minute
+   * window at 1-second granularity doesn't dump thousands of lines onto
+   * the root-cause page; the caller can always drill in via the metric
+   * explorer link.
+   *
+   *     1. `2026-08-14T10:30:00.000Z` — **1.07 GB**
+   *        - `a`: 537 MB
+   *        - `b`: 1.5 sec
+   *        - `k8s.pod.name`: `web-1`
+   *
+   * It used to be a table — Timestamp | Metric | Alias | Value, a column
+   * per formula component and one per attribute key — which grew wider
+   * with every attribute and read badly in the email card, in Slack and on
+   * the dashboard (see RootCauseList). Its Metric and Alias columns
+   * repeated the same two values on every row, the ones the Metric
+   * Details list directly above already states, so the list leaves them
+   * out; and a component or attribute a sample has no value for is left
+   * out rather than printed as "-".
    *
    * Timestamps are emitted as inline code wrapping ISO 8601 strings so
    * the client-side markdown viewer can localize them to the viewer's
@@ -1628,16 +1653,14 @@ ${contextBlock}
     samples: Array<MetricBreachingSample>;
     totalSamples?: number | undefined;
     unit: string | null;
-    metricName: string;
     /**
      * The metric name to reason about, or undefined for a formula.
      * See MonitorCriteriaEvaluator.metricNameForUnitHeuristics.
      */
     unitHeuristicMetricName: string | undefined;
-    alias: string;
     components: Array<MetricComponent>;
   }): string {
-    const MAX_ROWS: number = 20;
+    const MAX_SAMPLES_SHOWN: number = 20;
 
     // Sort chronologically and de-duplicate any accidental repeats
     const sorted: Array<MetricBreachingSample> = [...input.samples].sort(
@@ -1650,10 +1673,13 @@ ${contextBlock}
 
     const displayedSamples: Array<MetricBreachingSample> = sorted.slice(
       0,
-      MAX_ROWS,
+      MAX_SAMPLES_SHOWN,
     );
 
-    // Collect attribute keys that appear on any displayed sample
+    /*
+     * Collect attribute keys that appear on any displayed sample, so every
+     * item lists the attributes it has in the same order.
+     */
     const attrKeySet: Set<string> = new Set<string>();
     for (const s of displayedSamples) {
       for (const k of Object.keys(s.attributes || {})) {
@@ -1662,107 +1688,74 @@ ${contextBlock}
     }
     const attrKeys: Array<string> = Array.from(attrKeySet);
 
-    /*
-     * Escape pipe characters that could appear in the metric display
-     * name (formulas like "a | b" are unlikely but possible) so they
-     * don't break GitHub-flavored-markdown tables.
-     */
-    const escapeCell: (value: string) => string = (value: string): string => {
-      return value.replace(/\|/g, "\\|");
-    };
-
-    /*
-     * Column layout:
-     *   Timestamp | Metric | Alias | Value | <component_1> | ... | <attr_1> | ...
-     *
-     * The component columns let the reader see what each variable of a
-     * formula resolved to at the breach time — e.g. "when c = a + b
-     * breached 100, a was 55, b was 46". They are omitted for plain
-     * metric criteria.
-     */
-    const headerCells: Array<string> = [
-      "Timestamp",
-      "Metric",
-      "Alias",
-      "Value",
-    ];
-
-    /*
-     * The component columns carry no unit in their header any more. Each
-     * cell now names the unit it actually landed on, and auto-scaling
-     * means neighbouring rows can land on different ones — a header that
-     * read "a (By)" over cells reading "900 KB" and "1.2 MB" states a
-     * unit that neither row uses. The configured unit is not lost: the
-     * Components bullet list above the table still records it.
-     */
-    for (const component of input.components) {
-      headerCells.push(component.alias);
-    }
-
-    headerCells.push(...attrKeys);
-
-    const headerRow: string = `| ${headerCells.join(" | ")} |`;
-    const dividerRow: string = `| ${headerCells
-      .map(() => {
-        return "---";
-      })
-      .join(" | ")} |`;
-
-    const metricCell: string = `\`${escapeCell(input.metricName)}\``;
-    const aliasCell: string = input.alias
-      ? `\`${escapeCell(input.alias)}\``
-      : "-";
-
-    const dataRows: Array<string> = displayedSamples.map(
-      (s: MetricBreachingSample) => {
-        const timestampIso: string = new Date(s.timestamp).toISOString();
-        const cells: Array<string> = [
-          `\`${timestampIso}\``,
-          metricCell,
-          aliasCell,
-          MetricValueFormatter.format({
-            value: s.value,
-            unit: input.unit,
-            metricName: input.unitHeuristicMetricName,
-          }),
-        ];
+    const items: Array<RootCauseListItem> = displayedSamples.map(
+      (s: MetricBreachingSample): RootCauseListItem => {
+        const details: Array<RootCauseListDetail> = [];
 
+        /*
+         * What each variable of a formula resolved to at the breach time —
+         * e.g. "when c = a + b breached 100, a was 55, b was 46". Absent
+         * for plain metric criteria.
+         */
         for (const component of input.components) {
           const match: MetricComponentValue | undefined = (
             s.componentValues || []
           ).find((cv: MetricComponentValue) => {
             return cv.alias === component.alias;
           });
-          if (match && typeof match.value === "number") {
-            /*
-             * Each component keeps its OWN unit. Component values are
-             * indexed off the per-query series and never go through the
-             * sample→threshold-unit conversion the main Value column
-             * does, so they are in the component's legendUnit — mixing
-             * units across the row is deliberate, and is why every cell
-             * has to say which one it is in.
-             */
-            cells.push(
-              MetricValueFormatter.format({
-                value: match.value,
-                unit: component.unit,
-                metricName:
-                  MonitorCriteriaEvaluator.metricNameForUnitHeuristics({
-                    metricName: component.name,
-                    isFormula: component.isFormula,
-                  }),
-              }),
-            );
-          } else {
-            cells.push("-");
+
+          if (!match || typeof match.value !== "number") {
+            continue;
           }
+
+          /*
+           * Each component keeps its OWN unit. Component values are
+           * indexed off the per-query series and never go through the
+           * sample→threshold-unit conversion the sample's value does, so
+           * they are in the component's legendUnit — which is why each
+           * one names the unit it is in. Its configured unit is in the
+           * Components list of the Metric Details above.
+           */
+          details.push({
+            label: RootCauseList.code(component.alias),
+            value: MetricValueFormatter.format({
+              value: match.value,
+              unit: component.unit,
+              metricName: MonitorCriteriaEvaluator.metricNameForUnitHeuristics({
+                metricName: component.name,
+                isFormula: component.isFormula,
+              }),
+            }),
+          });
         }
 
+        /*
+         * The attributes identify the series the sample came from. Keys
+         * and values are telemetry, so both go in code spans:
+         * RootCauseList.code keeps whatever they contain inside the span.
+         */
         for (const k of attrKeys) {
           const v: unknown = (s.attributes as Record<string, unknown>)[k];
-          cells.push(v === undefined || v === null ? "-" : String(v));
+
+          if (v === undefined || v === null) {
+            continue;
+          }
+
+          details.push({
+            label: RootCauseList.code(k),
+            value: RootCauseList.code(String(v)),
+          });
         }
-        return `| ${cells.join(" | ")} |`;
+
+        return {
+          title: RootCauseList.code(new Date(s.timestamp).toISOString()),
+          value: `**${MetricValueFormatter.format({
+            value: s.value,
+            unit: input.unit,
+            metricName: input.unitHeuristicMetricName,
+          })}**`,
+          details: details,
+        };
       },
     );
 
@@ -1773,14 +1766,17 @@ ${contextBlock}
         totalSamples: input.totalSamples,
       }),
       "",
-      headerRow,
-      dividerRow,
-      ...dataRows,
+      RootCauseList.render(items),
     ];
 
     if (sorted.length > displayedSamples.length) {
+      /*
+       * The blank line matters: without it the note is a lazy
+       * continuation of the last item, not a paragraph of its own.
+       */
       lines.push(
-        `\n_Showing the first ${displayedSamples.length} of ${sorted.length} breaching samples._`,
+        "",
+        `_Showing the first ${displayedSamples.length} of ${sorted.length} breaching samples._`,
       );
     }
 
@@ -2172,6 +2168,7 @@ ${contextBlock}
     dataToProcess: DataToProcess;
     monitorStep: MonitorStep;
     monitor: Monitor;
+    criteriaInstance?: MonitorCriteriaInstance | undefined;
   }): Promise<string | null> {
     const metricResponse: MetricMonitorResponse =
       input.dataToProcess as MetricMonitorResponse;
@@ -2204,18 +2201,40 @@ ${contextBlock}
 
     // Affected resources
     if (breakdown.affectedResources && breakdown.affectedResources.length > 0) {
-      // Sort by metric value descending (worst first) and filter out zero-value resources
-      const sortedResources: Array<KubernetesAffectedResource> = [
-        ...breakdown.affectedResources,
-      ]
-        .filter((r: KubernetesAffectedResource) => {
-          return r.metricValue > 0;
-        })
-        .sort(
-          (a: KubernetesAffectedResource, b: KubernetesAffectedResource) => {
-            return b.metricValue - a.metricValue;
-          },
-        );
+      /*
+       * Keep the rows that satisfy the criteria that just matched, worst
+       * first — NOT simply the non-zero rows. k8s-node-not-ready fires on
+       * `k8s.node.condition_ready = 0`, so the zero rows are the NotReady
+       * nodes; the old hardcoded `> 0` dropped exactly those and named a
+       * healthy node in the analysis below.
+       */
+      const breach: AffectedResourceBreachPredicate =
+        MonitorCriteriaEvaluator.getAffectedResourceBreachPredicate({
+          criteriaInstance: input.criteriaInstance,
+          floorEqualityFiresOnFall: true,
+        });
+
+      const sortedResources: Array<KubernetesAffectedResource> =
+        breakdown.affectedResources
+          .map((r: KubernetesAffectedResource) => {
+            /*
+             * A fall criteria breached on the resource's lowest sample
+             * in the window, not its highest.
+             */
+            return breach.worstIsLowest
+              ? { ...r, metricValue: r.lowestMetricValue ?? r.metricValue }
+              : r;
+          })
+          .filter((r: KubernetesAffectedResource) => {
+            return breach.matches(r.metricValue);
+          })
+          .sort(
+            (a: KubernetesAffectedResource, b: KubernetesAffectedResource) => {
+              return breach.worstIsLowest
+                ? a.metricValue - b.metricValue
+                : b.metricValue - a.metricValue;
+            },
+          );
 
       if (sortedResources.length === 0) {
         return sections.join("\n");
@@ -3273,37 +3292,61 @@ ${contextBlock}
   }
 
   /*
-   * Which rows of the Ceph affected-resources list actually BREACHED.
+   * Which rows of an affected-resources list actually BREACHED.
    *
    * The blanket `metricValue > 0` below is right for every count- or
    * level-style criteria ("> 0 active health checks", "> 85 % used"), but
-   * it is exactly backwards for the availability signals: the OSD Down /
-   * OSD Out / Quorum Degraded criteria fire on `< 1` over ceph_osd_up /
-   * ceph_osd_in / ceph_mon_quorum_status, so the ZERO rows are the down
-   * daemons. Dropping them rendered a list of the HEALTHY daemons under
-   * an incident that tells the reader to "check the root cause for the
-   * affected ceph_daemon label".
+   * it is exactly backwards for the availability signals: the Ceph OSD
+   * Down / OSD Out / Quorum Degraded criteria fire on `< 1` over
+   * ceph_osd_up / ceph_osd_in / ceph_mon_quorum_status, so the ZERO rows
+   * are the down daemons. Dropping them rendered a list of the HEALTHY
+   * daemons under an incident that tells the reader to "check the root
+   * cause for the affected ceph_daemon label".
    *
    * Invert only for a criteria that fires when the metric FALLS.
    * Everything else — every `> 0` health check, PG count, latency and
-   * capacity ratio, and every `= 0` recovery criteria — keeps the
-   * existing `> 0`, worst-highest-first list byte for byte.
+   * capacity ratio, and (by default) every `= 0` recovery criteria —
+   * keeps the existing `> 0`, worst-highest-first list byte for byte.
+   *
+   * `floorEqualityFiresOnFall` additionally counts a FIRING `= 0` (or
+   * below) as a fall. Kubernetes needs it: k8s-node-not-ready and
+   * k8s-etcd-no-leader fire on `= 0` over k8s.node.condition_ready /
+   * etcd_server_has_leader, and zero is the floor of those metrics, so
+   * the only way to reach it is down. It is gated on the criteria opening
+   * an incident or alert because the same `= 0` is also how the restart,
+   * failed-pod and unavailable-replica templates express RECOVERY, and
+   * that all-clear must not become a list of healthy zero rows. Ceph
+   * leaves it off: all of its `= 0` criteria are recoveries.
    */
-  private static getCephBreachPredicate(
-    criteriaInstance?: MonitorCriteriaInstance | undefined,
-  ): { matches: (value: number) => boolean; worstIsLowest: boolean } {
+  private static getAffectedResourceBreachPredicate(input: {
+    criteriaInstance?: MonitorCriteriaInstance | undefined;
+    floorEqualityFiresOnFall?: boolean | undefined;
+  }): AffectedResourceBreachPredicate {
     const metricFilters: Array<CriteriaFilter> = (
-      criteriaInstance?.data?.filters || []
+      input.criteriaInstance?.data?.filters || []
     ).filter((f: CriteriaFilter) => {
       return f.checkOn === CheckOn.MetricValue && typeof f.value === "number";
     });
 
+    const opensIncidentOrAlert: boolean =
+      input.criteriaInstance?.data?.createIncidents === true ||
+      input.criteriaInstance?.data?.createAlerts === true;
+
     const firesWhenMetricFalls: boolean =
       metricFilters.length > 0 &&
       metricFilters.every((f: CriteriaFilter) => {
-        return (
+        if (
           f.filterType === FilterType.LessThan ||
           f.filterType === FilterType.LessThanOrEqualTo
+        ) {
+          return true;
+        }
+
+        return (
+          input.floorEqualityFiresOnFall === true &&
+          opensIncidentOrAlert &&
+          f.filterType === FilterType.EqualTo &&
+          (f.value as number) <= 0
         );
       });
 
@@ -3321,6 +3364,10 @@ ${contextBlock}
         return metricFilters.some((f: CriteriaFilter) => {
           const threshold: number = f.value as number;
 
+          if (f.filterType === FilterType.EqualTo) {
+            return value === threshold;
+          }
+
           return f.filterType === FilterType.LessThan
             ? value < threshold
             : value <= threshold;
@@ -3478,12 +3525,10 @@ ${contextBlock}
        * ceph_osd_in / ceph_mon_quorum_status the criteria is `< 1`, so
        * the zero rows are the whole point of the incident.
        */
-      const breach: {
-        matches: (value: number) => boolean;
-        worstIsLowest: boolean;
-      } = MonitorCriteriaEvaluator.getCephBreachPredicate(
-        input.criteriaInstance,
-      );
+      const breach: AffectedResourceBreachPredicate =
+        MonitorCriteriaEvaluator.getAffectedResourceBreachPredicate({
+          criteriaInstance: input.criteriaInstance,
+        });
 
       const sortedResources: Array<CephAffectedResource> = [
         ...breakdown.affectedResources,