@oneuptime/common 14.0.0 → 14.0.2

package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts

@@ -34,6 +34,10 @@ import AffectedResourceList, {
   AffectedResourceListDetail,
   AffectedResourceListEntry,
 } from "./AffectedResourceList";
+import RootCauseList, {
+  RootCauseListDetail,
+  RootCauseListItem,
+} from "./RootCauseList";
 import DataToProcess from "./DataToProcess";
 import Monitor from "../../../Models/DatabaseModels/Monitor";
 import MonitorCriteria from "../../../Types/Monitor/MonitorCriteria";
@@ -124,6 +128,15 @@ export interface TelemetryExplorerDeepLink {
   dropped: Array<string>;
 }
 
+/**
+ * Which affected-resources rows breached the matched criteria, and which
+ * end of the value range is the worst.
+ */
+interface AffectedResourceBreachPredicate {
+  matches: (value: number) => boolean;
+  worstIsLowest: boolean;
+}
+
 export default class MonitorCriteriaEvaluator {
   public static async processMonitorStep(input: {
     dataToProcess: DataToProcess;
@@ -1556,9 +1569,7 @@ ${contextBlock}
           samples: breachingSamples,
           totalSamples: ctx.totalSamplesInWindow,
           unit: ctx.unit,
-          metricName: ctx.metricName,
           unitHeuristicMetricName: unitHeuristicMetricName,
-          alias: ctx.alias,
           components: ctx.components || [],
         })}`,
       );
@@ -1614,11 +1625,25 @@ ${contextBlock}
   }
 
   /**
-   * Build the **Breaching Samples** markdown section — a table of
-   * timestamps, values, and any group-by attributes. Caps the row count
-   * so a 30-minute window at 1-second granularity doesn't dump thousands
-   * of lines onto the root-cause page; the caller can always drill in
-   * via the metric explorer link.
+   * Build the **Breaching Samples** markdown section — a numbered list of
+   * the samples that breached, oldest first. Caps the list so a 30-minute
+   * window at 1-second granularity doesn't dump thousands of lines onto
+   * the root-cause page; the caller can always drill in via the metric
+   * explorer link.
+   *
+   *     1. `2026-08-14T10:30:00.000Z` — **1.07 GB**
+   *        - `a`: 537 MB
+   *        - `b`: 1.5 sec
+   *        - `k8s.pod.name`: `web-1`
+   *
+   * It used to be a table — Timestamp | Metric | Alias | Value, a column
+   * per formula component and one per attribute key — which grew wider
+   * with every attribute and read badly in the email card, in Slack and on
+   * the dashboard (see RootCauseList). Its Metric and Alias columns
+   * repeated the same two values on every row, the ones the Metric
+   * Details list directly above already states, so the list leaves them
+   * out; and a component or attribute a sample has no value for is left
+   * out rather than printed as "-".
    *
    * Timestamps are emitted as inline code wrapping ISO 8601 strings so
    * the client-side markdown viewer can localize them to the viewer's
@@ -1628,16 +1653,14 @@ ${contextBlock}
     samples: Array<MetricBreachingSample>;
     totalSamples?: number | undefined;
     unit: string | null;
-    metricName: string;
     /**
      * The metric name to reason about, or undefined for a formula.
      * See MonitorCriteriaEvaluator.metricNameForUnitHeuristics.
      */
     unitHeuristicMetricName: string | undefined;
-    alias: string;
     components: Array<MetricComponent>;
   }): string {
-    const MAX_ROWS: number = 20;
+    const MAX_SAMPLES_SHOWN: number = 20;
 
     // Sort chronologically and de-duplicate any accidental repeats
     const sorted: Array<MetricBreachingSample> = [...input.samples].sort(
@@ -1650,10 +1673,13 @@ ${contextBlock}
 
     const displayedSamples: Array<MetricBreachingSample> = sorted.slice(
       0,
-      MAX_ROWS,
+      MAX_SAMPLES_SHOWN,
     );
 
-    // Collect attribute keys that appear on any displayed sample
+    /*
+     * Collect attribute keys that appear on any displayed sample, so every
+     * item lists the attributes it has in the same order.
+     */
     const attrKeySet: Set<string> = new Set<string>();
     for (const s of displayedSamples) {
       for (const k of Object.keys(s.attributes || {})) {
@@ -1662,107 +1688,74 @@ ${contextBlock}
     }
     const attrKeys: Array<string> = Array.from(attrKeySet);
 
-    /*
-     * Escape pipe characters that could appear in the metric display
-     * name (formulas like "a | b" are unlikely but possible) so they
-     * don't break GitHub-flavored-markdown tables.
-     */
-    const escapeCell: (value: string) => string = (value: string): string => {
-      return value.replace(/\|/g, "\\|");
-    };
-
-    /*
-     * Column layout:
-     *   Timestamp | Metric | Alias | Value | <component_1> | ... | <attr_1> | ...
-     *
-     * The component columns let the reader see what each variable of a
-     * formula resolved to at the breach time — e.g. "when c = a + b
-     * breached 100, a was 55, b was 46". They are omitted for plain
-     * metric criteria.
-     */
-    const headerCells: Array<string> = [
-      "Timestamp",
-      "Metric",
-      "Alias",
-      "Value",
-    ];
-
-    /*
-     * The component columns carry no unit in their header any more. Each
-     * cell now names the unit it actually landed on, and auto-scaling
-     * means neighbouring rows can land on different ones — a header that
-     * read "a (By)" over cells reading "900 KB" and "1.2 MB" states a
-     * unit that neither row uses. The configured unit is not lost: the
-     * Components bullet list above the table still records it.
-     */
-    for (const component of input.components) {
-      headerCells.push(component.alias);
-    }
-
-    headerCells.push(...attrKeys);
-
-    const headerRow: string = `| ${headerCells.join(" | ")} |`;
-    const dividerRow: string = `| ${headerCells
-      .map(() => {
-        return "---";
-      })
-      .join(" | ")} |`;
-
-    const metricCell: string = `\`${escapeCell(input.metricName)}\``;
-    const aliasCell: string = input.alias
-      ? `\`${escapeCell(input.alias)}\``
-      : "-";
-
-    const dataRows: Array<string> = displayedSamples.map(
-      (s: MetricBreachingSample) => {
-        const timestampIso: string = new Date(s.timestamp).toISOString();
-        const cells: Array<string> = [
-          `\`${timestampIso}\``,
-          metricCell,
-          aliasCell,
-          MetricValueFormatter.format({
-            value: s.value,
-            unit: input.unit,
-            metricName: input.unitHeuristicMetricName,
-          }),
-        ];
+    const items: Array<RootCauseListItem> = displayedSamples.map(
+      (s: MetricBreachingSample): RootCauseListItem => {
+        const details: Array<RootCauseListDetail> = [];
 
+        /*
+         * What each variable of a formula resolved to at the breach time —
+         * e.g. "when c = a + b breached 100, a was 55, b was 46". Absent
+         * for plain metric criteria.
+         */
         for (const component of input.components) {
           const match: MetricComponentValue | undefined = (
             s.componentValues || []
           ).find((cv: MetricComponentValue) => {
             return cv.alias === component.alias;
           });
-          if (match && typeof match.value === "number") {
-            /*
-             * Each component keeps its OWN unit. Component values are
-             * indexed off the per-query series and never go through the
-             * sample→threshold-unit conversion the main Value column
-             * does, so they are in the component's legendUnit — mixing
-             * units across the row is deliberate, and is why every cell
-             * has to say which one it is in.
-             */
-            cells.push(
-              MetricValueFormatter.format({
-                value: match.value,
-                unit: component.unit,
-                metricName:
-                  MonitorCriteriaEvaluator.metricNameForUnitHeuristics({
-                    metricName: component.name,
-                    isFormula: component.isFormula,
-                  }),
-              }),
-            );
-          } else {
-            cells.push("-");
+
+          if (!match || typeof match.value !== "number") {
+            continue;
           }
+
+          /*
+           * Each component keeps its OWN unit. Component values are
+           * indexed off the per-query series and never go through the
+           * sample→threshold-unit conversion the sample's value does, so
+           * they are in the component's legendUnit — which is why each
+           * one names the unit it is in. Its configured unit is in the
+           * Components list of the Metric Details above.
+           */
+          details.push({
+            label: RootCauseList.code(component.alias),
+            value: MetricValueFormatter.format({
+              value: match.value,
+              unit: component.unit,
+              metricName: MonitorCriteriaEvaluator.metricNameForUnitHeuristics({
+                metricName: component.name,
+                isFormula: component.isFormula,
+              }),
+            }),
+          });
         }
 
+        /*
+         * The attributes identify the series the sample came from. Keys
+         * and values are telemetry, so both go in code spans:
+         * RootCauseList.code keeps whatever they contain inside the span.
+         */
         for (const k of attrKeys) {
           const v: unknown = (s.attributes as Record<string, unknown>)[k];
-          cells.push(v === undefined || v === null ? "-" : String(v));
+
+          if (v === undefined || v === null) {
+            continue;
+          }
+
+          details.push({
+            label: RootCauseList.code(k),
+            value: RootCauseList.code(String(v)),
+          });
         }
-        return `| ${cells.join(" | ")} |`;
+
+        return {
+          title: RootCauseList.code(new Date(s.timestamp).toISOString()),
+          value: `**${MetricValueFormatter.format({
+            value: s.value,
+            unit: input.unit,
+            metricName: input.unitHeuristicMetricName,
+          })}**`,
+          details: details,
+        };
       },
     );
 
@@ -1773,14 +1766,17 @@ ${contextBlock}
         totalSamples: input.totalSamples,
       }),
       "",
-      headerRow,
-      dividerRow,
-      ...dataRows,
+      RootCauseList.render(items),
     ];
 
     if (sorted.length > displayedSamples.length) {
+      /*
+       * The blank line matters: without it the note is a lazy
+       * continuation of the last item, not a paragraph of its own.
+       */
       lines.push(
-        `\n_Showing the first ${displayedSamples.length} of ${sorted.length} breaching samples._`,
+        "",
+        `_Showing the first ${displayedSamples.length} of ${sorted.length} breaching samples._`,
       );
     }
 
@@ -2172,6 +2168,7 @@ ${contextBlock}
     dataToProcess: DataToProcess;
     monitorStep: MonitorStep;
     monitor: Monitor;
+    criteriaInstance?: MonitorCriteriaInstance | undefined;
   }): Promise<string | null> {
     const metricResponse: MetricMonitorResponse =
       input.dataToProcess as MetricMonitorResponse;
@@ -2204,18 +2201,40 @@ ${contextBlock}
 
     // Affected resources
     if (breakdown.affectedResources && breakdown.affectedResources.length > 0) {
-      // Sort by metric value descending (worst first) and filter out zero-value resources
-      const sortedResources: Array<KubernetesAffectedResource> = [
-        ...breakdown.affectedResources,
-      ]
-        .filter((r: KubernetesAffectedResource) => {
-          return r.metricValue > 0;
-        })
-        .sort(
-          (a: KubernetesAffectedResource, b: KubernetesAffectedResource) => {
-            return b.metricValue - a.metricValue;
-          },
-        );
+      /*
+       * Keep the rows that satisfy the criteria that just matched, worst
+       * first — NOT simply the non-zero rows. k8s-node-not-ready fires on
+       * `k8s.node.condition_ready = 0`, so the zero rows are the NotReady
+       * nodes; the old hardcoded `> 0` dropped exactly those and named a
+       * healthy node in the analysis below.
+       */
+      const breach: AffectedResourceBreachPredicate =
+        MonitorCriteriaEvaluator.getAffectedResourceBreachPredicate({
+          criteriaInstance: input.criteriaInstance,
+          floorEqualityFiresOnFall: true,
+        });
+
+      const sortedResources: Array<KubernetesAffectedResource> =
+        breakdown.affectedResources
+          .map((r: KubernetesAffectedResource) => {
+            /*
+             * A fall criteria breached on the resource's lowest sample
+             * in the window, not its highest.
+             */
+            return breach.worstIsLowest
+              ? { ...r, metricValue: r.lowestMetricValue ?? r.metricValue }
+              : r;
+          })
+          .filter((r: KubernetesAffectedResource) => {
+            return breach.matches(r.metricValue);
+          })
+          .sort(
+            (a: KubernetesAffectedResource, b: KubernetesAffectedResource) => {
+              return breach.worstIsLowest
+                ? a.metricValue - b.metricValue
+                : b.metricValue - a.metricValue;
+            },
+          );
 
       if (sortedResources.length === 0) {
         return sections.join("\n");
@@ -3273,37 +3292,61 @@ ${contextBlock}
   }
 
   /*
-   * Which rows of the Ceph affected-resources list actually BREACHED.
+   * Which rows of an affected-resources list actually BREACHED.
    *
    * The blanket `metricValue > 0` below is right for every count- or
    * level-style criteria ("> 0 active health checks", "> 85 % used"), but
-   * it is exactly backwards for the availability signals: the OSD Down /
-   * OSD Out / Quorum Degraded criteria fire on `< 1` over ceph_osd_up /
-   * ceph_osd_in / ceph_mon_quorum_status, so the ZERO rows are the down
-   * daemons. Dropping them rendered a list of the HEALTHY daemons under
-   * an incident that tells the reader to "check the root cause for the
-   * affected ceph_daemon label".
+   * it is exactly backwards for the availability signals: the Ceph OSD
+   * Down / OSD Out / Quorum Degraded criteria fire on `< 1` over
+   * ceph_osd_up / ceph_osd_in / ceph_mon_quorum_status, so the ZERO rows
+   * are the down daemons. Dropping them rendered a list of the HEALTHY
+   * daemons under an incident that tells the reader to "check the root
+   * cause for the affected ceph_daemon label".
    *
    * Invert only for a criteria that fires when the metric FALLS.
    * Everything else — every `> 0` health check, PG count, latency and
-   * capacity ratio, and every `= 0` recovery criteria — keeps the
-   * existing `> 0`, worst-highest-first list byte for byte.
+   * capacity ratio, and (by default) every `= 0` recovery criteria —
+   * keeps the existing `> 0`, worst-highest-first list byte for byte.
+   *
+   * `floorEqualityFiresOnFall` additionally counts a FIRING `= 0` (or
+   * below) as a fall. Kubernetes needs it: k8s-node-not-ready and
+   * k8s-etcd-no-leader fire on `= 0` over k8s.node.condition_ready /
+   * etcd_server_has_leader, and zero is the floor of those metrics, so
+   * the only way to reach it is down. It is gated on the criteria opening
+   * an incident or alert because the same `= 0` is also how the restart,
+   * failed-pod and unavailable-replica templates express RECOVERY, and
+   * that all-clear must not become a list of healthy zero rows. Ceph
+   * leaves it off: all of its `= 0` criteria are recoveries.
    */
-  private static getCephBreachPredicate(
-    criteriaInstance?: MonitorCriteriaInstance | undefined,
-  ): { matches: (value: number) => boolean; worstIsLowest: boolean } {
+  private static getAffectedResourceBreachPredicate(input: {
+    criteriaInstance?: MonitorCriteriaInstance | undefined;
+    floorEqualityFiresOnFall?: boolean | undefined;
+  }): AffectedResourceBreachPredicate {
     const metricFilters: Array<CriteriaFilter> = (
-      criteriaInstance?.data?.filters || []
+      input.criteriaInstance?.data?.filters || []
     ).filter((f: CriteriaFilter) => {
       return f.checkOn === CheckOn.MetricValue && typeof f.value === "number";
     });
 
+    const opensIncidentOrAlert: boolean =
+      input.criteriaInstance?.data?.createIncidents === true ||
+      input.criteriaInstance?.data?.createAlerts === true;
+
     const firesWhenMetricFalls: boolean =
       metricFilters.length > 0 &&
       metricFilters.every((f: CriteriaFilter) => {
-        return (
+        if (
           f.filterType === FilterType.LessThan ||
           f.filterType === FilterType.LessThanOrEqualTo
+        ) {
+          return true;
+        }
+
+        return (
+          input.floorEqualityFiresOnFall === true &&
+          opensIncidentOrAlert &&
+          f.filterType === FilterType.EqualTo &&
+          (f.value as number) <= 0
         );
       });
 
@@ -3321,6 +3364,10 @@ ${contextBlock}
         return metricFilters.some((f: CriteriaFilter) => {
           const threshold: number = f.value as number;
 
+          if (f.filterType === FilterType.EqualTo) {
+            return value === threshold;
+          }
+
           return f.filterType === FilterType.LessThan
             ? value < threshold
             : value <= threshold;
@@ -3478,12 +3525,10 @@ ${contextBlock}
        * ceph_osd_in / ceph_mon_quorum_status the criteria is `< 1`, so
        * the zero rows are the whole point of the incident.
        */
-      const breach: {
-        matches: (value: number) => boolean;
-        worstIsLowest: boolean;
-      } = MonitorCriteriaEvaluator.getCephBreachPredicate(
-        input.criteriaInstance,
-      );
+      const breach: AffectedResourceBreachPredicate =
+        MonitorCriteriaEvaluator.getAffectedResourceBreachPredicate({
+          criteriaInstance: input.criteriaInstance,
+        });
 
       const sortedResources: Array<CephAffectedResource> = [
         ...breakdown.affectedResources,

package/Server/Utils/Monitor/MonitorCriteriaObservationBuilder.ts

@@ -1785,7 +1785,11 @@ export default class MonitorCriteriaObservationBuilder {
     )}`;
 
     if (status.remediation) {
-      message += ` (${status.remediation})`;
+      /*
+       * A remediation can span lines - Azure SQL Database's names two
+       * statements for two databases - but this is one sentence.
+       */
+      message += ` (${status.remediation.replace(/\s*[\r\n]+\s*/g, " ")})`;
     }
 
     return `${message}.`;

package/Server/Utils/Monitor/MonitorIncident.ts

@@ -17,9 +17,14 @@ import { TelemetryQuery } from "../../../Types/Telemetry/TelemetryQuery";
 import { DisableAutomaticIncidentCreation } from "../../EnvironmentConfig";
 import IncidentService from "../../Services/IncidentService";
 import IncidentSeverityService from "../../Services/IncidentSeverityService";
+import LabelService from "../../Services/LabelService";
+import OnCallDutyPolicyService from "../../Services/OnCallDutyPolicyService";
+import DatabaseService from "../../Services/DatabaseService";
+import DatabaseBaseModel from "../../../Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
 import ProjectScopedReferenceValidator from "../Database/ProjectScopedReferenceValidator";
 import IncidentStateTimelineService from "../../Services/IncidentStateTimelineService";
 import IncidentMemberService from "../../Services/IncidentMemberService";
+import TeamMemberService from "../../Services/TeamMemberService";
 import NetworkDeviceOwnerUserService, {
   NetworkDeviceOwners,
 } from "../../Services/NetworkDeviceOwnerUserService";
@@ -770,20 +775,45 @@ export default class MonitorIncident {
             resolved: resourceContext,
           });
 
-          incident.onCallDutyPolicies =
-            criteriaIncident.onCallPolicyIds?.map((id: ObjectID) => {
+          /*
+           * On-call policies and labels from the criteria. IncidentService
+           * rejects any that belong to another project or no longer exist,
+           * and monitorSteps can still hold such ids (see the severity above).
+           * Drop those here rather than fail the whole ingest job; a
+           * criteria with a stale policy still pages its remaining ones.
+           */
+          const onCallPolicyIds: Array<ObjectID | string> =
+            await this.getCriteriaIdsUsableInProject({
+              monitor: input.monitor,
+              criteriaName: input.criteriaInstance.data?.name,
+              modelName: "on-call policy",
+              ids: criteriaIncident.onCallPolicyIds,
+              service: OnCallDutyPolicyService,
+            });
+
+          incident.onCallDutyPolicies = onCallPolicyIds.map(
+            (id: ObjectID | string) => {
               const onCallPolicy: OnCallDutyPolicy = new OnCallDutyPolicy();
               onCallPolicy._id = id.toString();
               return onCallPolicy;
-            }) || [];
+            },
+          );
 
           // Set labels from criteria
-          incident.labels =
-            criteriaIncident.labelIds?.map((id: ObjectID) => {
-              const label: Label = new Label();
-              label._id = id.toString();
-              return label;
-            }) || [];
+          const labelIds: Array<ObjectID | string> =
+            await this.getCriteriaIdsUsableInProject({
+              monitor: input.monitor,
+              criteriaName: input.criteriaInstance.data?.name,
+              modelName: "label",
+              ids: criteriaIncident.labelIds,
+              service: LabelService,
+            });
+
+          incident.labels = labelIds.map((id: ObjectID | string) => {
+            const label: Label = new Label();
+            label._id = id.toString();
+            return label;
+          });
 
           incident.isCreatedAutomatically = true;
 
@@ -874,6 +904,26 @@ export default class MonitorIncident {
                 const assignment: IncidentMemberRoleAssignment =
                   roleAssignment as IncidentMemberRoleAssignment;
 
+                /*
+                 * The criteria is saved configuration and can name a user
+                 * who has since left the project; they get no role on new
+                 * incidents.
+                 */
+                if (
+                  assignment.roleId &&
+                  assignment.userId &&
+                  !(await TeamMemberService.isUserMemberOfProject({
+                    projectId: input.monitor.projectId!,
+                    userId: new ObjectID(assignment.userId.toString()),
+                  }))
+                ) {
+                  logger.debug(
+                    `${input.monitor.id?.toString()} - Skipped incident member role ${assignment.roleId.toString()} for user ${assignment.userId.toString()}: not a member of the project`,
+                    incidentLogAttributes,
+                  );
+                  continue;
+                }
+
                 if (assignment.roleId && assignment.userId) {
                   const incidentMember: IncidentMember = new IncidentMember();
                   incidentMember.incidentId = createdIncident.id!;
@@ -953,6 +1003,51 @@ export default class MonitorIncident {
     }
   }
 
+  /*
+   * The ids in a criteria list that this monitor's project can use, logging
+   * the ones it drops. See ProjectScopedReferenceValidator.filterUsableInProject.
+   */
+  private static async getCriteriaIdsUsableInProject(input: {
+    monitor: Monitor;
+    criteriaName: string | undefined;
+    modelName: string;
+    ids: Array<ObjectID> | undefined;
+    service: DatabaseService<DatabaseBaseModel>;
+  }): Promise<Array<ObjectID | string>> {
+    if (!input.ids || input.ids.length === 0) {
+      return [];
+    }
+
+    const result: {
+      usableIds: Array<ObjectID | string>;
+      droppedIds: Array<ObjectID | string>;
+    } = await ProjectScopedReferenceValidator.filterUsableInProject({
+      projectId: input.monitor.projectId,
+      ids: input.ids,
+      service: input.service,
+    });
+
+    if (result.droppedIds.length > 0) {
+      logger.error(
+        `${input.monitor.id?.toString()} - Criteria "${
+          input.criteriaName
+        }" references ${input.modelName} ${result.droppedIds
+          .map((id: ObjectID | string) => {
+            return id.toString();
+          })
+          .join(
+            ", ",
+          )}, which does not exist in project ${input.monitor.projectId?.toString()}. Creating the incident without it.`,
+        {
+          projectId: input.monitor.projectId?.toString(),
+          monitorId: input.monitor.id?.toString(),
+        } as LogAttributes,
+      );
+    }
+
+    return result.usableIds;
+  }
+
   /*
    * Merges two owner id lists, deduping by string value. Criteria-authored
    * ids can arrive as plain strings from stored JSON, so normalise through