@oneuptime/common 12.0.32 → 12.0.34

package/Server/Utils/Monitor/Criteria/EvaluateOverTime.ts

@@ -177,6 +177,7 @@ export default class EvaluateOverTime {
         data.criteriaFilter.checkOn === CheckOn.IsRequestTimeout ||
         data.criteriaFilter.checkOn === CheckOn.DnsIsOnline ||
         data.criteriaFilter.checkOn === CheckOn.SnmpIsOnline ||
+        data.criteriaFilter.checkOn === CheckOn.DatabaseIsOnline ||
         data.criteriaFilter.checkOn === CheckOn.ExternalStatusPageIsOnline;
 
       return {
@@ -220,9 +221,15 @@ export default class EvaluateOverTime {
       };
     }
 
+    /*
+     * Resolved from the whole filter, not from the CheckOn alone: a Database
+     * Metric filter names its series in databaseMonitorOptions.metricType, so
+     * the CheckOn-only lookup would return null for every one of them and
+     * silently downgrade the filter to an instantaneous comparison.
+     */
     const metricName: MonitorMetricType | null =
-      MonitorMetricTypeUtil.getMonitorMetricTypeByCheckOnOrNull(
-        data.criteriaFilter.checkOn,
+      MonitorMetricTypeUtil.getMonitorMetricTypeByCriteriaFilterOrNull(
+        data.criteriaFilter,
       );
 
     if (!metricName) {

package/Server/Utils/Monitor/Criteria/IncomingRequestCriteria.ts

@@ -199,6 +199,20 @@ export default class IncomingRequestCriteria {
       }
     }
 
+    /*
+     * Header names and values are compared case-insensitively, and BOTH
+     * sides are lower-cased for it.
+     *
+     * Only the observed side used to be: a filter typed as "X-Api-Key" (the
+     * spelling every HTTP client and every piece of documentation uses) was
+     * matched against a list that had already been lower-cased, so Contains
+     * could never fire and NotContains fired on every single request. The
+     * lower-casing of the observed side is what says the intent was
+     * case-insensitive; this is the other half of it.
+     */
+    const criteriaValueLowerCase: string | null =
+      typeof value === Typeof.String ? (value as string).toLowerCase() : null;
+
     if (input.criteriaFilter.checkOn === CheckOn.RequestHeader) {
       const headerKeys: Array<string> = Object.keys(
         (input.dataToProcess as IncomingMonitorRequest).requestHeaders || {},
@@ -208,14 +222,20 @@ export default class IncomingRequestCriteria {
 
       // contains
       if (input.criteriaFilter.filterType === FilterType.Contains) {
-        if (value && headerKeys && headerKeys.includes(value as string)) {
+        if (
+          criteriaValueLowerCase &&
+          headerKeys.includes(criteriaValueLowerCase)
+        ) {
           return `Request header contains ${value}.`;
         }
         return null;
       }
 
       if (input.criteriaFilter.filterType === FilterType.NotContains) {
-        if (value && headerKeys && !headerKeys.includes(value as string)) {
+        if (
+          criteriaValueLowerCase &&
+          !headerKeys.includes(criteriaValueLowerCase)
+        ) {
           return `Request header does not contain ${value}.`;
         }
         return null;
@@ -231,14 +251,20 @@ export default class IncomingRequestCriteria {
 
       // contains
       if (input.criteriaFilter.filterType === FilterType.Contains) {
-        if (value && headerValues && headerValues.includes(value as string)) {
+        if (
+          criteriaValueLowerCase &&
+          headerValues.includes(criteriaValueLowerCase)
+        ) {
           return `Request header value contains ${value}.`;
         }
         return null;
       }
 
       if (input.criteriaFilter.filterType === FilterType.NotContains) {
-        if (value && headerValues && !headerValues.includes(value as string)) {
+        if (
+          criteriaValueLowerCase &&
+          !headerValues.includes(criteriaValueLowerCase)
+        ) {
           return `Request header value does not contain ${value}.`;
         }
         return null;

package/Server/Utils/Monitor/Criteria/SnmpMonitorCriteria.ts

@@ -27,6 +27,22 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
 import logger from "../../Logger";
 
 export default class SnmpMonitorCriteria {
+  /*
+   * Every CheckOn whose value comes out of the SNMP walk. Reachability
+   * (SnmpIsOnline) and traps are the two that do not.
+   */
+  private static isWalkDependentCheckOn(checkOn: CheckOn): boolean {
+    return (
+      checkOn === CheckOn.SnmpWalkIsSucceeding ||
+      checkOn === CheckOn.SnmpResponseTime ||
+      checkOn === CheckOn.SnmpOidExists ||
+      checkOn === CheckOn.SnmpOidValue ||
+      checkOn === CheckOn.SnmpInterfaceIsDown ||
+      checkOn === CheckOn.SnmpInterfaceUtilizationPercent ||
+      checkOn === CheckOn.SnmpInterfaceErrorsPerSecond
+    );
+  }
+
   /*
    * Interface scope for the interface CheckOns: when the criteria carries
    * snmpMonitorOptions.interfaceName, only interfaces whose name or alias
@@ -256,6 +272,29 @@ export default class SnmpMonitorCriteria {
       return null;
     }
 
+    /*
+     * No walk, no verdict. A device without usable SNMP credentials is only
+     * pinged, and the walk pipeline hands its monitors a response with
+     * `snmpResponse` undefined - never a synthesized failure. Every criterion
+     * that reads the walk is therefore NOT EVALUATED on such a poll (null),
+     * rather than breaching: "OID Exists is False" must not raise an
+     * incident on a device nobody ever asked an OID of, and "Walk Is
+     * Succeeding is False" must mean "attempted and failed". Reachability
+     * (SnmpIsOnline) is read from the top-level isOnline further down and
+     * is unaffected.
+     *
+     * This runs before the over-time lookup on purpose: a window of stored
+     * walk-time samples says nothing about a poll that walked nothing.
+     */
+    if (
+      SnmpMonitorCriteria.isWalkDependentCheckOn(
+        input.criteriaFilter.checkOn,
+      ) &&
+      !snmpResponse
+    ) {
+      return null;
+    }
+
     const overTime: OverTimeCriteriaValue =
       await EvaluateOverTime.getOverTimeValueForCriteriaFilter({
         projectId: (input.dataToProcess as ProbeMonitorResponse).projectId,
@@ -282,7 +321,12 @@ export default class SnmpMonitorCriteria {
       | boolean
       | undefined = overTime.value;
 
-    // Check if SNMP device is online
+    /*
+     * Device reachability: the top-level isOnline is "answered ping OR the
+     * walk succeeded", stamped by the walk pipeline - the same verdict the
+     * device list's status pill shows. Deliberately not the walk's own
+     * isOnline, which the next CheckOn covers.
+     */
     if (input.criteriaFilter.checkOn === CheckOn.SnmpIsOnline) {
       const currentIsOnline: boolean | Array<boolean> =
         (overTimeValue as Array<boolean>) ??
@@ -294,6 +338,22 @@ export default class SnmpMonitorCriteria {
       });
     }
 
+    /*
+     * The walk itself. Gated above: a poll with no walk returns null before
+     * reaching here, so `snmpResponse` is present and "False" genuinely
+     * means the walk was attempted and failed.
+     */
+    if (input.criteriaFilter.checkOn === CheckOn.SnmpWalkIsSucceeding) {
+      if (!snmpResponse) {
+        return null;
+      }
+
+      return CompareCriteria.compareCriteriaBoolean({
+        value: snmpResponse.isOnline === true,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
     // Check SNMP response time
     if (input.criteriaFilter.checkOn === CheckOn.SnmpResponseTime) {
       threshold = CompareCriteria.convertToNumber(threshold);

package/Server/Utils/Monitor/MonitorAlert.ts

@@ -26,6 +26,7 @@ import CaptureSpan from "../Telemetry/CaptureSpan";
 import DataToProcess from "./DataToProcess";
 import MonitorResourceContextUtil from "./MonitorResourceContext";
 import MonitorTemplateUtil from "./MonitorTemplateUtil";
+import SeriesContextEnricher from "./SeriesContextEnricher";
 import SeriesResourceLinker, {
   SeriesResolvedResourceIds,
 } from "./SeriesResourceLinker";
@@ -534,13 +535,35 @@ export default class MonitorAlert {
               seriesLabels,
             });
 
-          alert.title = MonitorTemplateUtil.processTemplateString({
-            value: criteriaAlert.title,
-            storageMap,
+          /*
+           * Render the criteria's template, then make it say WHICH
+           * series it is about.
+           *
+           * The criteria title is one fixed string shared by every
+           * series on the monitor, so without the enricher a cluster
+           * with fifty saturated pods produced fifty alerts whose
+           * titles were character-for-character identical - and the
+           * alert list, the Slack message and the phone notification
+           * all showed the same undifferentiated line. The enricher is
+           * a no-op when the monitor is not grouped, and when the
+           * rendered text already names the series (because the user
+           * put `{{seriesResourceSuffix}}` or the raw label variables
+           * in their own template).
+           */
+          alert.title = SeriesContextEnricher.enrichTitle({
+            title: MonitorTemplateUtil.processTemplateString({
+              value: criteriaAlert.title,
+              storageMap,
+            }),
+            seriesLabels,
           });
-          alert.description = MonitorTemplateUtil.processTemplateString({
-            value: criteriaAlert.description,
-            storageMap,
+          alert.description = SeriesContextEnricher.enrichDescription({
+            description: MonitorTemplateUtil.processTemplateString({
+              value: criteriaAlert.description,
+              storageMap,
+            }),
+            seriesLabels,
+            monitorType: input.monitor.monitorType,
           });
 
           /*

package/Server/Utils/Monitor/MonitorCriteriaDataExtractor.ts

@@ -9,6 +9,7 @@ import MetricMonitorResponse from "../../../Types/Monitor/MetricMonitor/MetricMo
 import CustomCodeMonitorResponse from "../../../Types/Monitor/CustomCodeMonitor/CustomCodeMonitorResponse";
 import SyntheticMonitorResponse from "../../../Types/Monitor/SyntheticMonitors/SyntheticMonitorResponse";
 import SslMonitorResponse from "../../../Types/Monitor/SSLMonitor/SslMonitorResponse";
+import DatabaseMonitorResponse from "../../../Types/Monitor/DatabaseMonitor/DatabaseMonitorResponse";
 import MonitorStep from "../../../Types/Monitor/MonitorStep";
 import { CriteriaFilter } from "../../../Types/Monitor/CriteriaFilter";
 import AggregatedResult from "../../../Types/BaseDatabase/AggregatedResult";
@@ -142,6 +143,19 @@ export default class MonitorCriteriaDataExtractor {
     return null;
   }
 
+  public static getDatabaseMonitorResponse(
+    dataToProcess: DataToProcess,
+  ): DatabaseMonitorResponse | null {
+    const probeResponse: ProbeMonitorResponse | null =
+      MonitorCriteriaDataExtractor.getProbeMonitorResponse(dataToProcess);
+
+    if (probeResponse?.databaseMonitorResponse) {
+      return probeResponse.databaseMonitorResponse;
+    }
+
+    return null;
+  }
+
   public static extractMetricValues(input: {
     criteriaFilter: CriteriaFilter;
     dataToProcess: DataToProcess;

package/Server/Utils/Monitor/MonitorCriteriaEvaluator.ts

@@ -25,6 +25,7 @@ import DnsMonitorCriteria from "./Criteria/DnsMonitorCriteria";
 import DomainMonitorCriteria from "./Criteria/DomainMonitorCriteria";
 import DnssecMonitorCriteria from "./Criteria/DnssecMonitorCriteria";
 import SqlMonitorCriteria from "./Criteria/SqlMonitorCriteria";
+import DatabaseMonitorCriteria from "./Criteria/DatabaseMonitorCriteria";
 import ExternalStatusPageMonitorCriteria from "./Criteria/ExternalStatusPageMonitorCriteria";
 import MonitorCriteriaMessageBuilder from "./MonitorCriteriaMessageBuilder";
 import MonitorCriteriaDataExtractor from "./MonitorCriteriaDataExtractor";
@@ -48,8 +49,15 @@ import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
 import RequestFailedDetails from "../../../Types/Probe/RequestFailedDetails";
 import IncomingMonitorRequest from "../../../Types/Monitor/IncomingMonitor/IncomingMonitorRequest";
 import SqlMonitorResponse from "../../../Types/Monitor/SqlMonitor/SqlMonitorResponse";
+import DatabaseMonitorResponse, {
+  DatabaseMetricGroupStatus,
+} from "../../../Types/Monitor/DatabaseMonitor/DatabaseMonitorResponse";
 import MonitorType from "../../../Types/Monitor/MonitorType";
-import { CheckOn, CriteriaFilter } from "../../../Types/Monitor/CriteriaFilter";
+import {
+  CheckOn,
+  CriteriaFilter,
+  FilterType,
+} from "../../../Types/Monitor/CriteriaFilter";
 import OneUptimeDate from "../../../Types/Date";
 import { JSONObject } from "../../../Types/JSON";
 import Dictionary from "../../../Types/Dictionary";
@@ -883,6 +891,38 @@ ${contextBlock}
         };
       }
 
+      if (input.monitor.monitorType === MonitorType.Database) {
+        const databaseResponse: DatabaseMonitorResponse | undefined = (
+          input.dataToProcess as ProbeMonitorResponse
+        ).databaseMonitorResponse;
+
+        /*
+         * metrics is exposed raw, keyed by series name, so an expression can
+         * read any of the catalog's series without a variable per metric:
+         * {{metrics['oneuptime.monitor.database.connections.used.percent']}} > 90
+         *
+         * Connection details are deliberately absent - host, username and
+         * password never travel into an expression sandbox.
+         */
+        storageMap = {
+          isOnline: (input.dataToProcess as ProbeMonitorResponse).isOnline,
+          engineVersion: databaseResponse?.engineVersion,
+          connectionError: databaseResponse?.connectionError,
+          collectedGroups: databaseResponse?.collectedGroups,
+          unavailableGroups: (databaseResponse?.unavailableGroups || []).map(
+            (status: DatabaseMetricGroupStatus): JSONObject => {
+              return {
+                group: status.group,
+                reason: status.reason,
+                message: status.message,
+                remediation: status.remediation,
+              };
+            },
+          ),
+          metrics: databaseResponse?.metrics,
+        };
+      }
+
       let expression: string = input.criteriaFilter.value as string;
       expression = VMUtil.replaceValueInPlace(storageMap, expression, false);
 
@@ -1173,6 +1213,19 @@ ${contextBlock}
       }
     }
 
+    if (input.monitor.monitorType === MonitorType.Database) {
+      const databaseMonitorResult: string | null =
+        await DatabaseMonitorCriteria.isMonitorInstanceCriteriaFilterMet({
+          dataToProcess: input.dataToProcess,
+          criteriaFilter: input.criteriaFilter,
+          monitoringInterval: input.monitor.monitoringInterval,
+        });
+
+      if (databaseMonitorResult) {
+        return databaseMonitorResult;
+      }
+    }
+
     if (input.monitor.monitorType === MonitorType.ExternalStatusPage) {
       const externalStatusPageResult: string | null =
         await ExternalStatusPageMonitorCriteria.isMonitorInstanceCriteriaFilterMet(
@@ -2628,10 +2681,73 @@ ${contextBlock}
     return sections.length > 0 ? sections.join("\n") : null;
   }
 
+  /*
+   * Which rows of the Ceph affected-resources table actually BREACHED.
+   *
+   * The blanket `metricValue > 0` below is right for every count- or
+   * level-style criteria ("> 0 active health checks", "> 85 % used"), but
+   * it is exactly backwards for the availability signals: the OSD Down /
+   * OSD Out / Quorum Degraded criteria fire on `< 1` over ceph_osd_up /
+   * ceph_osd_in / ceph_mon_quorum_status, so the ZERO rows are the down
+   * daemons. Dropping them rendered a table of the HEALTHY daemons under
+   * an incident that tells the reader to "check the root cause for the
+   * affected ceph_daemon label".
+   *
+   * Invert only for a criteria that fires when the metric FALLS.
+   * Everything else — every `> 0` health check, PG count, latency and
+   * capacity ratio, and every `= 0` recovery criteria — keeps the
+   * existing `> 0`, worst-highest-first table byte for byte.
+   */
+  private static getCephBreachPredicate(
+    criteriaInstance?: MonitorCriteriaInstance | undefined,
+  ): { matches: (value: number) => boolean; worstIsLowest: boolean } {
+    const metricFilters: Array<CriteriaFilter> = (
+      criteriaInstance?.data?.filters || []
+    ).filter((f: CriteriaFilter) => {
+      return f.checkOn === CheckOn.MetricValue && typeof f.value === "number";
+    });
+
+    const firesWhenMetricFalls: boolean =
+      metricFilters.length > 0 &&
+      metricFilters.every((f: CriteriaFilter) => {
+        return (
+          f.filterType === FilterType.LessThan ||
+          f.filterType === FilterType.LessThanOrEqualTo
+        );
+      });
+
+    if (!firesWhenMetricFalls) {
+      return {
+        matches: (value: number): boolean => {
+          return value > 0;
+        },
+        worstIsLowest: false,
+      };
+    }
+
+    return {
+      matches: (value: number): boolean => {
+        return metricFilters.some((f: CriteriaFilter) => {
+          const threshold: number = f.value as number;
+
+          return f.filterType === FilterType.LessThan
+            ? value < threshold
+            : value <= threshold;
+        });
+      },
+      /*
+       * A criteria that fires when the metric falls ranks the smallest
+       * value worst, so "worst first" is ascending here.
+       */
+      worstIsLowest: true,
+    };
+  }
+
   private static buildCephRootCauseContext(input: {
     dataToProcess: DataToProcess;
     monitorStep: MonitorStep;
     monitor: Monitor;
+    criteriaInstance?: MonitorCriteriaInstance | undefined;
   }): string | null {
     const metricResponse: MetricMonitorResponse =
       input.dataToProcess as MetricMonitorResponse;
@@ -2687,21 +2803,28 @@ ${contextBlock}
 
     if (breakdown && breakdown.affectedResources.length > 0) {
       /*
-       * K8s-parity render: drop zero-value rows, worst (highest)
-       * first, top 10. Note that for availability metrics
-       * (ceph_osd_up / ceph_osd_in / ceph_mon_quorum_status)
-       * zero-valued rows ARE the down resources — those still drive
-       * alerting through the per-series criteria evaluation; this
-       * table is supplementary context only.
+       * Keep the rows that satisfy the criteria that just matched, worst
+       * first, top 10 — NOT simply the non-zero rows. For ceph_osd_up /
+       * ceph_osd_in / ceph_mon_quorum_status the criteria is `< 1`, so
+       * the zero rows are the whole point of the incident.
        */
+      const breach: {
+        matches: (value: number) => boolean;
+        worstIsLowest: boolean;
+      } = MonitorCriteriaEvaluator.getCephBreachPredicate(
+        input.criteriaInstance,
+      );
+
       const sortedResources: Array<CephAffectedResource> = [
         ...breakdown.affectedResources,
       ]
         .filter((r: CephAffectedResource) => {
-          return r.metricValue > 0;
+          return breach.matches(r.metricValue);
         })
         .sort((a: CephAffectedResource, b: CephAffectedResource) => {
-          return b.metricValue - a.metricValue;
+          return breach.worstIsLowest
+            ? a.metricValue - b.metricValue
+            : b.metricValue - a.metricValue;
         });
 
       /*
@@ -2809,10 +2932,17 @@ ${contextBlock}
       lines.push(
         `Recommended actions: Check container logs with \`kubectl logs ${topResource.podName || "<pod-name>"} -c ${topResource.containerName || "<container>"} --previous\` and inspect events with \`kubectl describe pod ${topResource.podName || "<pod-name>"}\`.`,
       );
-    } else if (
-      metricName === "k8s.pod.phase" &&
-      breakdown.attributes["k8s.pod.phase"] === "Pending"
-    ) {
+    } else if (metricName === "k8s.pod.phase") {
+      /*
+       * The second half of this condition used to be
+       * `breakdown.attributes["k8s.pod.phase"] === "Pending"`, which could
+       * never be true. `breakdown.attributes` is the QUERY's attribute map
+       * (MonitorTelemetryMonitor stamps `resource.`-prefixed keys into it),
+       * and no receiver or processor in the shipped agent emits a pod
+       * phase attribute at all — the phase is the metric's VALUE
+       * (1 = Pending ... 5 = Unknown). So this branch never ran and every
+       * pod-phase alert fell through to the generic Kubernetes text.
+       */
       lines.push(`Pods are stuck in Pending phase and unable to be scheduled.`);
       lines.push(
         `Common causes: insufficient CPU/memory resources on nodes, node affinity/taint restrictions preventing scheduling, PersistentVolumeClaim pending, or resource quota exceeded.`,
@@ -2840,7 +2970,18 @@ ${contextBlock}
       );
     } else if (
       metricName === "k8s.node.cpu.utilization" ||
-      (metricName.includes("cpu") && metricName.includes("utilization"))
+      /*
+       * Node-scoped ONLY. The bare substring test used to swallow every
+       * pod- and container-scoped CPU metric — `k8s.pod.cpu.utilization`,
+       * and now `k8s.pod.cpu_limit_utilization` — and answer them with
+       * "Node X is at N% CPU utilization... consider scaling the cluster",
+       * directly contradicting the pod-limit template's own description,
+       * which tells the reader NOT to add replicas, and naming a node for
+       * a pod-scoped alert.
+       */
+      (metricName.startsWith("k8s.node.") &&
+        metricName.includes("cpu") &&
+        metricName.includes("utilization"))
     ) {
       lines.push(`Node CPU utilization has exceeded the configured threshold.`);
       if (topResource.nodeName) {
@@ -2856,7 +2997,10 @@ ${contextBlock}
       );
     } else if (
       metricName === "k8s.node.memory.usage" ||
-      (metricName.includes("memory") && metricName.includes("usage"))
+      // Node-scoped only — see the CPU branch above.
+      (metricName.startsWith("k8s.node.") &&
+        metricName.includes("memory") &&
+        metricName.includes("usage"))
     ) {
       lines.push(
         `Node memory utilization has exceeded the configured threshold.`,