@oneuptime/common 12.0.32 → 12.0.34

package/Server/Utils/Monitor/MonitorCriteriaObservationBuilder.ts

@@ -24,6 +24,15 @@ import ExceptionMonitorResponse from "../../../Types/Monitor/ExceptionMonitor/Ex
 import SnmpMonitorResponse, {
   SnmpOidResponse,
 } from "../../../Types/Monitor/SnmpMonitor/SnmpMonitorResponse";
+import DatabaseMonitorResponse, {
+  DatabaseMetricGroupStatus,
+  DatabaseMetricGroupUnavailableReason,
+} from "../../../Types/Monitor/DatabaseMonitor/DatabaseMonitorResponse";
+import {
+  DatabaseMetricDefinition,
+  getDatabaseMetricByMetricType,
+} from "../../../Types/Monitor/DatabaseMetricCatalog";
+import MonitorMetricType from "../../../Types/Monitor/MonitorMetricType";
 import MonitorCriteriaMessageFormatter from "./MonitorCriteriaMessageFormatter";
 import MonitorCriteriaDataExtractor from "./MonitorCriteriaDataExtractor";
 import MonitorCriteriaExpectationBuilder from "./MonitorCriteriaExpectationBuilder";
@@ -183,6 +192,10 @@ export default class MonitorCriteriaObservationBuilder {
         return MonitorCriteriaObservationBuilder.describeSnmpIsOnlineObservation(
           input,
         );
+      case CheckOn.SnmpWalkIsSucceeding:
+        return MonitorCriteriaObservationBuilder.describeSnmpWalkIsSucceedingObservation(
+          input,
+        );
       case CheckOn.SnmpResponseTime:
         return MonitorCriteriaObservationBuilder.describeSnmpResponseTimeObservation(
           input,
@@ -195,6 +208,18 @@ export default class MonitorCriteriaObservationBuilder {
         return MonitorCriteriaObservationBuilder.describeSnmpOidValueObservation(
           input,
         );
+      case CheckOn.DatabaseIsOnline:
+        return MonitorCriteriaObservationBuilder.describeDatabaseIsOnlineObservation(
+          input,
+        );
+      case CheckOn.DatabaseMetric:
+        return MonitorCriteriaObservationBuilder.describeDatabaseMetricObservation(
+          input,
+        );
+      case CheckOn.DatabaseCollectionError:
+        return MonitorCriteriaObservationBuilder.describeDatabaseCollectionErrorObservation(
+          input,
+        );
       default:
         return null;
     }
@@ -1374,6 +1399,24 @@ export default class MonitorCriteriaObservationBuilder {
     dataToProcess: DataToProcess;
     monitorStep: MonitorStep;
   }): string | undefined {
+    /*
+     * Database Health thresholds are typed in the metric's own unit, which
+     * only the catalog knows - without this the expectation clause reads
+     * "greater than 90" next to an observation of "95.0%".
+     */
+    if (input.criteriaFilter.checkOn === CheckOn.DatabaseMetric) {
+      const metricType: MonitorMetricType | undefined =
+        input.criteriaFilter.databaseMonitorOptions?.metricType;
+
+      const definition: DatabaseMetricDefinition | null = metricType
+        ? getDatabaseMetricByMetricType(metricType)
+        : null;
+
+      return MonitorCriteriaObservationBuilder.normalizeDisplayUnit(
+        definition?.unit,
+      );
+    }
+
     if (input.criteriaFilter.checkOn !== CheckOn.MetricValue) {
       return undefined;
     }
@@ -1429,25 +1472,58 @@ export default class MonitorCriteriaObservationBuilder {
     return { snmpResponse, oid, oidResponse };
   }
 
+  /*
+   * "Is Online" is device reachability - ping OR a successful walk - which
+   * is the top-level isOnline the walk pipeline stamps, not the walk's own
+   * verdict. A device with no credentials is only pinged, so reading the
+   * walk here would describe every such device as "SNMP response was
+   * unavailable" on a poll that found it perfectly reachable.
+   */
   private static describeSnmpIsOnlineObservation(input: {
     dataToProcess: DataToProcess;
+  }): string | null {
+    const probeResponse: ProbeMonitorResponse | null =
+      MonitorCriteriaDataExtractor.getProbeMonitorResponse(input.dataToProcess);
+
+    if (!probeResponse || probeResponse.isOnline === undefined) {
+      return "Device reachability was not recorded.";
+    }
+
+    if (probeResponse.isOnline) {
+      return "Device is reachable (answered ping or SNMP).";
+    }
+
+    if (probeResponse.failureCause) {
+      return `Device is unreachable by ping and SNMP: ${probeResponse.failureCause}`;
+    }
+
+    return "Device is unreachable by ping and SNMP.";
+  }
+
+  /*
+   * The walk itself, separately from reachability. No walk on this poll is
+   * a distinct, non-failure state: it is what a ping-only device looks like
+   * every cycle, and the criterion is not evaluated for it.
+   */
+  private static describeSnmpWalkIsSucceedingObservation(input: {
+    dataToProcess: DataToProcess;
   }): string | null {
     const snmpResponse: SnmpMonitorResponse | null =
       MonitorCriteriaObservationBuilder.getSnmpResponse(input);
 
     if (!snmpResponse) {
-      return "SNMP response was unavailable.";
+      return "No SNMP walk ran on this poll (the device is only pinged).";
     }
 
     if (snmpResponse.isOnline) {
-      return "SNMP device is online.";
+      return "SNMP walk succeeded.";
     }
 
     if (snmpResponse.failureCause) {
-      return `SNMP device is offline: ${snmpResponse.failureCause}`;
+      return `SNMP walk failed: ${snmpResponse.failureCause}`;
     }
 
-    return "SNMP device is offline.";
+    return "SNMP walk failed.";
   }
 
   private static describeSnmpResponseTimeObservation(input: {
@@ -1541,4 +1617,179 @@ export default class MonitorCriteriaObservationBuilder {
 
     return String(value);
   }
+
+  private static describeDatabaseIsOnlineObservation(input: {
+    dataToProcess: DataToProcess;
+  }): string | null {
+    const probeResponse: ProbeMonitorResponse | null =
+      MonitorCriteriaDataExtractor.getProbeMonitorResponse(input.dataToProcess);
+
+    if (!probeResponse) {
+      return null;
+    }
+
+    const databaseResponse: DatabaseMonitorResponse | null =
+      MonitorCriteriaDataExtractor.getDatabaseMonitorResponse(
+        input.dataToProcess,
+      );
+
+    if (databaseResponse?.isOnline ?? probeResponse.isOnline) {
+      return "The database was reachable.";
+    }
+
+    /*
+     * Both of these are sanitized by the collector before they leave the
+     * probe - the connection string, the password and the login never reach
+     * an operator-facing sentence.
+     */
+    const failureCause: string =
+      databaseResponse?.connectionError ||
+      databaseResponse?.failureCause ||
+      probeResponse.failureCause ||
+      "";
+
+    if (failureCause) {
+      return `The database was not reachable: ${failureCause}`;
+    }
+
+    return "The database was not reachable.";
+  }
+
+  private static describeDatabaseMetricObservation(input: {
+    criteriaFilter: CriteriaFilter;
+    dataToProcess: DataToProcess;
+  }): string | null {
+    const metricType: MonitorMetricType | undefined =
+      input.criteriaFilter.databaseMonitorOptions?.metricType;
+
+    if (!metricType) {
+      return "No database metric is configured on this criteria. Pick one from the metric dropdown in the criteria editor.";
+    }
+
+    const definition: DatabaseMetricDefinition | null =
+      getDatabaseMetricByMetricType(metricType);
+
+    if (!definition) {
+      return `${metricType} is not a metric the Database Health monitor collects.`;
+    }
+
+    const databaseResponse: DatabaseMonitorResponse | null =
+      MonitorCriteriaDataExtractor.getDatabaseMonitorResponse(
+        input.dataToProcess,
+      );
+
+    const value: number | undefined = databaseResponse?.metrics[metricType];
+
+    /*
+     * An absent metric is the normal outcome of partial collection, and the
+     * operator's next action depends entirely on why: a grant to run, an
+     * engine that has no such counter, or a group that timed out. Saying so -
+     * rather than declining to describe the check at all - is what turns a
+     * blank chart into something fixable.
+     */
+    if (value === undefined || value === null) {
+      return MonitorCriteriaObservationBuilder.describeUncollectedDatabaseMetric(
+        {
+          definition: definition,
+          databaseResponse: databaseResponse,
+        },
+      );
+    }
+
+    return `${definition.friendlyName} was ${MonitorCriteriaObservationBuilder.formatDatabaseMetricValue(
+      value,
+      definition.unit,
+    )}.`;
+  }
+
+  private static describeUncollectedDatabaseMetric(input: {
+    definition: DatabaseMetricDefinition;
+    databaseResponse: DatabaseMonitorResponse | null;
+  }): string {
+    const status: DatabaseMetricGroupStatus | undefined =
+      input.databaseResponse?.unavailableGroups?.find(
+        (groupStatus: DatabaseMetricGroupStatus) => {
+          return groupStatus.group === input.definition.group;
+        },
+      );
+
+    if (!status) {
+      return `${input.definition.friendlyName} was not collected on this check.`;
+    }
+
+    let message: string = `${input.definition.friendlyName} was not collected on this check: ${MonitorCriteriaObservationBuilder.describeDatabaseGroupUnavailableReason(
+      status.reason,
+    )}`;
+
+    if (status.remediation) {
+      message += ` (${status.remediation})`;
+    }
+
+    return `${message}.`;
+  }
+
+  private static describeDatabaseGroupUnavailableReason(
+    reason: DatabaseMetricGroupUnavailableReason,
+  ): string {
+    switch (reason) {
+      case DatabaseMetricGroupUnavailableReason.MissingPermission:
+        return "the monitoring login is missing a grant";
+      case DatabaseMetricGroupUnavailableReason.NotSupportedByEngine:
+        return "the connected engine does not report it";
+      case DatabaseMetricGroupUnavailableReason.Timeout:
+        return "its metric group timed out";
+      default:
+        return "its metric group could not be collected";
+    }
+  }
+
+  private static formatDatabaseMetricValue(
+    value: number,
+    unit: string,
+  ): string {
+    if (unit === "%") {
+      return (
+        MonitorCriteriaMessageFormatter.formatPercentage(value) ?? String(value)
+      );
+    }
+
+    if (unit === "bytes") {
+      return (
+        MonitorCriteriaMessageFormatter.formatBytes(value) ?? String(value)
+      );
+    }
+
+    const formatted: string =
+      MonitorCriteriaMessageFormatter.formatNumber(value, {
+        maximumFractionDigits: 2,
+      }) ?? String(value);
+
+    return unit ? `${formatted} ${unit}` : formatted;
+  }
+
+  private static describeDatabaseCollectionErrorObservation(input: {
+    dataToProcess: DataToProcess;
+  }): string | null {
+    const databaseResponse: DatabaseMonitorResponse | null =
+      MonitorCriteriaDataExtractor.getDatabaseMonitorResponse(
+        input.dataToProcess,
+      );
+
+    if (!databaseResponse) {
+      return null;
+    }
+
+    const statuses: Array<DatabaseMetricGroupStatus> =
+      databaseResponse.unavailableGroups || [];
+
+    if (!statuses.length) {
+      return "All metric groups were collected.";
+    }
+
+    return `Collection issues: ${statuses
+      .map((status: DatabaseMetricGroupStatus) => {
+        return `${status.group}: ${status.message}`;
+      })
+      .join("; ")}.`;
+  }
 }

package/Server/Utils/Monitor/MonitorIncident.ts

@@ -27,6 +27,7 @@ import logger, { LogAttributes } from "../Logger";
 import CaptureSpan from "../Telemetry/CaptureSpan";
 import DataToProcess from "./DataToProcess";
 import MonitorTemplateUtil from "./MonitorTemplateUtil";
+import SeriesContextEnricher from "./SeriesContextEnricher";
 import MonitorDependencySuppression, {
   DependencySuppressionResult,
 } from "./MonitorDependencySuppression";
@@ -597,13 +598,28 @@ export default class MonitorIncident {
               seriesLabels,
             });
 
-          incident.title = MonitorTemplateUtil.processTemplateString({
-            value: criteriaIncident.title,
-            storageMap,
+          /*
+           * Render the criteria's template, then make it say WHICH
+           * series it is about. Mirrors MonitorAlert exactly - the two
+           * must agree, or the same breach reads differently depending
+           * on whether the criteria was configured to raise an alert or
+           * an incident. See SeriesContextEnricher for why this is not
+           * done inside the template itself.
+           */
+          incident.title = SeriesContextEnricher.enrichTitle({
+            title: MonitorTemplateUtil.processTemplateString({
+              value: criteriaIncident.title,
+              storageMap,
+            }),
+            seriesLabels,
           });
-          incident.description = MonitorTemplateUtil.processTemplateString({
-            value: criteriaIncident.description,
-            storageMap,
+          incident.description = SeriesContextEnricher.enrichDescription({
+            description: MonitorTemplateUtil.processTemplateString({
+              value: criteriaIncident.description,
+              storageMap,
+            }),
+            seriesLabels,
+            monitorType: input.monitor.monitorType,
           });
 
           /*

package/Server/Utils/Monitor/MonitorMetricUtil.ts

@@ -19,6 +19,8 @@ import BasicInfrastructureMetrics, {
 import Dictionary from "../../../Types/Dictionary";
 import { JSONObject } from "../../../Types/JSON";
 import CapturedMetric from "../../../Types/Monitor/CustomCodeMonitor/CapturedMetric";
+import { getAllDatabaseMetrics } from "../../../Types/Monitor/DatabaseMetricCatalog";
+import DatabaseMonitorResponse from "../../../Types/Monitor/DatabaseMonitor/DatabaseMonitorResponse";
 import HttpPhaseTimings from "../../../Types/Monitor/HttpPhaseTimings";
 import MonitorMetricType from "../../../Types/Monitor/MonitorMetricType";
 import PingMonitorResponse from "../../../Types/Monitor/PingMonitor/PingMonitorResponse";
@@ -956,6 +958,45 @@ export default class MonitorMetricUtil {
       metricNameServiceNameMap[MonitorMetricType.ResponseTime] = metricType;
     }
 
+    const databaseResponse: DatabaseMonitorResponse | undefined = (
+      data.dataToProcess as ProbeMonitorResponse
+    ).databaseMonitorResponse;
+
+    if (databaseResponse) {
+      const extraAttributes: JSONObject = {
+        probeId: (
+          data.dataToProcess as ProbeMonitorResponse
+        ).probeId.toString(),
+      };
+
+      /*
+       * Driven off the catalog rather than a list here so a metric is wired
+       * end to end the moment it is added there. Every database series is
+       * single-valued per check, so there is no per-series fan-out to cap
+       * and probeId is the only attribute worth carrying.
+       *
+       * A metric the engine could not report, or whose group was skipped,
+       * is simply absent from the map and pushMonitorMetric writes no row
+       * for it. Absent must never become zero: a replication lag that was
+       * not measured and a replication lag of zero mean opposite things.
+       */
+      for (const definition of getAllDatabaseMetrics()) {
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: definition.metricType,
+          value: databaseResponse.metrics[definition.metricType],
+          description: definition.description,
+          unit: definition.unit,
+          extraAttributes: extraAttributes,
+          metricRows: metricRows,
+          metricNameServiceNameMap: metricNameServiceNameMap,
+        });
+      }
+    }
+
     const snmpInterfaces: Array<SnmpInterface> | undefined = (
       data.dataToProcess as ProbeMonitorResponse
     ).snmpResponse?.interfaces;

package/Server/Utils/Monitor/MonitorTemplateUtil.ts

@@ -24,11 +24,16 @@ import DnsMonitorResponse, {
 } from "../../../Types/Monitor/DnsMonitor/DnsMonitorResponse";
 import DomainMonitorResponse from "../../../Types/Monitor/DomainMonitor/DomainMonitorResponse";
 import DnssecMonitorResponse from "../../../Types/Monitor/DnssecMonitor/DnssecMonitorResponse";
+import DatabaseMonitorResponse, {
+  DatabaseMetricGroupStatus,
+} from "../../../Types/Monitor/DatabaseMonitor/DatabaseMonitorResponse";
 import ExternalStatusPageMonitorResponse, {
   ExternalStatusPageComponentStatus,
 } from "../../../Types/Monitor/ExternalStatusPageMonitor/ExternalStatusPageMonitorResponse";
 import MetricMonitorResponse from "../../../Types/Monitor/MetricMonitor/MetricMonitorResponse";
 import Typeof from "../../../Types/Typeof";
+import SeriesDebugHints from "../../../Types/Monitor/SeriesContext/SeriesDebugHints";
+import SeriesLabelDisplay from "../../../Types/Monitor/SeriesContext/SeriesLabelDisplay";
 import VMUtil from "../VM/VMAPI";
 import DataToProcess from "./DataToProcess";
 import logger from "../Logger";
@@ -464,6 +469,46 @@ export default class MonitorTemplateUtil {
         } as JSONObject;
       }
 
+      if (data.monitorType === MonitorType.Database) {
+        const databaseResponse: DatabaseMonitorResponse | undefined = (
+          data.dataToProcess as ProbeMonitorResponse
+        ).databaseMonitorResponse;
+
+        const unavailableGroups: Array<DatabaseMetricGroupStatus> =
+          databaseResponse?.unavailableGroups || [];
+
+        storageMap = {
+          isOnline: (data.dataToProcess as ProbeMonitorResponse).isOnline,
+          responseTimeInMs: databaseResponse?.responseTimeInMs,
+          failureCause: databaseResponse?.failureCause,
+          connectionError: databaseResponse?.connectionError,
+          engineVersion: databaseResponse?.engineVersion,
+          collectedGroups: databaseResponse?.collectedGroups || [],
+          unavailableGroups: unavailableGroups.map(
+            (status: DatabaseMetricGroupStatus) => {
+              return {
+                group: status.group,
+                reason: status.reason,
+                message: status.message,
+                remediation: status.remediation,
+              };
+            },
+          ),
+          /*
+           * Partial collection is the normal state of this monitor, so the
+           * ready-made sentence matters more here than the array does - an
+           * incident title has room for one line, not a loop.
+           */
+          collectionIssueSummary: unavailableGroups
+            .map((status: DatabaseMetricGroupStatus) => {
+              return `${status.group}: ${status.message}`;
+            })
+            .join("; "),
+          // A metric absent from this map was not collected on this check.
+          metrics: databaseResponse?.metrics || {},
+        } as JSONObject;
+      }
+
       if (
         data.monitorType === MonitorType.Metrics ||
         data.monitorType === MonitorType.Kubernetes ||
@@ -584,6 +629,33 @@ export default class MonitorTemplateUtil {
       storageMap["seriesLabels"] = data.seriesLabels;
     }
 
+    /*
+     * Ready-made renderings of the series identity.
+     *
+     * These are set unconditionally - to "" when the monitor is not
+     * grouped, or when its labels carry no usable value - and that is
+     * the whole point. `VMUtil.replaceValueInPlace` leaves a placeholder
+     * it cannot resolve in the output verbatim, so a title written as
+     * `"Pod CPU high{{seriesResourceSuffix}}"` would otherwise render
+     * with the braces still in it on any monitor without a group-by.
+     * Always defining them makes the variables safe to use in a shipped
+     * template that has to work for grouped and ungrouped monitors
+     * alike.
+     */
+    storageMap["seriesResourceSuffix"] = SeriesLabelDisplay.buildTitleSuffix(
+      data.seriesLabels,
+    );
+    storageMap["seriesResourceSummary"] = SeriesLabelDisplay.buildInlineSummary(
+      data.seriesLabels,
+    );
+    storageMap["seriesResourceBlock"] = SeriesLabelDisplay.buildMarkdownBlock(
+      data.seriesLabels,
+    );
+    storageMap["seriesDebugCommands"] = SeriesDebugHints.buildMarkdownBlock({
+      monitorType: data.monitorType,
+      seriesLabels: data.seriesLabels,
+    });
+
     /*
      * Monitor identity fields. Always exposed (when a monitor is provided),
      * independent of monitorType, so templates like `{{monitorName}}` work