@oneuptime/common 10.0.70 → 10.0.72

package/Server/Utils/Monitor/MonitorMetricUtil.ts

@@ -10,7 +10,9 @@ import {
   ServiceType,
 } from "../../../Models/AnalyticsModels/Metric";
 import MetricType from "../../../Models/DatabaseModels/MetricType";
-import BasicInfrastructureMetrics from "../../../Types/Infrastructure/BasicMetrics";
+import BasicInfrastructureMetrics, {
+  NetworkInterfaceMetrics,
+} from "../../../Types/Infrastructure/BasicMetrics";
 import Dictionary from "../../../Types/Dictionary";
 import { JSONObject } from "../../../Types/JSON";
 import CapturedMetric from "../../../Types/Monitor/CustomCodeMonitor/CapturedMetric";
@@ -159,6 +161,73 @@ export default class MonitorMetricUtil {
     } as JSONObject;
   }
 
+  /*
+   * Helper that collapses the "build attributes → build row → push → register MetricType"
+   * pattern used repeatedly below. Silently skips emission when value is missing/non-finite
+   * so callers can pass optional agent fields directly.
+   */
+  private static async pushMonitorMetric(data: {
+    projectId: ObjectID;
+    monitorId: ObjectID;
+    monitorName: string | undefined;
+    probeName: string | undefined;
+    metricName: string;
+    value: number | null | undefined;
+    description: string;
+    unit: string;
+    extraAttributes?: JSONObject;
+    metricPointType?: MetricPointType;
+    metricRows: Array<JSONObject>;
+    metricNameServiceNameMap: Dictionary<MetricType>;
+  }): Promise<void> {
+    if (
+      data.value === undefined ||
+      data.value === null ||
+      typeof data.value !== "number" ||
+      !isFinite(data.value)
+    ) {
+      return;
+    }
+
+    const attributeInput: {
+      monitorId: ObjectID;
+      projectId: ObjectID;
+      monitorName?: string | undefined;
+      probeName?: string | undefined;
+      extraAttributes?: JSONObject;
+    } = {
+      monitorId: data.monitorId,
+      projectId: data.projectId,
+      monitorName: data.monitorName,
+      probeName: data.probeName,
+    };
+
+    if (data.extraAttributes) {
+      attributeInput.extraAttributes = data.extraAttributes;
+    }
+
+    const attributes: JSONObject =
+      this.buildMonitorMetricAttributes(attributeInput);
+
+    const metricRow: JSONObject = await this.buildMonitorMetricRow({
+      projectId: data.projectId,
+      monitorId: data.monitorId,
+      metricName: data.metricName,
+      value: data.value,
+      attributes: attributes,
+      metricPointType: data.metricPointType || MetricPointType.Sum,
+    });
+
+    data.metricRows.push(metricRow);
+
+    const metricType: MetricType = new MetricType();
+    metricType.name = data.metricName;
+    metricType.description = data.description;
+    metricType.unit = data.unit;
+
+    data.metricNameServiceNameMap[data.metricName] = metricType;
+  }
+
   @CaptureSpan()
   public static async saveMonitorMetrics(data: {
     monitorId: ObjectID;
@@ -333,6 +402,359 @@ export default class MonitorMetricUtil {
           metricNameServiceNameMap[MonitorMetricType.DiskUsagePercent] =
             metricType;
         }
+
+        // Per-disk I/O counters (cumulative since boot).
+        for (const diskMetric of basicMetrics.diskMetrics) {
+          const diskAttrs: JSONObject = {};
+          if (diskMetric.diskPath) {
+            diskAttrs["diskPath"] = diskMetric.diskPath;
+          }
+          if (diskMetric.device) {
+            diskAttrs["device"] = diskMetric.device;
+          }
+          if (diskMetric.fstype) {
+            diskAttrs["fstype"] = diskMetric.fstype;
+          }
+
+          await this.pushMonitorMetric({
+            projectId: data.projectId,
+            monitorId: data.monitorId,
+            monitorName: data.monitorName,
+            probeName: data.probeName,
+            metricName: MonitorMetricType.DiskReadBytesTotal,
+            value: diskMetric.readBytes,
+            description: "Total bytes read from disk since boot",
+            unit: "bytes",
+            extraAttributes: diskAttrs,
+            metricRows,
+            metricNameServiceNameMap,
+          });
+
+          await this.pushMonitorMetric({
+            projectId: data.projectId,
+            monitorId: data.monitorId,
+            monitorName: data.monitorName,
+            probeName: data.probeName,
+            metricName: MonitorMetricType.DiskWriteBytesTotal,
+            value: diskMetric.writeBytes,
+            description: "Total bytes written to disk since boot",
+            unit: "bytes",
+            extraAttributes: diskAttrs,
+            metricRows,
+            metricNameServiceNameMap,
+          });
+
+          await this.pushMonitorMetric({
+            projectId: data.projectId,
+            monitorId: data.monitorId,
+            monitorName: data.monitorName,
+            probeName: data.probeName,
+            metricName: MonitorMetricType.DiskReadOpsTotal,
+            value: diskMetric.readCount,
+            description: "Total disk read operations since boot",
+            unit: "ops",
+            extraAttributes: diskAttrs,
+            metricRows,
+            metricNameServiceNameMap,
+          });
+
+          await this.pushMonitorMetric({
+            projectId: data.projectId,
+            monitorId: data.monitorId,
+            monitorName: data.monitorName,
+            probeName: data.probeName,
+            metricName: MonitorMetricType.DiskWriteOpsTotal,
+            value: diskMetric.writeCount,
+            description: "Total disk write operations since boot",
+            unit: "ops",
+            extraAttributes: diskAttrs,
+            metricRows,
+            metricNameServiceNameMap,
+          });
+        }
+      }
+
+      // Load average (1/5/15 min). Emitted only when the agent provides it.
+      if (basicMetrics.loadMetrics) {
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.LoadAverage1Min,
+          value: basicMetrics.loadMetrics.load1,
+          description: "1-minute load average",
+          unit: "",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.LoadAverage5Min,
+          value: basicMetrics.loadMetrics.load5,
+          description: "5-minute load average",
+          unit: "",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.LoadAverage15Min,
+          value: basicMetrics.loadMetrics.load15,
+          description: "15-minute load average",
+          unit: "",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+      }
+
+      // Memory extras: swap + available.
+      if (basicMetrics.memoryMetrics) {
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.SwapUsagePercent,
+          value: basicMetrics.memoryMetrics.swapPercentUsed,
+          description: "Swap memory usage percentage",
+          unit: "%",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.MemoryAvailableBytes,
+          value: basicMetrics.memoryMetrics.available,
+          description: "Memory available to new allocations",
+          unit: "bytes",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+      }
+
+      // CPU time breakdown — drills down into what the CPU was doing.
+      if (basicMetrics.cpuMetrics) {
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.CPUTimeUserPercent,
+          value: basicMetrics.cpuMetrics.timeUserPercent,
+          description: "CPU time spent in user space",
+          unit: "%",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.CPUTimeSystemPercent,
+          value: basicMetrics.cpuMetrics.timeSystemPercent,
+          description: "CPU time spent in kernel space",
+          unit: "%",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.CPUTimeIoWaitPercent,
+          value: basicMetrics.cpuMetrics.timeIoWaitPercent,
+          description: "CPU time spent waiting on I/O",
+          unit: "%",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.CPUTimeIdlePercent,
+          value: basicMetrics.cpuMetrics.timeIdlePercent,
+          description: "CPU time spent idle",
+          unit: "%",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.CPUTimeStealPercent,
+          value: basicMetrics.cpuMetrics.timeStealPercent,
+          description: "CPU time stolen by the hypervisor",
+          unit: "%",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+      }
+
+      // Network counters — both per-interface and aggregate.
+      if (basicMetrics.networkMetrics) {
+        const net: typeof basicMetrics.networkMetrics =
+          basicMetrics.networkMetrics;
+
+        for (const iface of net.interfaces || []) {
+          const ifaceAttrs: JSONObject = {
+            interfaceName: (iface as NetworkInterfaceMetrics).interfaceName,
+          };
+
+          await this.pushMonitorMetric({
+            projectId: data.projectId,
+            monitorId: data.monitorId,
+            monitorName: data.monitorName,
+            probeName: data.probeName,
+            metricName: MonitorMetricType.NetworkBytesReceivedTotal,
+            value: iface.bytesReceived,
+            description: "Network bytes received since boot (per-interface)",
+            unit: "bytes",
+            extraAttributes: ifaceAttrs,
+            metricRows,
+            metricNameServiceNameMap,
+          });
+          await this.pushMonitorMetric({
+            projectId: data.projectId,
+            monitorId: data.monitorId,
+            monitorName: data.monitorName,
+            probeName: data.probeName,
+            metricName: MonitorMetricType.NetworkBytesSentTotal,
+            value: iface.bytesSent,
+            description: "Network bytes sent since boot (per-interface)",
+            unit: "bytes",
+            extraAttributes: ifaceAttrs,
+            metricRows,
+            metricNameServiceNameMap,
+          });
+          await this.pushMonitorMetric({
+            projectId: data.projectId,
+            monitorId: data.monitorId,
+            monitorName: data.monitorName,
+            probeName: data.probeName,
+            metricName: MonitorMetricType.NetworkPacketsReceivedTotal,
+            value: iface.packetsReceived,
+            description: "Network packets received since boot (per-interface)",
+            unit: "packets",
+            extraAttributes: ifaceAttrs,
+            metricRows,
+            metricNameServiceNameMap,
+          });
+          await this.pushMonitorMetric({
+            projectId: data.projectId,
+            monitorId: data.monitorId,
+            monitorName: data.monitorName,
+            probeName: data.probeName,
+            metricName: MonitorMetricType.NetworkPacketsSentTotal,
+            value: iface.packetsSent,
+            description: "Network packets sent since boot (per-interface)",
+            unit: "packets",
+            extraAttributes: ifaceAttrs,
+            metricRows,
+            metricNameServiceNameMap,
+          });
+          await this.pushMonitorMetric({
+            projectId: data.projectId,
+            monitorId: data.monitorId,
+            monitorName: data.monitorName,
+            probeName: data.probeName,
+            metricName: MonitorMetricType.NetworkErrorsIn,
+            value: iface.errorsIn,
+            description: "Network receive errors since boot (per-interface)",
+            unit: "errors",
+            extraAttributes: ifaceAttrs,
+            metricRows,
+            metricNameServiceNameMap,
+          });
+          await this.pushMonitorMetric({
+            projectId: data.projectId,
+            monitorId: data.monitorId,
+            monitorName: data.monitorName,
+            probeName: data.probeName,
+            metricName: MonitorMetricType.NetworkErrorsOut,
+            value: iface.errorsOut,
+            description: "Network transmit errors since boot (per-interface)",
+            unit: "errors",
+            extraAttributes: ifaceAttrs,
+            metricRows,
+            metricNameServiceNameMap,
+          });
+        }
+
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.NetworkConnectionsEstablished,
+          value: net.connectionsEstablished,
+          description: "Count of ESTABLISHED network connections",
+          unit: "connections",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.NetworkConnectionsListen,
+          value: net.connectionsListen,
+          description: "Count of LISTENing network sockets",
+          unit: "connections",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+      }
+
+      // Host uptime.
+      if (basicMetrics.hostMetrics) {
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.HostUptimeSeconds,
+          value: basicMetrics.hostMetrics.uptimeSeconds,
+          description: "Host uptime in seconds",
+          unit: "seconds",
+          metricRows,
+          metricNameServiceNameMap,
+        });
+      }
+
+      // Process count — simple scalar derived from the processes array.
+      const serverProcesses: Array<unknown> | undefined = (
+        data.dataToProcess as ServerMonitorResponse
+      ).processes;
+      if (serverProcesses && serverProcesses.length >= 0) {
+        await this.pushMonitorMetric({
+          projectId: data.projectId,
+          monitorId: data.monitorId,
+          monitorName: data.monitorName,
+          probeName: data.probeName,
+          metricName: MonitorMetricType.ProcessCountTotal,
+          value: serverProcesses.length,
+          description: "Total running processes on the server",
+          unit: "processes",
+          metricRows,
+          metricNameServiceNameMap,
+        });
       }
     }
 

package/Server/Utils/Monitor/MonitorResource.ts

@@ -709,6 +709,7 @@ export default class MonitorResourceUtil {
         props: {
           telemetryQuery: telemetryQuery,
         },
+        matchesPerSeries: response.perSeriesMatches,
       });
 
       await MonitorAlert.criteriaMetCreateAlertsAndUpdateMonitorStatus({
@@ -721,6 +722,7 @@ export default class MonitorResourceUtil {
         props: {
           telemetryQuery: telemetryQuery,
         },
+        matchesPerSeries: response.perSeriesMatches,
       });
     } else if (
       !response.criteriaMetId &&

package/Server/Utils/Monitor/MonitorTemplateUtil.ts

@@ -1,4 +1,5 @@
 import MonitorType from "../../../Types/Monitor/MonitorType";
+import Monitor from "../../../Models/DatabaseModels/Monitor";
 import { JSONObject } from "../../../Types/JSON";
 import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
 import IncomingMonitorRequest from "../../../Types/Monitor/IncomingMonitor/IncomingMonitorRequest";
@@ -21,6 +22,7 @@ import DomainMonitorResponse from "../../../Types/Monitor/DomainMonitor/DomainMo
 import ExternalStatusPageMonitorResponse, {
   ExternalStatusPageComponentStatus,
 } from "../../../Types/Monitor/ExternalStatusPageMonitor/ExternalStatusPageMonitorResponse";
+import MetricMonitorResponse from "../../../Types/Monitor/MetricMonitor/MetricMonitorResponse";
 import Typeof from "../../../Types/Typeof";
 import VMUtil from "../VM/VMAPI";
 import DataToProcess from "./DataToProcess";
@@ -37,6 +39,21 @@ export default class MonitorTemplateUtil {
   public static buildTemplateStorageMap(data: {
     monitorType: MonitorType;
     dataToProcess: DataToProcess;
+    /**
+     * The monitor that fired this criterion. Used to expose identity
+     * fields (`{{monitorName}}`, `{{monitorId}}`, etc.) to incident
+     * and alert title/description templates. Optional for backwards
+     * compatibility with existing callers.
+     */
+    monitor?: Monitor | undefined;
+    /**
+     * When set, the attribute values identifying the specific series
+     * this template is being rendered for. Each label is exposed to
+     * the template under its own key (so `{{host.name}}` works) and
+     * also collected under a `seriesLabels` object for iteration.
+     * Only populated when a metric monitor fires per-series.
+     */
+    seriesLabels?: JSONObject | undefined;
   }): JSONObject {
     let storageMap: JSONObject = {};
 
@@ -315,6 +332,31 @@ export default class MonitorTemplateUtil {
         } as JSONObject;
       }
 
+      if (
+        data.monitorType === MonitorType.Metrics ||
+        data.monitorType === MonitorType.Kubernetes ||
+        data.monitorType === MonitorType.Docker
+      ) {
+        const metricResponse: MetricMonitorResponse =
+          data.dataToProcess as MetricMonitorResponse;
+
+        const queryConfigs: Array<unknown> =
+          metricResponse.metricViewConfig?.queryConfigs || [];
+
+        const firstQuery: unknown = queryConfigs[0];
+        const metricName: string | undefined = (
+          firstQuery as
+            | {
+                metricQueryData?: { filterData?: { metricName?: string } };
+              }
+            | undefined
+        )?.metricQueryData?.filterData?.metricName;
+
+        storageMap = {
+          metricName: metricName || "",
+        } as JSONObject;
+      }
+
       if (data.monitorType === MonitorType.ExternalStatusPage) {
         const externalStatusPageResponse:
           | ExternalStatusPageMonitorResponse
@@ -347,6 +389,63 @@ export default class MonitorTemplateUtil {
       logger.error(err);
     }
 
+    /*
+     * Fold series labels onto the storage map so templates like
+     * `{{host.name}}` or `{{resource.k8s.container.name}}` resolve at
+     * render time. The template engine walks dotted paths as nested
+     * property access (`host` → `.name`), so for each dotted label
+     * key we build up a nested object rather than storing the flat
+     * key. Also expose the full label map under `seriesLabels` for
+     * iteration-style templates.
+     */
+    if (data.seriesLabels && Object.keys(data.seriesLabels).length > 0) {
+      for (const key of Object.keys(data.seriesLabels)) {
+        const value: unknown = data.seriesLabels[key];
+        if (value === undefined || value === null) {
+          continue;
+        }
+        const parts: Array<string> = key.split(".");
+        let cursor: JSONObject = storageMap;
+        for (let i: number = 0; i < parts.length - 1; i++) {
+          const part: string = parts[i]!;
+          const existing: unknown = cursor[part];
+          if (
+            !existing ||
+            typeof existing !== "object" ||
+            Array.isArray(existing)
+          ) {
+            cursor[part] = {};
+          }
+          cursor = cursor[part] as JSONObject;
+        }
+        cursor[parts[parts.length - 1]!] = value as JSONObject[string];
+      }
+      storageMap["seriesLabels"] = data.seriesLabels;
+    }
+
+    /*
+     * Monitor identity fields. Always exposed (when a monitor is provided),
+     * independent of monitorType, so templates like `{{monitorName}}` work
+     * uniformly across Server/VM, Probe, Synthetic, Metric monitors, etc.
+     */
+    if (data.monitor) {
+      if (data.monitor.name) {
+        storageMap["monitorName"] = data.monitor.name;
+      }
+      if (data.monitor.id) {
+        storageMap["monitorId"] = data.monitor.id.toString();
+      }
+      if (data.monitor.description) {
+        storageMap["monitorDescription"] = data.monitor.description;
+      }
+      if (data.monitor.slug) {
+        storageMap["monitorSlug"] = data.monitor.slug;
+      }
+      if (data.monitor.monitorType) {
+        storageMap["monitorType"] = data.monitor.monitorType;
+      }
+    }
+
     logger.debug(`Storage Map: ${JSON.stringify(storageMap, null, 2)}`);
 
     return storageMap;