@oneuptime/common 14.0.3 → 14.0.6

package/Server/Utils/AI/Toolbox/ResourceTools.ts

@@ -11,6 +11,7 @@ import ServerlessFunction from "../../../../Models/DatabaseModels/ServerlessFunc
 import CloudResource from "../../../../Models/DatabaseModels/CloudResource";
 import IoTFleet from "../../../../Models/DatabaseModels/IoTFleet";
 import NetworkDevice from "../../../../Models/DatabaseModels/NetworkDevice";
+import DatabaseServer from "../../../../Models/DatabaseModels/DatabaseServer";
 import Metric from "../../../../Models/AnalyticsModels/Metric";
 import Log from "../../../../Models/AnalyticsModels/Log";
 import Span from "../../../../Models/AnalyticsModels/Span";
@@ -55,6 +56,7 @@ import ServerlessFunctionService from "../../../Services/ServerlessFunctionServi
 import CloudResourceService from "../../../Services/CloudResourceService";
 import IoTFleetService from "../../../Services/IoTFleetService";
 import NetworkDeviceService from "../../../Services/NetworkDeviceService";
+import DatabaseServerService from "../../../Services/DatabaseServerService";
 import MetricService from "../../../Services/MetricService";
 import LogAggregationService, {
   HistogramBucket,
@@ -66,7 +68,9 @@ import FindBy from "../../../Types/Database/FindBy";
 import QueryHelper from "../../../Types/Database/QueryHelper";
 import DatabaseRequestType from "../../../Types/BaseDatabase/DatabaseRequestType";
 import ModelPermission from "../../../Types/AnalyticsDatabase/ModelPermission";
-import { ResourceEntityScope } from "../../Telemetry/ResourceEntityFilter";
+import ResourceEntityFilter, {
+  ResourceEntityScope,
+} from "../../Telemetry/ResourceEntityFilter";
 import ToolResultSerializer, { SerializedResult } from "./Serializer";
 import {
   ObservabilityTool,
@@ -87,6 +91,17 @@ interface ResourceDescriptor {
   attributeKey?: string;
   keyFor?: (projectId: string, identifier: string) => string;
   runtime?: string;
+  /*
+   * For a resource whose telemetry is not named by one identifier (a
+   * Database owns several endpoints plus the pods / containers it runs as):
+   * its scope, resolved exactly the way the explorer's resource facet and
+   * the resource's own pages resolve it. Called only after the
+   * permission-checked parent read.
+   */
+  resolveResourceScopes?: (data: {
+    projectId: ObjectID;
+    id: ObjectID;
+  }) => Promise<Array<ResourceEntityScope>>;
 }
 
 /*
@@ -274,6 +289,30 @@ const RESOURCE_DESCRIPTORS: Record<AIResourceType, ResourceDescriptor> = {
       "lastPolledAt",
     ],
   },
+  [AIResourceType.DatabaseServer]: {
+    model: DatabaseServer,
+    findBy: (data: FindBy<BaseModel>) => {
+      return DatabaseServerService.findBy(data as never);
+    },
+    fields: [
+      "dbSystem",
+      "dbVersion",
+      "serverAddress",
+      "serverPort",
+      "discoverySource",
+      "instanceCount",
+      "collectorLastSeenAt",
+    ],
+    resolveResourceScopes: (data: {
+      projectId: ObjectID;
+      id: ObjectID;
+    }): Promise<Array<ResourceEntityScope>> => {
+      return ResourceEntityFilter.resolveScopes({
+        projectId: data.projectId,
+        selections: { databaseServerId: [data.id.toString()] },
+      });
+    },
+  },
 };
 
 function getResourceType(args: JSONObject): AIResourceType {
@@ -503,10 +542,33 @@ export function buildAIResourceTelemetryScope(data: {
   resource: JSONObject;
   projectId: ObjectID;
   signal: ResourceSignal;
+  // From the descriptor's resolveResourceScopes, when it has one.
+  resolvedScopes?: Array<ResourceEntityScope> | undefined;
 }): AIResourceTelemetryScope {
   const descriptor: ResourceDescriptor = RESOURCE_DESCRIPTORS[data.type];
   const note: string =
     "Scope is the parent resource; no individual child identity or namespace was selected. Missing telemetry is not evidence of health.";
+  if (data.type === AIResourceType.DatabaseServer) {
+    /*
+     * Only a resolved scope of THIS database is used; one that could not be
+     * resolved narrows to the row's own collector / agent data, never widens.
+     */
+    const scopes: Array<ResourceEntityScope> = (
+      data.resolvedScopes || []
+    ).filter((scope: ResourceEntityScope): boolean => {
+      return (
+        scope.entityIds.length === 1 &&
+        scope.entityIds[0] === data.id.toString()
+      );
+    });
+    return {
+      resourceScopes:
+        scopes.length > 0
+          ? scopes
+          : [{ entityIds: [data.id.toString()], entityKeys: [] }],
+      note: `${note} A database's telemetry is its own collector / Database Agent data plus everything carrying one of its endpoints or workload members: application database calls, and the logs and metrics of the pods and containers it runs as.`,
+    };
+  }
   if (data.type === AIResourceType.CloudResource) {
     const attributes: Record<string, string> = {};
     for (const [column, key] of [
@@ -611,7 +673,18 @@ export const QueryResourceTelemetryTool: ObservabilityTool = {
       },
       aggregationType: {
         type: "string",
-        enum: ["Avg", "Min", "Max", "Sum", "Count", "P50", "P90", "P95", "P99"],
+        enum: [
+          "Avg",
+          "Min",
+          "Max",
+          "Sum",
+          "Count",
+          "P50",
+          "P75",
+          "P90",
+          "P95",
+          "P99",
+        ],
         description:
           "Metric aggregation (defaultAvg); counter sums are exported values, not derived rates.",
       },
@@ -707,6 +780,12 @@ export const QueryResourceTelemetryTool: ObservabilityTool = {
       resource: resources[0] as unknown as JSONObject,
       projectId: ctx.projectId,
       signal,
+      resolvedScopes: descriptor.resolveResourceScopes
+        ? await descriptor.resolveResourceScopes({
+            projectId: ctx.projectId,
+            id,
+          })
+        : undefined,
     });
     const target: AIChatCitationTarget = resourceTarget(type, id);
     const label: string = `${getAIResourceDefinition(type).label} ${signal}, ${startTime.toISOString()} – ${endTime.toISOString()}`;

package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts

@@ -1938,7 +1938,7 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
      * EXAMPLE:
      * SELECT sum(Metric.value) as avg_value, date_trunc('hour', toStartOfInterval(createdAt, INTERVAL 1 hour)) as createdAt
      *
-     * Percentile aggregations (P50/P90/P95/P99) compile to ClickHouse's
+     * Percentile aggregations (P50/P75/P90/P95/P99) compile to ClickHouse's
      * `quantile(level)(column)`. This is the right thing for scalar
      * columns (Span.duration, Metric.value when the metric is a Sum or
      * Gauge, etc.). MetricService overrides this method when it has

package/Server/Utils/Database/AffectedResourceRelations.ts

@@ -1,4 +1,5 @@
 import CephClusterService from "../../Services/CephClusterService";
+import DatabaseServerService from "../../Services/DatabaseServerService";
 import DockerHostService from "../../Services/DockerHostService";
 import DockerResourceService from "../../Services/DockerResourceService";
 import DockerSwarmClusterService from "../../Services/DockerSwarmClusterService";
@@ -108,6 +109,11 @@ function getAllAffectedResourceRelations(): Array<ProjectScopedRelation> {
       modelName: "Ceph Cluster",
       service: CephClusterService,
     },
+    {
+      column: "databaseServers",
+      modelName: "Database",
+      service: DatabaseServerService,
+    },
     {
       column: "dockerResources",
       modelName: "Docker Resource",

package/Server/Utils/Monitor/Criteria/CompareCriteria.ts

@@ -280,6 +280,39 @@ export default class CompareCriteria {
     return threshold as number;
   }
 
+  /*
+   * A metric-value threshold as a number, fraction included.
+   *
+   * The criteria form saves what the user typed as a STRING, and
+   * convertToNumber's parseInt truncated it: "0.9" on a ratio metric was
+   * compared as 0 and fired on every non-zero sample, and "85.5" was
+   * compared as 85. Metric values are where a fractional threshold is the
+   * natural one to type (ratios, seconds, load averages), so they are
+   * parsed with parseFloat. Every other checkOn keeps convertToNumber.
+   *
+   * Anything that does not parse to a finite number is treated as missing,
+   * so the caller ignores the filter instead of comparing against NaN.
+   */
+  @CaptureSpan()
+  public static convertMetricThresholdToNumber(
+    threshold: string | number | undefined,
+  ): number | null {
+    if (threshold === undefined || threshold === null) {
+      return null;
+    }
+
+    const parsed: number =
+      typeof threshold === Typeof.String
+        ? parseFloat(threshold as string)
+        : (threshold as number);
+
+    if (typeof parsed !== Typeof.Number || !Number.isFinite(parsed)) {
+      return null;
+    }
+
+    return parsed;
+  }
+
   @CaptureSpan()
   public static checkEqualToOrNotEqualTo(data: {
     value: string | number;

package/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.ts

@@ -1,5 +1,6 @@
 import AggregateModel from "../../../../Types/BaseDatabase/AggregatedModel";
 import AggregatedResult from "../../../../Types/BaseDatabase/AggregatedResult";
+import MetricAliasData from "../../../../Types/Metrics/MetricAliasData";
 import MetricFormulaConfigData from "../../../../Types/Metrics/MetricFormulaConfigData";
 import MetricQueryConfigData from "../../../../Types/Metrics/MetricQueryConfigData";
 import MetricsAggregationType from "../../../../Types/Metrics/MetricsAggregationType";
@@ -179,9 +180,10 @@ export default class MetricMonitorCriteria {
     projectId: { toString(): string } | undefined;
     nativeUnitsByMetricName?: { [key: string]: string } | undefined;
   }): Promise<MetricSeriesEvaluationResult> {
-    const rawThreshold: number | null = CompareCriteria.convertToNumber(
-      input.criteriaFilter.value,
-    );
+    const rawThreshold: number | null =
+      CompareCriteria.convertMetricThresholdToNumber(
+        input.criteriaFilter.value,
+      );
 
     const metricAlias: string =
       input.criteriaFilter.metricMonitorOptions?.metricAlias || "";
@@ -274,6 +276,7 @@ export default class MetricMonitorCriteria {
         seriesFingerprint: input.seriesFingerprint,
         seriesLabels: input.seriesLabels,
         projectId: input.projectId,
+        nativeUnitsByMetricName: input.nativeUnitsByMetricName,
       });
     }
 
@@ -363,6 +366,19 @@ export default class MetricMonitorCriteria {
       },
     );
 
+    const finiteNumbers: Array<number> = numbersInDisplayUnit.filter(
+      (n: number) => {
+        return Number.isFinite(n);
+      },
+    );
+
+    if (finiteNumbers.length > 0) {
+      metricContext.sampleValueRange = {
+        min: Math.min(...finiteNumbers),
+        max: Math.max(...finiteNumbers),
+      };
+    }
+
     const comparisonMessage: string | null =
       CompareCriteria.compareCriteriaNumbers({
         value: numbersInDisplayUnit.length > 0 ? numbersInDisplayUnit : 0,
@@ -438,7 +454,10 @@ export default class MetricMonitorCriteria {
         } = {
           value: convertedValue,
           timestamp: sample.timestamp,
-          attributes: MetricMonitorCriteria.extractLabelAttributes(sample),
+          attributes: MetricMonitorCriteria.extractLabelAttributes(
+            sample,
+            input.seriesLabels,
+          ),
         };
 
         if (componentValueLookup && metricContext.components) {
@@ -571,7 +590,10 @@ export default class MetricMonitorCriteria {
     }
   }
 
-  private static extractLabelAttributes(sample: AggregateModel): JSONObject {
+  private static extractLabelAttributes(
+    sample: AggregateModel,
+    seriesLabels?: JSONObject | undefined,
+  ): JSONObject {
     /*
      * AggregatedModel has a string index signature that holds group-by
      * attributes alongside `timestamp` and `value`. Strip the known keys
@@ -579,7 +601,7 @@ export default class MetricMonitorCriteria {
      */
     const labels: JSONObject = {};
     for (const key of Object.keys(sample)) {
-      if (key === "timestamp" || key === "value") {
+      if (key === "timestamp" || key === "value" || key === "attributes") {
         continue;
       }
       const v: unknown = (sample as unknown as JSONObject)[key];
@@ -588,6 +610,50 @@ export default class MetricMonitorCriteria {
       }
       labels[key] = v as JSONObject[string];
     }
+
+    /*
+     * Grouped rows carry their labels NESTED under `attributes` — both
+     * MetricService.aggregateBy with groupByAttributeKeys and the
+     * infrastructure monitors' raw-scan path return them that way. The
+     * map used to be copied through as a single `attributes` label, which
+     * the Breaching Samples list rendered as "`attributes`:
+     * `[object Object]`".
+     *
+     * The raw scan nests EVERY datapoint attribute of the bucket's first
+     * row, not just the grouped ones, so when the series labels are known
+     * only those keys are lifted out: they are what identifies the series.
+     * A formula's rows carry no attributes at all, so each key falls back
+     * to the series' own label — every sample of a series belongs to it.
+     */
+    const nested: unknown = (sample as unknown as JSONObject)["attributes"];
+    const nestedAttributes: JSONObject =
+      nested && typeof nested === "object" && !Array.isArray(nested)
+        ? (nested as JSONObject)
+        : {};
+
+    if (nested !== undefined && nested !== null && typeof nested !== "object") {
+      // A flat label that happens to be called `attributes`.
+      labels["attributes"] = nested as JSONObject[string];
+    }
+
+    const labelKeys: Array<string> = Object.keys(seriesLabels || {});
+    const keys: Array<string> =
+      labelKeys.length > 0 ? labelKeys : Object.keys(nestedAttributes);
+
+    for (const key of keys) {
+      if (key in labels) {
+        continue;
+      }
+
+      const v: unknown = nestedAttributes[key] ?? seriesLabels?.[key];
+
+      if (v === undefined || v === null || v === "" || typeof v === "object") {
+        continue;
+      }
+
+      labels[key] = v as JSONObject[string];
+    }
+
     return labels;
   }
 
@@ -659,12 +725,20 @@ export default class MetricMonitorCriteria {
           formulaConfig: f,
           queryConfigs: input.queryConfigs,
           formulaConfigs: input.formulaConfigs,
+          nativeUnitsByMetricName: input.nativeUnitsByMetricName,
         })
       : undefined;
 
+    const displayName: string | undefined =
+      MetricMonitorCriteria.getAliasDisplayName({
+        aliasData: q?.metricAliasData || f?.metricAliasData,
+        metricAlias: input.metricAlias,
+      });
+
     return {
       metricName,
       alias: input.metricAlias,
+      ...(displayName ? { displayName } : {}),
       unit,
       aggregationType,
       isFormula: Boolean(f),
@@ -677,6 +751,36 @@ export default class MetricMonitorCriteria {
     };
   }
 
+  /**
+   * The human legend of a query or formula, or undefined when it is empty
+   * or just repeats the alias. The shipped templates title every single
+   * query with its own alias ("container_restarts"), which is not a name
+   * worth promoting over the metric name; the ratio templates give their
+   * formula a real legend ("Node Memory Utilization (%)").
+   */
+  public static getAliasDisplayName(input: {
+    aliasData: MetricAliasData | undefined;
+    metricAlias: string;
+  }): string | undefined {
+    const candidates: Array<string | undefined> = [
+      input.aliasData?.legend,
+      input.aliasData?.title,
+    ];
+
+    for (const candidate of candidates) {
+      const trimmed: string = (candidate || "").trim();
+
+      if (
+        trimmed &&
+        trimmed.toLowerCase() !== input.metricAlias.trim().toLowerCase()
+      ) {
+        return trimmed;
+      }
+    }
+
+    return undefined;
+  }
+
   /**
    * Resolve the variables the formula references to their source
    * query/formula definitions so the root cause can label each
@@ -686,6 +790,7 @@ export default class MetricMonitorCriteria {
     formulaConfig: MetricFormulaConfigData;
     queryConfigs: Array<MetricQueryConfigData>;
     formulaConfigs: Array<MetricFormulaConfigData>;
+    nativeUnitsByMetricName?: { [key: string]: string } | undefined;
   }): Array<MetricComponent> {
     const formula: string =
       input.formulaConfig.metricFormulaData?.metricFormula || "";
@@ -717,10 +822,25 @@ export default class MetricMonitorCriteria {
             | undefined) ||
           queryMatch.metricAliasData?.title ||
           normalizedAlias;
+        /*
+         * A component's values are in its legendUnit when it has one (the
+         * worker converted them), and in the metric's own unit when it
+         * does not — the ratio templates leave both sides unconverted, so
+         * `used_mem` and `alloc_mem` are bytes. Without the fallback those
+         * printed as bare 12-digit numbers under a formula that read
+         * "88.82%".
+         */
+        const rawMetricName: string | undefined =
+          (queryMatch.metricQueryData?.filterData?.metricName as
+            | string
+            | undefined) || undefined;
+        const nativeUnit: string | undefined = rawMetricName
+          ? input.nativeUnitsByMetricName?.[rawMetricName.toLowerCase()]
+          : undefined;
         components.push({
           alias: normalizedAlias,
           name,
-          unit: queryMatch.metricAliasData?.legendUnit || null,
+          unit: queryMatch.metricAliasData?.legendUnit || nativeUnit || null,
           isFormula: false,
         });
         continue;
@@ -774,6 +894,7 @@ export default class MetricMonitorCriteria {
     seriesFingerprint: string | undefined;
     seriesLabels: JSONObject;
     projectId: { toString(): string } | undefined;
+    nativeUnitsByMetricName?: { [key: string]: string } | undefined;
   }): Promise<MetricSeriesEvaluationResult> {
     const { criteriaFilter, metricContext } = input;
 
@@ -858,6 +979,19 @@ export default class MetricMonitorCriteria {
       return noBreach();
     }
 
+    /*
+     * The baseline is aggregated from the raw stored values, so it is in
+     * the metric's NATIVE unit — but the samples were already converted
+     * into the query's legendUnit by the worker. Rescale the baseline
+     * into the sample unit before anything compares against it or
+     * renders it; otherwise a "By" baseline read as "MB" puts the mean a
+     * million times too high — AnomalouslyHigh can never fire,
+     * AnomalouslyLow fires on every sample, and the root cause prints
+     * the mean in TB.
+     */
+    const baselineUnit: string | undefined =
+      input.nativeUnitsByMetricName?.[metricContext.metricName.toLowerCase()];
+
     const baselineByHour: Map<number, BaselineSummary> = new Map();
     for (const hour of hoursInWindow) {
       const baseline: BaselineSummary | null =
@@ -869,7 +1003,14 @@ export default class MetricMonitorCriteria {
           minSamples,
         });
       if (baseline && baseline.isReliable) {
-        baselineByHour.set(hour, baseline);
+        baselineByHour.set(
+          hour,
+          MetricMonitorCriteria.convertBaselineToUnit({
+            baseline,
+            fromUnit: baselineUnit,
+            toUnit: metricContext.unit || undefined,
+          }),
+        );
       }
     }
 
@@ -945,7 +1086,10 @@ export default class MetricMonitorCriteria {
         breachingSamples.push({
           value: sample.value,
           timestamp: ts,
-          attributes: MetricMonitorCriteria.extractLabelAttributes(sample),
+          attributes: MetricMonitorCriteria.extractLabelAttributes(
+            sample,
+            input.seriesLabels,
+          ),
         });
         if (!firstBreach) {
           firstBreach = { sample, baseline, sigma: observedSigma };
@@ -1035,6 +1179,45 @@ export default class MetricMonitorCriteria {
     };
   }
 
+  /**
+   * Rescale every quantity in a baseline from one unit into another.
+   * Every supported unit family converts by a pure multiplier (no
+   * offsets), so the spread terms (stddev, MAD) scale by the same
+   * factor as the location terms. When the units match, either is
+   * missing, or they aren't convertible, the baseline is returned
+   * unchanged — the same pass-through the worker applies to samples.
+   */
+  private static convertBaselineToUnit(input: {
+    baseline: BaselineSummary;
+    fromUnit: string | undefined;
+    toUnit: string | undefined;
+  }): BaselineSummary {
+    const { baseline, fromUnit, toUnit } = input;
+
+    if (!fromUnit || !toUnit || fromUnit === toUnit) {
+      return baseline;
+    }
+
+    const convert: (value: number) => number = (value: number): number => {
+      return MetricUnitUtil.convertToMetricUnit({
+        value,
+        fromUnit,
+        metricUnit: toUnit,
+      });
+    };
+
+    return {
+      ...baseline,
+      mean: convert(baseline.mean),
+      stddev: convert(baseline.stddev),
+      median: convert(baseline.median),
+      p95: convert(baseline.p95),
+      minObserved: convert(baseline.minObserved),
+      maxObserved: convert(baseline.maxObserved),
+      ...(baseline.mad !== undefined ? { mad: convert(baseline.mad) } : {}),
+    };
+  }
+
   private static anomalyBreaches(
     value: number,
     expectedHigh: number,

package/Server/Utils/Monitor/MonitorAlert.ts

@@ -37,6 +37,7 @@ import SeriesResourceLinker, {
 import MonitorDependencySuppression, {
   DependencySuppressionResult,
 } from "./MonitorDependencySuppression";
+import PerSeriesResolutionRootCause from "./PerSeriesResolutionRootCause";
 import { JSONObject } from "../../../Types/JSON";
 import OneUptimeDate from "../../../Types/Date";
 import MonitorEvaluationSummary from "../../../Types/Monitor/MonitorEvaluationSummary";
@@ -76,6 +77,13 @@ export default class MonitorAlert {
      * code paths for grouped incoming-request monitors.
      */
     disableSeriesAbsenceResolution?: boolean | undefined;
+    /**
+     * Every criteria on the monitor, keyed by id. Used to name the
+     * criteria a recovered series no longer satisfies when its alert is
+     * resolved per series; without it the resolution text falls back to
+     * "the criteria that raised it".
+     */
+    criteriaInstancesById?: Dictionary<MonitorCriteriaInstance> | undefined;
   }): Promise<Array<Alert>> {
     // check active alerts and if there are open alerts, do not create another alert.
     const openAlerts: Array<Alert> = await AlertService.findBy({
@@ -123,10 +131,29 @@ export default class MonitorAlert {
       if (shouldClose) {
         resolvedAlertIds.add(openAlert.id!.toString());
 
+        /*
+         * A series resolved per series closes on a tick where other series
+         * may still breach, and input.rootCause describes those. Record why
+         * THIS series resolved instead. Whole-monitor resolves keep the
+         * caller's root cause.
+         */
+        const isPerSeriesResolution: boolean =
+          input.breachingSeriesFingerprints !== undefined &&
+          Boolean(openAlert.seriesFingerprint);
+
+        const rootCause: string = isPerSeriesResolution
+          ? PerSeriesResolutionRootCause.build({
+              seriesLabels: openAlert.seriesLabels,
+              createdCriteriaId:
+                openAlert.createdCriteriaId?.toString() || undefined,
+              criteriaInstancesById: input.criteriaInstancesById,
+            })
+          : input.rootCause;
+
         // then resolve alert.
         await this.resolveOpenAlert({
           openAlert: openAlert,
-          rootCause: input.rootCause,
+          rootCause: rootCause,
           dataToProcess: input.dataToProcess,
         });