@oneuptime/common 12.0.32 → 12.0.34

package/Server/Utils/Monitor/Criteria/CompareCriteria.ts

@@ -9,24 +9,120 @@ import Typeof from "../../../../Types/Typeof";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
 export default class CompareCriteria {
+  /*
+   * Reduce an evaluation window to the value(s) a threshold is actually
+   * compared against.
+   *
+   * Before this existed, every comparator branched on `AnyValue` and let
+   * EVERY other evaluation type fall through to `.every()`. That silently
+   * turned Average, Sum, Maximum Value and Minimum Value — all four
+   * offered in the criteria UI and all four stored on real monitors — into
+   * "All Values". A user who asked to be paged on the five-minute AVERAGE
+   * was instead paged only when every single sample breached, which is a
+   * strictly rarer event and never the one they asked for.
+   *
+   * Returns a single-element array for the reducing aggregations so the
+   * caller compares against the aggregate, and the untouched window for
+   * the two set-quantifier types.
+   */
+  public static reduceWindow(data: {
+    values: Array<number>;
+    evaluationType?: EvaluateOverTimeType | undefined;
+  }): Array<number> {
+    const values: Array<number> = data.values;
+
+    if (values.length === 0) {
+      return values;
+    }
+
+    switch (data.evaluationType) {
+      case EvaluateOverTimeType.Average: {
+        const sum: number = values.reduce((a: number, b: number) => {
+          return a + b;
+        }, 0);
+        return [sum / values.length];
+      }
+      case EvaluateOverTimeType.Sum:
+        return [
+          values.reduce((a: number, b: number) => {
+            return a + b;
+          }, 0),
+        ];
+      case EvaluateOverTimeType.MaximumValue:
+        return [Math.max(...values)];
+      case EvaluateOverTimeType.MunimumValue:
+        return [Math.min(...values)];
+      default:
+        // AnyValue / AllValues quantify over the window itself.
+        return values;
+    }
+  }
+
+  /*
+   * The one place a numeric criteria filter decides whether it is met.
+   *
+   * `AnyValue` is an existential quantifier, everything else is universal
+   * once `reduceWindow` has collapsed the reducing aggregations to a
+   * single sample — so "Average > 90" is `[avg].every(v => v > 90)`, which
+   * is just `avg > 90`.
+   */
+  private static evaluateWindow(data: {
+    value: number | Array<number>;
+    evaluationType?: EvaluateOverTimeType | undefined;
+    predicate: (value: number) => boolean;
+  }): boolean {
+    if (!Array.isArray(data.value)) {
+      return data.predicate(data.value);
+    }
+
+    const values: Array<number> = CompareCriteria.reduceWindow({
+      values: data.value,
+      evaluationType: data.evaluationType,
+    });
+
+    if (values.length === 0) {
+      return false;
+    }
+
+    if (data.evaluationType === EvaluateOverTimeType.AnyValue) {
+      return values.some(data.predicate);
+    }
+
+    return values.every(data.predicate);
+  }
+
+  /*
+   * The samples that actually satisfied the filter.
+   *
+   * Only used to render the root cause. `getCompareMessage` used to print
+   * the WHOLE window and then assert every printed number breached the
+   * threshold, so a real alert read "is 72.35, 81.54, 79.95, 91.53, 87.73 %
+   * which is greater than 90 %" — four of those five are below 90.
+   */
+  public static getBreachingValues(data: {
+    values: Array<number>;
+    evaluationType?: EvaluateOverTimeType | undefined;
+    predicate: (value: number) => boolean;
+  }): Array<number> {
+    return CompareCriteria.reduceWindow({
+      values: data.values,
+      evaluationType: data.evaluationType,
+    }).filter(data.predicate);
+  }
+
   @CaptureSpan()
   public static greaterThan(data: {
     value: number | Array<number>;
     evaluationType?: EvaluateOverTimeType | undefined;
     threshold: number;
   }): boolean {
-    if (Array.isArray(data.value)) {
-      if (data.evaluationType === EvaluateOverTimeType.AnyValue) {
-        return data.value.some((value: number) => {
-          return value > data.threshold;
-        });
-      }
-      return data.value.every((value: number) => {
+    return CompareCriteria.evaluateWindow({
+      value: data.value,
+      evaluationType: data.evaluationType,
+      predicate: (value: number) => {
         return value > data.threshold;
-      });
-    }
-
-    return data.value > data.threshold;
+      },
+    });
   }
 
   @CaptureSpan()
@@ -77,18 +173,13 @@ export default class CompareCriteria {
     evaluationType?: EvaluateOverTimeType | undefined;
     threshold: number;
   }): boolean {
-    if (Array.isArray(data.value)) {
-      if (data.evaluationType === EvaluateOverTimeType.AnyValue) {
-        return data.value.some((value: number) => {
-          return value < data.threshold;
-        });
-      }
-      return data.value.every((value: number) => {
+    return CompareCriteria.evaluateWindow({
+      value: data.value,
+      evaluationType: data.evaluationType,
+      predicate: (value: number) => {
         return value < data.threshold;
-      });
-    }
-
-    return data.value < data.threshold;
+      },
+    });
   }
 
   @CaptureSpan()
@@ -97,18 +188,13 @@ export default class CompareCriteria {
     evaluationType?: EvaluateOverTimeType | undefined;
     threshold: number;
   }): boolean {
-    if (Array.isArray(data.value)) {
-      if (data.evaluationType === EvaluateOverTimeType.AnyValue) {
-        return data.value.some((value: number) => {
-          return value >= data.threshold;
-        });
-      }
-      return data.value.every((value: number) => {
+    return CompareCriteria.evaluateWindow({
+      value: data.value,
+      evaluationType: data.evaluationType,
+      predicate: (value: number) => {
         return value >= data.threshold;
-      });
-    }
-
-    return data.value >= data.threshold;
+      },
+    });
   }
 
   @CaptureSpan()
@@ -117,18 +203,13 @@ export default class CompareCriteria {
     evaluationType?: EvaluateOverTimeType | undefined;
     threshold: number;
   }): boolean {
-    if (Array.isArray(data.value)) {
-      if (data.evaluationType === EvaluateOverTimeType.AnyValue) {
-        return data.value.some((value: number) => {
-          return value <= data.threshold;
-        });
-      }
-      return data.value.every((value: number) => {
+    return CompareCriteria.evaluateWindow({
+      value: data.value,
+      evaluationType: data.evaluationType,
+      predicate: (value: number) => {
         return value <= data.threshold;
-      });
-    }
-
-    return data.value <= data.threshold;
+      },
+    });
   }
 
   @CaptureSpan()
@@ -137,18 +218,13 @@ export default class CompareCriteria {
     evaluationType?: EvaluateOverTimeType | undefined;
     threshold: number;
   }): boolean {
-    if (Array.isArray(data.value)) {
-      if (data.evaluationType === EvaluateOverTimeType.AnyValue) {
-        return data.value.some((value: number) => {
-          return value === data.threshold;
-        });
-      }
-      return data.value.every((value: number) => {
+    return CompareCriteria.evaluateWindow({
+      value: data.value,
+      evaluationType: data.evaluationType,
+      predicate: (value: number) => {
         return value === data.threshold;
-      });
-    }
-
-    return data.value === data.threshold;
+      },
+    });
   }
 
   @CaptureSpan()
@@ -157,18 +233,13 @@ export default class CompareCriteria {
     evaluationType?: EvaluateOverTimeType | undefined;
     threshold: number;
   }): boolean {
-    if (Array.isArray(data.value)) {
-      if (data.evaluationType === EvaluateOverTimeType.AnyValue) {
-        return data.value.some((value: number) => {
-          return value !== data.threshold;
-        });
-      }
-      return data.value.every((value: number) => {
+    return CompareCriteria.evaluateWindow({
+      value: data.value,
+      evaluationType: data.evaluationType,
+      predicate: (value: number) => {
         return value !== data.threshold;
-      });
-    }
-
-    return data.value !== data.threshold;
+      },
+    });
   }
 
   @CaptureSpan()
@@ -557,6 +628,7 @@ export default class CompareCriteria {
   }): string {
     // CPU Percent over the last 5 minutes is 10 which is less than the threshold of 20
     let message: string = "";
+    let breachSummary: string = "";
 
     let evaluationType: EvaluateOverTimeType | undefined =
       data.criteriaFilter.evaluateOverTimeOptions?.evaluateOverTimeType;
@@ -566,12 +638,34 @@ export default class CompareCriteria {
         data.criteriaFilter.metricMonitorOptions.metricAggregationType;
     }
 
-    if (evaluationType === EvaluateOverTimeType.AnyValue) {
-      message += "Any value of";
-    }
-
-    if (evaluationType === EvaluateOverTimeType.AllValues) {
-      message += "All values of";
+    /*
+     * Name the quantifier or the aggregation. Before this, only the two
+     * set quantifiers were named and the four reducing aggregations
+     * produced a bare "Metric Value is 82.62 which is greater than 90",
+     * which reads as a contradiction unless you already know the number
+     * shown is a five-minute average.
+     */
+    switch (evaluationType) {
+      case EvaluateOverTimeType.AnyValue:
+        message += "Any value of";
+        break;
+      case EvaluateOverTimeType.AllValues:
+        message += "All values of";
+        break;
+      case EvaluateOverTimeType.Average:
+        message += "The average of";
+        break;
+      case EvaluateOverTimeType.Sum:
+        message += "The sum of";
+        break;
+      case EvaluateOverTimeType.MaximumValue:
+        message += "The maximum of";
+        break;
+      case EvaluateOverTimeType.MunimumValue:
+        message += "The minimum of";
+        break;
+      default:
+        break;
     }
 
     /*
@@ -600,19 +694,86 @@ export default class CompareCriteria {
       message += ` over the last ${data.criteriaFilter.evaluateOverTimeOptions.timeValueInMinutes} minutes`;
     }
 
-    const unitSuffix: string = data.unit ? ` ${data.unit}` : "";
+    /*
+     * Suppress the units that read as noise — or as a wrong number —
+     * next to a value. OTel's dimensionless "1" marks ratio metrics
+     * whose samples are fractions in [0, 1]; rendering "0.06 1" is both
+     * ugly and misleading (it is not 1% — it is 6%), and a CLS breach
+     * read "is 0.31 1 which is greater than or equal to 0.25 1". UCUM
+     * annotation-only units ("{cpu}", "{packets}", "{errors}") are
+     * descriptive braces, not a unit a reader should be shown. Every
+     * real unit passes through unchanged, so "%" stays "%" and "ms"
+     * stays "ms".
+     *
+     * The dashboard side already applies the "1" half of this rule in
+     * MonitorCriteriaObservationBuilder.normalizeDisplayUnit, and
+     * ValueFormatter applies both halves when it formats a value. The
+     * email path never picked either up, so the same evaluation rendered
+     * worse in the alert than on the dashboard.
+     */
+    const trimmedUnit: string = (data.unit || "").trim();
+    const annotationOnlyUnitPattern: RegExp = /^\{[^{}]*\}$/;
+    const unitSuffix: string =
+      trimmedUnit &&
+      trimmedUnit !== "1" &&
+      !annotationOnlyUnitPattern.test(trimmedUnit)
+        ? ` ${trimmedUnit}`
+        : "";
 
     if (
       data.criteriaFilter.filterType !== FilterType.True &&
       data.criteriaFilter.filterType !== FilterType.False
     ) {
-      const formattedValues: string = CompareCriteria.formatCriteriaValues(
-        data.values,
-      );
+      /*
+       * Print the samples that actually breached, not the whole window.
+       *
+       * `AnyValue` is an existential quantifier: it is met when ONE sample
+       * crosses the threshold. Printing the entire window and then
+       * asserting "which is greater than 90" produced the sentence a real
+       * customer received — "is 72.35, 81.54, 79.95, 91.53, 87.73 % which
+       * is greater than 90 %" — in which four of the five listed numbers
+       * are below the threshold the sentence claims they all exceed.
+       *
+       * For every other evaluation type the reported set and the compared
+       * set are the same thing (a reducing aggregation collapses to one
+       * value; AllValues requires all of them), so this narrowing is a
+       * no-op and the existing wording is preserved exactly.
+       */
+      const reportedValues:
+        | Array<number | boolean>
+        | number
+        | boolean
+        | string = CompareCriteria.getReportedValues({
+        values: data.values,
+        threshold: data.threshold,
+        filterType: data.criteriaFilter.filterType as FilterType,
+        evaluationType: evaluationType,
+      });
+
+      const formattedValues: string =
+        CompareCriteria.formatCriteriaValues(reportedValues);
 
       message += ` is ${formattedValues}${unitSuffix}`;
 
       message += " which is";
+
+      /*
+       * How much of the window actually breached.
+       *
+       * Without it "Any value ... is 91.53 % which is greater than 90 %"
+       * is true but hides the thing the reader needs in order to judge
+       * whether to get out of bed: whether that was one transient sample
+       * or the whole five minutes. One-in-five is a spike; five-in-five is
+       * an outage. Only added when the two counts differ, so a fully
+       * breaching window keeps the shorter sentence.
+       */
+      if (
+        Array.isArray(data.values) &&
+        Array.isArray(reportedValues) &&
+        reportedValues.length < data.values.length
+      ) {
+        breachSummary = ` ${reportedValues.length} of ${data.values.length} samples in the evaluation window breached this threshold.`;
+      }
     }
 
     switch (data.criteriaFilter.filterType) {
@@ -654,7 +815,107 @@ export default class CompareCriteria {
         break;
     }
 
-    return message.trim();
+    return `${message.trim()}${breachSummary}`;
+  }
+
+  /*
+   * Narrow the window down to what the message should quote.
+   *
+   * Only `AnyValue` needs narrowing — see the comment at the call site.
+   * A reducing aggregation is collapsed to its single aggregate so the
+   * sentence quotes the average it actually compared, not the raw samples.
+   * If nothing is found to breach (a caller rendering a message for a
+   * filter that did not match, or a non-numeric window) the original
+   * values are returned unchanged rather than an empty list, so the
+   * message degrades to its previous behaviour instead of to "is ".
+   */
+  private static getReportedValues(data: {
+    values: Array<number | boolean> | number | boolean | string;
+    threshold: number | string | boolean;
+    filterType: FilterType;
+    evaluationType?: EvaluateOverTimeType | undefined;
+  }): Array<number | boolean> | number | boolean | string {
+    if (!Array.isArray(data.values)) {
+      return data.values;
+    }
+
+    if (typeof data.threshold !== Typeof.Number) {
+      return data.values;
+    }
+
+    const numericValues: Array<number> = data.values.filter(
+      (value: number | boolean): value is number => {
+        return typeof value === Typeof.Number && Number.isFinite(value);
+      },
+    );
+
+    if (numericValues.length !== data.values.length) {
+      return data.values;
+    }
+
+    const predicate: ((value: number) => boolean) | null =
+      CompareCriteria.getNumericPredicate({
+        filterType: data.filterType,
+        threshold: data.threshold as number,
+      });
+
+    if (!predicate) {
+      return data.values;
+    }
+
+    const breaching: Array<number> = CompareCriteria.getBreachingValues({
+      values: numericValues,
+      evaluationType: data.evaluationType,
+      predicate: predicate,
+    });
+
+    if (breaching.length === 0) {
+      return data.values;
+    }
+
+    return breaching;
+  }
+
+  /*
+   * The comparison a numeric filter type performs, as a predicate.
+   *
+   * Kept next to the comparator methods it mirrors so the two cannot
+   * drift: a filter type added to `compareCriteriaNumbers` without a case
+   * here falls back to quoting the whole window, which is the old
+   * behaviour rather than a wrong one.
+   */
+  private static getNumericPredicate(data: {
+    filterType: FilterType;
+    threshold: number;
+  }): ((value: number) => boolean) | null {
+    switch (data.filterType) {
+      case FilterType.GreaterThan:
+        return (value: number) => {
+          return value > data.threshold;
+        };
+      case FilterType.GreaterThanOrEqualTo:
+        return (value: number) => {
+          return value >= data.threshold;
+        };
+      case FilterType.LessThan:
+        return (value: number) => {
+          return value < data.threshold;
+        };
+      case FilterType.LessThanOrEqualTo:
+        return (value: number) => {
+          return value <= data.threshold;
+        };
+      case FilterType.EqualTo:
+        return (value: number) => {
+          return value === data.threshold;
+        };
+      case FilterType.NotEqualTo:
+        return (value: number) => {
+          return value !== data.threshold;
+        };
+      default:
+        return null;
+    }
   }
 
   private static formatCriteriaValues(

package/Server/Utils/Monitor/Criteria/DatabaseMonitorCriteria.ts

@@ -0,0 +1,190 @@
+import DataToProcess from "../DataToProcess";
+import CompareCriteria from "./CompareCriteria";
+import EvaluateOverTime, { OverTimeCriteriaValue } from "./EvaluateOverTime";
+import {
+  CheckOn,
+  CriteriaFilter,
+} from "../../../../Types/Monitor/CriteriaFilter";
+import {
+  DatabaseMetricDefinition,
+  getDatabaseMetricByMetricType,
+} from "../../../../Types/Monitor/DatabaseMetricCatalog";
+import DatabaseMonitorResponse, {
+  DatabaseMetricGroupStatus,
+} from "../../../../Types/Monitor/DatabaseMonitor/DatabaseMonitorResponse";
+import MonitorMetricType from "../../../../Types/Monitor/MonitorMetricType";
+import ProbeMonitorResponse from "../../../../Types/Probe/ProbeMonitorResponse";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+
+export default class DatabaseMonitorCriteria {
+  @CaptureSpan()
+  public static async isMonitorInstanceCriteriaFilterMet(input: {
+    dataToProcess: DataToProcess;
+    criteriaFilter: CriteriaFilter;
+    /*
+     * The monitor's monitoringInterval cron. Over-time filters use it to
+     * work out how many samples a fully covered window should hold, so a
+     * monitor that has only just started is not mistaken for one whose
+     * whole window is breaching.
+     */
+    monitoringInterval?: string | undefined;
+  }): Promise<string | null> {
+    const dataToProcess: ProbeMonitorResponse =
+      input.dataToProcess as ProbeMonitorResponse;
+
+    const databaseResponse: DatabaseMonitorResponse | undefined =
+      dataToProcess.databaseMonitorResponse;
+
+    /*
+     * Resolved once for every CheckOn. Database Metric filters name their
+     * series in databaseMonitorOptions.metricType rather than in the CheckOn,
+     * which is why EvaluateOverTime resolves the series from the whole filter.
+     */
+    const overTime: OverTimeCriteriaValue =
+      await EvaluateOverTime.getOverTimeValueForCriteriaFilter({
+        projectId: dataToProcess.projectId,
+        monitorId: input.dataToProcess.monitorId!,
+        criteriaFilter: input.criteriaFilter,
+        monitoringInterval: input.monitoringInterval,
+      });
+
+    /*
+     * The window could not back this over-time filter, so the no-data policy
+     * has already decided it - do not fall through to the value that arrived
+     * with this one check. That fallback is what let "all values over the
+     * last N minutes" fire off a single bad reading.
+     *
+     * "Database Is Online" is exempt, mirroring the server monitor. An
+     * unreachable database is the one signal that must always reach its
+     * criteria: a monitor whose window is still filling, or whose metric
+     * write has not caught up, would otherwise be unable to go offline at
+     * all while the dashboard shows the connection failing.
+     */
+    if (
+      overTime.earlyReturn &&
+      input.criteriaFilter.checkOn !== CheckOn.DatabaseIsOnline
+    ) {
+      return overTime.earlyReturn.result;
+    }
+
+    const overTimeValue:
+      | Array<number | boolean>
+      | number
+      | boolean
+      | undefined = overTime.value;
+
+    // Could the probe connect and run the baseline probe query?
+    if (input.criteriaFilter.checkOn === CheckOn.DatabaseIsOnline) {
+      const currentIsOnline: boolean | Array<boolean> =
+        (overTimeValue as Array<boolean>) ?? dataToProcess.isOnline;
+
+      return CompareCriteria.compareCriteriaBoolean({
+        value: currentIsOnline,
+        criteriaFilter: input.criteriaFilter,
+      });
+    }
+
+    /*
+     * One branch for all of the catalog's series: the filter names the one it
+     * reads in databaseMonitorOptions.metricType, exactly as disk usage names
+     * a mount and an SNMP value names an OID.
+     *
+     * Thresholds are whole numbers - CompareCriteria.convertToNumber parses
+     * with parseInt, so "99.5" is compared as 99. Every catalog unit is
+     * chosen so an integral threshold is the natural one to type.
+     */
+    if (input.criteriaFilter.checkOn === CheckOn.DatabaseMetric) {
+      const metricType: MonitorMetricType | undefined =
+        input.criteriaFilter.databaseMonitorOptions?.metricType;
+
+      if (!metricType) {
+        return null;
+      }
+
+      const definition: DatabaseMetricDefinition | null =
+        getDatabaseMetricByMetricType(metricType);
+
+      if (!definition) {
+        return null;
+      }
+
+      const value: number | undefined = databaseResponse?.metrics[metricType];
+
+      /*
+       * The metric was not collected on this check - the engine cannot report
+       * it, the monitoring login is missing the grant its group needs, or the
+       * group timed out. Absent is not zero, and a criteria that cannot see a
+       * value must never claim one breached: a revoked grant would otherwise
+       * raise an incident about a database that is perfectly healthy.
+       * CheckOn.DatabaseCollectionError is how an operator alerts on lost
+       * visibility.
+       */
+      if (value === undefined || value === null) {
+        return null;
+      }
+
+      const threshold: number | null = CompareCriteria.convertToNumber(
+        input.criteriaFilter.value,
+      );
+
+      if (threshold === null) {
+        return null;
+      }
+
+      const result: string | null = CompareCriteria.compareCriteriaNumbers({
+        value: (overTimeValue as Array<number>) ?? value,
+        threshold: threshold,
+        criteriaFilter: input.criteriaFilter,
+        unit: definition.unit || undefined,
+      });
+
+      if (result) {
+        return `${definition.friendlyName} - ${result}`;
+      }
+
+      return null;
+    }
+
+    // Groups that could not be collected, as one operator-facing line each.
+    if (input.criteriaFilter.checkOn === CheckOn.DatabaseCollectionError) {
+      const collectionIssues: string = (
+        databaseResponse?.unavailableGroups || []
+      )
+        .map((status: DatabaseMetricGroupStatus) => {
+          return `${status.group}: ${status.message}`;
+        })
+        .join("; ");
+
+      /*
+       * compareEmptyAndNotEmpty tests for null/undefined, and an empty string
+       * would read as "not empty" - so a check with nothing to report has to
+       * arrive as undefined for "Is Empty" to mean "collection was clean".
+       */
+      const emptyNotEmptyResult: string | null =
+        CompareCriteria.compareEmptyAndNotEmpty({
+          value: collectionIssues || undefined,
+          criteriaFilter: input.criteriaFilter,
+        });
+
+      if (emptyNotEmptyResult) {
+        return emptyNotEmptyResult;
+      }
+
+      if (
+        input.criteriaFilter.value !== null &&
+        input.criteriaFilter.value !== undefined &&
+        collectionIssues
+      ) {
+        return CompareCriteria.compareCriteriaStrings({
+          value: collectionIssues,
+          threshold: input.criteriaFilter.value.toString(),
+          criteriaFilter: input.criteriaFilter,
+        });
+      }
+
+      return null;
+    }
+
+    return null;
+  }
+}