@oneuptime/common 14.0.2 → 14.0.5

package/Server/Utils/Monitor/MonitorIncident.ts

@@ -44,6 +44,7 @@ import MonitorSummarySnapshotUtil from "../../../Utils/Monitor/MonitorSummarySna
 import { IncidentMemberRoleAssignment } from "../../../Types/Monitor/CriteriaIncident";
 import { PerSeriesCriteriaMatch } from "../../../Types/Probe/ProbeApiIngestResponse";
 import MonitorResourceContextUtil from "./MonitorResourceContext";
+import PerSeriesResolutionRootCause from "./PerSeriesResolutionRootCause";
 import SeriesResourceLinker, {
   SeriesResolvedResourceIds,
 } from "./SeriesResourceLinker";
@@ -90,6 +91,13 @@ export default class MonitorIncident {
      * no-criteria-met code paths for grouped incoming-request monitors.
      */
     disableSeriesAbsenceResolution?: boolean | undefined;
+    /**
+     * Every criteria on the monitor, keyed by id. Used to name the
+     * criteria a recovered series no longer satisfies when its incident
+     * is resolved per series; without it the resolution text falls back
+     * to "the criteria that raised it".
+     */
+    criteriaInstancesById?: Dictionary<MonitorCriteriaInstance> | undefined;
   }): Promise<Array<Incident>> {
     // check active incidents and if there are open incidents, do not create another incident.
     const openIncidents: Array<Incident> = await IncidentService.findBy({
@@ -138,10 +146,29 @@ export default class MonitorIncident {
       if (shouldClose) {
         resolvedIncidentIds.add(openIncident.id!.toString());
 
+        /*
+         * A series resolved per series closes on a tick where other series
+         * may still breach, and input.rootCause describes those. Record why
+         * THIS series resolved instead. Whole-monitor resolves keep the
+         * caller's root cause.
+         */
+        const isPerSeriesResolution: boolean =
+          input.breachingSeriesFingerprints !== undefined &&
+          Boolean(openIncident.seriesFingerprint);
+
+        const rootCause: string = isPerSeriesResolution
+          ? PerSeriesResolutionRootCause.build({
+              seriesLabels: openIncident.seriesLabels,
+              createdCriteriaId:
+                openIncident.createdCriteriaId?.toString() || undefined,
+              criteriaInstancesById: input.criteriaInstancesById,
+            })
+          : input.rootCause;
+
         // then resolve incident.
         await this.resolveOpenIncident({
           openIncident: openIncident,
-          rootCause: input.rootCause,
+          rootCause: rootCause,
           dataToProcess: input.dataToProcess,
         });
 

package/Server/Utils/Monitor/MonitorMaintenanceSuppression.ts

@@ -1,12 +1,20 @@
+import Monitor from "../../../Models/DatabaseModels/Monitor";
 import ScheduledMaintenance from "../../../Models/DatabaseModels/ScheduledMaintenance";
 import { LIMIT_PER_PROJECT } from "../../../Types/Database/LimitMax";
+import OneUptimeDate from "../../../Types/Date";
+import MonitorCriteriaInstance from "../../../Types/Monitor/MonitorCriteriaInstance";
+import MonitorEvaluationSummary from "../../../Types/Monitor/MonitorEvaluationSummary";
 import ObjectID from "../../../Types/ObjectID";
 import { PerSeriesCriteriaMatch } from "../../../Types/Probe/ProbeApiIngestResponse";
 import ScheduledMaintenanceService from "../../Services/ScheduledMaintenanceService";
+import logger from "../Logger";
 import CaptureSpan from "../Telemetry/CaptureSpan";
+import MonitorResourceContextUtil from "./MonitorResourceContext";
+import MonitorStepResourceIdentity from "./MonitorStepResourceIdentity";
 import SeriesResourceLabels, {
   SeriesResourceRefs,
 } from "./SeriesResourceLabels";
+import { SeriesResolvedResourceIds } from "./SeriesResourceLinker";
 
 /*
  * Ids and names of one resource type that are currently inside an
@@ -51,6 +59,30 @@ export interface MaintainedResourceKeys {
    */
   iotFleets: ResourceKeySet;
   services: ResourceKeySet;
+  /*
+   * Databases are the reverse of the name-only clusters above: a series
+   * identifies one only by the `oneuptime.database.server.id` stamp (see
+   * DatabaseServerIdLabelKeys), so only the id set is ever matched; the
+   * name set exists for shape parity and is never filled.
+   */
+  databaseServers: ResourceKeySet;
+}
+
+/*
+ * What scheduled maintenance silences in ONE evaluation: the breaching
+ * series whose own resource is under maintenance (grouped monitors), and —
+ * when the monitor's own configuration names a database — the whole
+ * evaluation. See getMaintenanceSuppression.
+ */
+export interface MonitorMaintenanceSuppressionResult {
+  suppressedSeriesFingerprints: Set<string>;
+  /*
+   * The databases under an ongoing maintenance window that silence every
+   * incident and alert this evaluation would create. Empty unless the
+   * monitor names at least one database and every resource it names is
+   * under maintenance.
+   */
+  suppressingDatabaseServerIds: Array<string>;
 }
 
 /*
@@ -72,7 +104,7 @@ export interface MaintainedResourceKeys {
  * other 90 hosts keep alerting. It covers every resource type a
  * maintenance event can attach to AND a series can identify: Host,
  * DockerHost, KubernetesCluster, ProxmoxCluster, VMwareVCenter,
- * CephCluster, and Service.
+ * CephCluster, DatabaseServer, and Service.
  */
 export default class MonitorMaintenanceSuppression {
   /*
@@ -103,6 +135,225 @@ export default class MonitorMaintenanceSuppression {
     });
   }
 
+  /*
+   * Everything scheduled maintenance silences in one evaluation: the
+   * per-series set above, plus the whole evaluation when the monitor's OWN
+   * configuration names a database under maintenance.
+   *
+   * The second half exists because series labels only exist on grouped
+   * criteria, and a database's monitors are not grouped: every monitor its
+   * Recommendations tab creates, and every one built from a database chart,
+   * is ungrouped and scoped by the `oneuptime.database.server.id` filter,
+   * and a Database Health or SQL Query probe names its database by the
+   * host:port it connects to. Without it, "Engine Metrics Stopped" paged
+   * on-call for the very upgrade the team had put the database into
+   * maintenance for.
+   *
+   * A monitor is silenced only while EVERY resource its configuration
+   * names is under maintenance: an ungrouped monitor cannot say which of
+   * two databases (or which of a database and the service calling it)
+   * breached, and silencing it for one would hide an outage on the other.
+   * Scoped to monitors that name a database — the resource whose monitors
+   * are all ungrouped by design; monitors of other resources keep the
+   * per-series behaviour.
+   *
+   * The caller skips CREATION only: the monitor's status timeline and the
+   * resolve path for already-open incidents and alerts are untouched, the
+   * same contract as the per-series set and dependency suppression.
+   *
+   * Costs nothing for a monitor that names no database and has no
+   * per-series matches, one query (the ongoing events, shared by both
+   * halves) otherwise, and the monitor's resource lookups only while a
+   * database of the project is under maintenance.
+   */
+  @CaptureSpan()
+  public static async getMaintenanceSuppression(input: {
+    monitor: Monitor;
+    matchesPerSeries?: Array<PerSeriesCriteriaMatch> | undefined;
+  }): Promise<MonitorMaintenanceSuppressionResult> {
+    const result: MonitorMaintenanceSuppressionResult = {
+      suppressedSeriesFingerprints: new Set<string>(),
+      suppressingDatabaseServerIds: [],
+    };
+
+    const projectId: ObjectID | undefined = input.monitor.projectId;
+
+    if (!projectId) {
+      return result;
+    }
+
+    const hasSeries: boolean = Boolean(
+      input.matchesPerSeries && input.matchesPerSeries.length > 0,
+    );
+
+    const namesDatabase: boolean = this.monitorNamesDatabase(input.monitor);
+
+    if (!hasSeries && !namesDatabase) {
+      return result;
+    }
+
+    const maintained: MaintainedResourceKeys =
+      await this.getResourcesUnderOngoingMaintenance(projectId);
+
+    if (!this.hasAnyMaintainedResource(maintained)) {
+      return result;
+    }
+
+    if (hasSeries) {
+      result.suppressedSeriesFingerprints =
+        this.getSuppressedFingerprintsForMaintainedResources({
+          matchesPerSeries: input.matchesPerSeries || [],
+          maintained,
+        });
+    }
+
+    if (namesDatabase && maintained.databaseServers.ids.size > 0) {
+      /*
+       * Resolved rather than read raw: an id or endpoint that names no
+       * database of THIS project (deleted, mistyped, another tenant's)
+       * silences nothing. The resolver never throws — a failed lookup
+       * resolves to nothing, which keeps the monitor alerting.
+       */
+      const resolved: SeriesResolvedResourceIds =
+        await MonitorResourceContextUtil.resolveResourceContextForMonitor({
+          monitor: input.monitor,
+        });
+
+      result.suppressingDatabaseServerIds =
+        this.getSuppressingDatabaseServerIds({
+          resolved: resolved,
+          maintained: maintained,
+        });
+    }
+
+    return result;
+  }
+
+  public static isMonitorSuppressed(
+    result: MonitorMaintenanceSuppressionResult,
+  ): boolean {
+    return result.suppressingDatabaseServerIds.length > 0;
+  }
+
+  /*
+   * Pure decision step of the whole-monitor half, split out so it can be
+   * unit tested without a database: the databases the monitor names, when
+   * it names at least one and every resource it names (of every type) is
+   * under an ongoing maintenance window; otherwise none.
+   */
+  public static getSuppressingDatabaseServerIds(input: {
+    resolved: SeriesResolvedResourceIds;
+    maintained: MaintainedResourceKeys;
+  }): Array<string> {
+    const resolved: SeriesResolvedResourceIds = input.resolved;
+    const maintained: MaintainedResourceKeys = input.maintained;
+
+    if (resolved.databaseServerIds.length === 0) {
+      return [];
+    }
+
+    const namedByType: Array<{ ids: Array<string>; keys: ResourceKeySet }> = [
+      { ids: resolved.hostIds, keys: maintained.hosts },
+      { ids: resolved.dockerHostIds, keys: maintained.dockerHosts },
+      { ids: resolved.podmanHostIds, keys: maintained.podmanHosts },
+      {
+        ids: resolved.kubernetesClusterIds,
+        keys: maintained.kubernetesClusters,
+      },
+      { ids: resolved.serviceIds, keys: maintained.services },
+      { ids: resolved.proxmoxClusterIds, keys: maintained.proxmoxClusters },
+      { ids: resolved.vmwareVCenterIds, keys: maintained.vmwareVCenters },
+      { ids: resolved.cephClusterIds, keys: maintained.cephClusters },
+      {
+        ids: resolved.dockerSwarmClusterIds,
+        keys: maintained.dockerSwarmClusters,
+      },
+      { ids: resolved.iotFleetIds, keys: maintained.iotFleets },
+      { ids: resolved.databaseServerIds, keys: maintained.databaseServers },
+    ];
+
+    for (const named of namedByType) {
+      for (const id of named.ids) {
+        if (!named.keys.ids.has(id)) {
+          return [];
+        }
+      }
+    }
+
+    return [...resolved.databaseServerIds];
+  }
+
+  /*
+   * Record, on the monitor's evaluation summary, what one matching
+   * criteria did NOT create because the monitor is silenced as a whole —
+   * the counterpart of the "suppressed by scheduled maintenance" events
+   * the creators record per series. Only what the criteria would have
+   * created is recorded.
+   */
+  public static recordMonitorSuppressed(input: {
+    evaluationSummary?: MonitorEvaluationSummary | undefined;
+    criteriaInstance: MonitorCriteriaInstance;
+    suppression: MonitorMaintenanceSuppressionResult;
+  }): void {
+    const databases: string =
+      input.suppression.suppressingDatabaseServerIds.length === 1
+        ? `the database ${input.suppression.suppressingDatabaseServerIds[0]} this monitor watches is`
+        : `the databases ${input.suppression.suppressingDatabaseServerIds.join(", ")} this monitor watches are`;
+
+    logger.debug(
+      `Skipping incident and alert creation for criteria ${input.criteriaInstance.data?.id}: ${databases} under an active scheduled maintenance window.`,
+    );
+
+    if (!input.evaluationSummary) {
+      return;
+    }
+
+    if (input.criteriaInstance.data?.createIncidents) {
+      input.evaluationSummary.events.push({
+        type: "incident-skipped",
+        title: "Incident suppressed by scheduled maintenance",
+        message: `Skipped creating incidents because ${databases} under an active scheduled maintenance window. This monitor's status still updates normally; open incidents still auto-resolve.`,
+        relatedCriteriaId: input.criteriaInstance.data?.id,
+        at: OneUptimeDate.getCurrentDate(),
+      });
+    }
+
+    if (input.criteriaInstance.data?.createAlerts) {
+      input.evaluationSummary.events.push({
+        type: "alert-skipped",
+        title: "Alert suppressed by scheduled maintenance",
+        message: `Skipped creating alerts because ${databases} under an active scheduled maintenance window. This monitor's status still updates normally; open alerts still auto-resolve.`,
+        relatedCriteriaId: input.criteriaInstance.data?.id,
+        at: OneUptimeDate.getCurrentDate(),
+      });
+    }
+  }
+
+  /*
+   * Whether the monitor's own configuration names a database — the pure,
+   * query-free gate in front of the maintenance lookup. The step JSON is
+   * not schema-checked, so an unreadable one names nothing rather than
+   * throwing out of the evaluation.
+   */
+  private static monitorNamesDatabase(monitor: Monitor): boolean {
+    try {
+      const refs: SeriesResourceRefs =
+        MonitorStepResourceIdentity.extractResourceRefsFromMonitor({
+          monitor: monitor,
+        });
+
+      return (
+        refs.databaseServerIds.length > 0 ||
+        refs.databaseServerEndpoints.length > 0
+      );
+    } catch (err) {
+      logger.error(
+        `Failed to read the resources monitor ${monitor.id?.toString()} names for maintenance suppression: ${err}`,
+      );
+      return false;
+    }
+  }
+
   /*
    * Pure matching step, split out from the query so it can be unit
    * tested without a database. For each series, pull the resource
@@ -163,7 +414,11 @@ export default class MonitorMaintenanceSuppression {
         ) ||
         this.intersects(refs.iotFleetNames, input.maintained.iotFleets.names) ||
         this.intersects(refs.serviceIds, input.maintained.services.ids) ||
-        this.intersects(refs.serviceNames, input.maintained.services.names);
+        this.intersects(refs.serviceNames, input.maintained.services.names) ||
+        this.intersects(
+          refs.databaseServerIds,
+          input.maintained.databaseServers.ids,
+        );
 
       if (isUnderMaintenance) {
         suppressed.add(series.fingerprint);
@@ -205,16 +460,17 @@ export default class MonitorMaintenanceSuppression {
       maintained.iotFleets.ids.size > 0 ||
       maintained.iotFleets.names.size > 0 ||
       maintained.services.ids.size > 0 ||
-      maintained.services.names.size > 0
+      maintained.services.names.size > 0 ||
+      maintained.databaseServers.ids.size > 0
     );
   }
 
   /*
    * Collect the ids + identifiers of every Host / DockerHost /
    * PodmanHost / KubernetesCluster / ProxmoxCluster / VMwareVCenter /
-   * CephCluster / DockerSwarmCluster / IoTFleet / Service attached to an ongoing
-   * maintenance event in this project. Monitors attached
-   * to the event are intentionally not collected here — those are
+   * CephCluster / DockerSwarmCluster / IoTFleet / Service / DatabaseServer
+   * attached to an ongoing maintenance event in this project. Monitors
+   * attached to the event are intentionally not collected here — those are
    * already handled upstream by the whole-monitor disable flag, which
    * short-circuits evaluation before we ever reach per-series creation.
    */
@@ -235,6 +491,7 @@ export default class MonitorMaintenanceSuppression {
       },
       iotFleets: { ids: new Set<string>(), names: new Set<string>() },
       services: { ids: new Set<string>(), names: new Set<string>() },
+      databaseServers: { ids: new Set<string>(), names: new Set<string>() },
     };
 
     const ongoingEvents: Array<ScheduledMaintenance> =
@@ -257,6 +514,7 @@ export default class MonitorMaintenanceSuppression {
           dockerSwarmClusters: { _id: true, name: true },
           iotFleets: { _id: true, name: true },
           services: { _id: true, name: true },
+          databaseServers: { _id: true },
         },
         skip: 0,
         limit: LIMIT_PER_PROJECT,
@@ -320,6 +578,10 @@ export default class MonitorMaintenanceSuppression {
       for (const service of event.services || []) {
         this.addKey(maintained.services, service._id, service.name);
       }
+      for (const databaseServer of event.databaseServers || []) {
+        // Id only — the display name is not an identity (see the interface).
+        this.addKey(maintained.databaseServers, databaseServer._id, undefined);
+      }
     }
 
     return maintained;

package/Server/Utils/Monitor/MonitorResource.ts

@@ -51,7 +51,9 @@ import MonitorAlert from "./MonitorAlert";
 import MonitorSummaryCapture from "./MonitorSummaryCapture";
 import MonitorSummarySnapshot from "../../../Types/Monitor/MonitorSummarySnapshot";
 import IncomingRequestIncidentGrouping from "./IncomingRequestIncidentGrouping";
-import MonitorMaintenanceSuppression from "./MonitorMaintenanceSuppression";
+import MonitorMaintenanceSuppression, {
+  MonitorMaintenanceSuppressionResult,
+} from "./MonitorMaintenanceSuppression";
 import MonitorDependencySuppression, {
   DependencySuppressionResult,
 } from "./MonitorDependencySuppression";
@@ -1032,17 +1034,26 @@ export default class MonitorResourceUtil {
         }
 
         /*
-         * For grouped metric monitors, work out which breaching series
-         * belong to a resource that is currently inside an ongoing
-         * scheduled maintenance window. Those series are suppressed
-         * below so the monitor keeps alerting on the rest. Computed once
-         * and shared by both the incident and alert paths. Cheap on the
-         * common path: no per-series matches, or no ongoing maintenance,
-         * returns an empty set after at most one query.
+         * Scheduled-maintenance suppression, computed once and shared by
+         * both the incident and alert paths:
+         *
+         *   - per series: for grouped monitors, the breaching series whose
+         *     own resource is inside an ongoing maintenance window. Those
+         *     series are suppressed below so the monitor keeps alerting on
+         *     the rest.
+         *   - whole monitor: a monitor whose own configuration names a
+         *     database under maintenance (and nothing outside the window) —
+         *     every recommended database monitor is ungrouped, so it has no
+         *     series for the first half to match. Creation is skipped in the
+         *     loop below.
+         *
+         * Cheap on the common path: a monitor with no per-series matches
+         * that names no database costs nothing, and no ongoing maintenance
+         * costs one query.
          */
-        const suppressedSeriesFingerprints: Set<string> =
-          await MonitorMaintenanceSuppression.getSuppressedSeriesFingerprints({
-            projectId: monitor.projectId!,
+        const maintenanceSuppression: MonitorMaintenanceSuppressionResult =
+          await MonitorMaintenanceSuppression.getMaintenanceSuppression({
+            monitor: monitor,
             matchesPerSeries: isPerSeriesEvaluation
               ? criteriaFanOut.flatMap((matched: MatchedCriteriaResult) => {
                   return matched.perSeriesMatches;
@@ -1050,6 +1061,9 @@ export default class MonitorResourceUtil {
               : undefined,
           });
 
+        const suppressedSeriesFingerprints: Set<string> =
+          maintenanceSuppression.suppressedSeriesFingerprints;
+
         /*
          * Alert-dependency suppression: if any parent monitor this monitor
          * depends on is currently in a suppressing status (offline by
@@ -1119,6 +1133,7 @@ export default class MonitorResourceUtil {
               ? breachingSeriesFingerprintsByCriteriaId
               : undefined,
             disableSeriesAbsenceResolution,
+            criteriaInstancesById: criteriaInstanceMap,
           });
 
         const openAlerts: Array<Alert> =
@@ -1134,6 +1149,7 @@ export default class MonitorResourceUtil {
               ? breachingSeriesFingerprintsByCriteriaId
               : undefined,
             disableSeriesAbsenceResolution,
+            criteriaInstancesById: criteriaInstanceAlertMap,
           });
 
         /*
@@ -1155,6 +1171,26 @@ export default class MonitorResourceUtil {
             continue;
           }
 
+          /*
+           * The monitor watches a database that is inside an ongoing
+           * scheduled maintenance window: create nothing for this
+           * criteria. The status timeline was updated above and the
+           * resolve pass has already run, so only creation is silenced —
+           * the same contract as per-series and dependency suppression.
+           */
+          if (
+            MonitorMaintenanceSuppression.isMonitorSuppressed(
+              maintenanceSuppression,
+            )
+          ) {
+            MonitorMaintenanceSuppression.recordMonitorSuppressed({
+              evaluationSummary: evaluationSummary,
+              criteriaInstance: criteriaInstance,
+              suppression: maintenanceSuppression,
+            });
+            continue;
+          }
+
           await MonitorIncident.criteriaMetCreateIncidentsAndUpdateMonitorStatus(
             {
               monitor: monitor,

package/Server/Utils/Monitor/MonitorResourceContext.ts

@@ -104,6 +104,7 @@ export default class MonitorResourceContextUtil {
       cephClusterIds: [],
       dockerSwarmClusterIds: [],
       iotFleetIds: [],
+      databaseServerIds: [],
     };
   }
 }