@oneuptime/common 14.0.8 → 14.0.9

package/Server/Services/IncidentService.ts

@@ -171,10 +171,28 @@ interface AddedStatusPagesDecision {
 
 // key is incidentId for this dictionary.
 type UpdateCarryForward = Dictionary<{
+  /*
+   * The monitors the update takes off and puts on the incident. Both are
+   * empty unless the update sends the monitor list.
+   */
   monitorsRemoved: Array<Monitor>;
   monitorsAdded: Array<Monitor>;
+  /*
+   * Whether the incident was resolved before the update, which decides if
+   * the monitors taken off it are restored. Read before the write, since the
+   * same update may resolve or reopen it, and only when a monitor is taken
+   * off (undefined otherwise).
+   */
+  isResolvedBeforeUpdate?: boolean | undefined;
+  // The monitor status the incident put its monitors in before the update.
   oldChangeMonitorStatusIdTo: ObjectID | undefined;
+  // The monitor status the update writes; undefined when it writes none.
   newMonitorChangeStatusIdTo: ObjectID | undefined;
+  /*
+   * The update sets the incident's monitor status to nothing, so the
+   * incident no longer puts its monitors, old or added, in any status.
+   */
+  isChangeMonitorStatusToCleared?: boolean | undefined;
   statusPageScopeChange?: StatusPageScopeCarryForward | undefined;
 }>;
 
@@ -523,9 +541,10 @@ export class Service extends DatabaseService<Model> {
     updateBy: UpdateBy<Model>,
   ): Promise<OnUpdate<Model>> {
     /*
-     * get monitors for this incident.
-     * if the monitors are removed then change them to operational state.
-     * then change all of the monitors in this incident to the changeMonitorStatusToId.
+     * Records which monitors the update takes off and puts on each incident,
+     * the monitor status it writes, and whether the incident was resolved
+     * before it, for onUpdateSuccess to decide what that does to the
+     * monitors (updateMonitorsForIncidentEdit).
      */
 
     updateBy.query = applyIncidentSelfPrivacyFilter(
@@ -554,8 +573,19 @@ export class Service extends DatabaseService<Model> {
 
     const carryForward: UpdateCarryForward = {};
 
+    const data: Dictionary<unknown> = updateBy.data as Dictionary<unknown>;
+
+    /*
+     * Only an update that sends the monitor list changes which monitors the
+     * incident holds. One that sends just the monitor status (the API, the
+     * CLI or a workflow) takes no monitor off: reading the missing list as
+     * empty would restore every monitor, and walking it threw. A list set to
+     * null is sent: TypeORM clears a many-to-many set to null, like [].
+     */
+    const isMonitorListUpdated: boolean = data["monitors"] !== undefined;
+
     if (
-      updateBy.data.monitors ||
+      isMonitorListUpdated ||
       updateBy.data.changeMonitorStatusTo ||
       updateBy.data.changeMonitorStatusToId
     ) {
@@ -573,64 +603,69 @@ export class Service extends DatabaseService<Model> {
         props: updateBy.props,
       });
 
-      for (const incident of incidentsToUpdate) {
-        carryForward[incident.id!.toString()] = {
-          monitorsRemoved: [],
-          monitorsAdded: [],
-          oldChangeMonitorStatusIdTo: incident.changeMonitorStatusToId,
-          newMonitorChangeStatusIdTo:
-            (updateBy.data.changeMonitorStatusToId as ObjectID) ||
-            (updateBy.data.changeMonitorStatusTo as unknown as MonitorStatus)
-              ?._id ||
-            undefined,
-        };
-
-        for (const monitor of incident.monitors || []) {
-          // check if this monitor is actually removed.
-          let isRemoved: boolean = true;
-
-          for (const updatedMonitor of updateBy.data
-            ?.monitors as unknown as Array<Monitor>) {
-            if (
-              updatedMonitor._id &&
-              updatedMonitor._id.toString() === monitor._id?.toString()
-            ) {
-              isRemoved = false;
-              break;
-            }
-          }
+      /*
+       * The monitor status the update writes, in any shape the API accepts
+       * (an id, a bare uuid string or a relation object), so the status an
+       * update writes is never mistaken for one it leaves alone.
+       */
+      const monitorStatusIdInUpdate: ObjectID | string | undefined =
+        resolveReferenceId(data["changeMonitorStatusToId"]) ||
+        resolveReferenceId(data["changeMonitorStatusTo"]);
+
+      const newMonitorChangeStatusIdTo: ObjectID | undefined =
+        monitorStatusIdInUpdate
+          ? new ObjectID(monitorStatusIdInUpdate.toString())
+          : undefined;
+
+      const isChangeMonitorStatusToCleared: boolean =
+        !newMonitorChangeStatusIdTo &&
+        (data["changeMonitorStatusToId"] === null ||
+          data["changeMonitorStatusTo"] === null);
+
+      const monitorIdsAfterUpdate: Array<string> = isMonitorListUpdated
+        ? this.getMonitorIdsInUpdate(data["monitors"])
+        : [];
 
-          if (isRemoved) {
-            carryForward[incident.id!.toString()]?.monitorsRemoved?.push(
-              monitor,
-            );
-          }
-        }
+      for (const incident of incidentsToUpdate) {
+        const storedMonitors: Array<Monitor> = incident.monitors || [];
 
-        if (updateBy.data.monitors && updateBy.data.monitors.length > 0) {
-          for (const monitor of updateBy.data
-            ?.monitors as unknown as Array<Monitor>) {
-            // check if this monitor is actually added.
-            let isAdded: boolean = true;
-
-            for (const existingMonitor of incident.monitors || []) {
-              if (
-                existingMonitor._id &&
-                existingMonitor._id.toString() === monitor._id?.toString()
-              ) {
-                isAdded = false;
-                break;
-              }
-            }
+        const storedMonitorIds: Array<string> = storedMonitors.map(
+          (monitor: Monitor): string => {
+            return this.getMonitorIdForComparison(monitor);
+          },
+        );
 
-            if (isAdded) {
-              // this monitor is added.
-              carryForward[incident.id!.toString()]?.monitorsAdded?.push(
-                monitor,
+        const monitorsRemoved: Array<Monitor> = isMonitorListUpdated
+          ? storedMonitors.filter((monitor: Monitor): boolean => {
+              return !monitorIdsAfterUpdate.includes(
+                this.getMonitorIdForComparison(monitor),
               );
-            }
-          }
-        }
+            })
+          : [];
+
+        carryForward[incident.id!.toString()] = {
+          monitorsRemoved: monitorsRemoved,
+          /*
+           * Read now, before the write: an update that resolves the incident
+           * and takes a monitor off must still restore that monitor, and one
+           * that reopens a resolved incident must not restore it twice.
+           * After the write, the state no longer tells these cases apart.
+           */
+          isResolvedBeforeUpdate:
+            monitorsRemoved.length > 0
+              ? await this.isIncidentResolved({ incidentId: incident.id! })
+              : undefined,
+          monitorsAdded: monitorIdsAfterUpdate
+            .filter((monitorId: string): boolean => {
+              return !storedMonitorIds.includes(monitorId);
+            })
+            .map((monitorId: string): Monitor => {
+              return new Monitor(new ObjectID(monitorId));
+            }),
+          oldChangeMonitorStatusIdTo: incident.changeMonitorStatusToId,
+          newMonitorChangeStatusIdTo: newMonitorChangeStatusIdTo,
+          isChangeMonitorStatusToCleared: isChangeMonitorStatusToCleared,
+        };
       }
     }
 
@@ -705,6 +740,35 @@ export class Service extends DatabaseService<Model> {
     };
   }
 
+  /*
+   * The ids of the monitors an update's list holds, deduplicated and
+   * lowercased like getMonitorIdForComparison. The list reaches the hook as
+   * models, `{ _id }` objects, ObjectIDs or bare uuid strings (API update),
+   * and comparing only `_id` read every bare id as a monitor taken off and
+   * put back on again.
+   */
+  private getMonitorIdsInUpdate(monitors: unknown): Array<string> {
+    const monitorIds: Array<string> = [];
+
+    for (const monitorId of resolveReferenceIds(monitors)) {
+      const normalizedMonitorId: string = monitorId
+        .toString()
+        .trim()
+        .toLowerCase();
+
+      if (!monitorIds.includes(normalizedMonitorId)) {
+        monitorIds.push(normalizedMonitorId);
+      }
+    }
+
+    return monitorIds;
+  }
+
+  // A monitor's id as getMonitorIdsInUpdate writes it, for comparing the two.
+  private getMonitorIdForComparison(monitor: Monitor): string {
+    return (monitor._id?.toString() || "").trim().toLowerCase();
+  }
+
   /*
    * The incidents an update will write to, as they are stored, for the hooks
    * below that decide what else joins the update.
@@ -4352,12 +4416,6 @@ ${incidentSeverity.name}
                   },
                 });
 
-              // change these monitors back to operational state.
-              await this.markMonitorsActiveForMonitoring(
-                projectId!,
-                incidentCarryForward.monitorsRemoved,
-              );
-
               feedInfoInMarkdown += `\n\n**🗑️ Monitors Removed**:\n`;
 
               for (const monitor of monitorsRemoved) {
@@ -4417,9 +4475,11 @@ ${incidentSeverity.name}
               change: incidentCarryForward.statusPageScopeChange,
             });
 
+            // Saving the status the incident already had changes nothing.
             if (
               incidentCarryForward.oldChangeMonitorStatusIdTo &&
-              incidentCarryForward.newMonitorChangeStatusIdTo
+              incidentCarryForward.newMonitorChangeStatusIdTo &&
+              this.isMonitorStatusChangedByUpdate(incidentCarryForward)
             ) {
               const oldMonitorStatus: MonitorStatus | null =
                 await MonitorStatusService.findOneBy({
@@ -4453,45 +4513,13 @@ ${incidentSeverity.name}
               }
             }
 
-            const changeNewMonitorStatusTo: ObjectID | undefined =
-              incidentCarryForward.newMonitorChangeStatusIdTo ||
-              incidentCarryForward.oldChangeMonitorStatusIdTo;
-
-            if (incidentCarryForward.monitorsAdded?.length > 0) {
-              await this.disableActiveMonitoringIfManualIncident(incidentId);
-            }
-
-            if (changeNewMonitorStatusTo) {
-              const incident: Model | null = await this.findOneById({
-                id: incidentId,
-                select: {
-                  projectId: true,
-                  monitors: {
-                    _id: true,
-                  },
-                },
-                props: {
-                  isRoot: true,
-                },
-              });
-
-              const monitorsForThisIncident: Array<Monitor> =
-                incident?.monitors || [];
-
-              await MonitorService.changeMonitorStatus(
-                projectId!,
-                monitorsForThisIncident.map((monitor: Monitor) => {
-                  return new ObjectID(monitor._id?.toString() || "");
-                }),
-                changeNewMonitorStatusTo,
-                true, // notifyMonitorOwners
-                "Status was changed because Incident " +
-                  incidentNumberDisplay +
-                  " was updated.",
-                undefined,
-                onUpdate.updateBy.props,
-              );
-            }
+            await this.updateMonitorsForIncidentEdit({
+              projectId: projectId,
+              incidentId: incidentId,
+              incidentNumberDisplay: incidentNumberDisplay,
+              carryForward: incidentCarryForward,
+              props: onUpdate.updateBy.props,
+            });
           }
         }
 
@@ -4514,6 +4542,154 @@ ${incidentSeverity.name}
     return onUpdate;
   }
 
+  /*
+   * Whether an update puts the incident's monitors in a different status
+   * from the one it held: it writes a status, and not the same one again.
+   */
+  private isMonitorStatusChangedByUpdate(
+    carryForward: UpdateCarryForward[string],
+  ): boolean {
+    if (!carryForward.newMonitorChangeStatusIdTo) {
+      return false;
+    }
+
+    return (
+      carryForward.newMonitorChangeStatusIdTo.toString().toLowerCase() !==
+      (carryForward.oldChangeMonitorStatusIdTo?.toString() || "").toLowerCase()
+    );
+  }
+
+  /*
+   * What an edit of an incident's monitors, or of the status it puts them
+   * in, does to the monitors themselves. The feed item records the edit
+   * either way; this only decides which monitors change.
+   *
+   * The monitors taken off the incident are restored, unless another open
+   * incident still holds them (markMonitorsActiveForMonitoring), when the
+   * incident was open before the update. That includes an update that
+   * resolves it as well: resolving restores only the monitors it still
+   * holds (IncidentStateTimelineService), so this is the only restore the
+   * ones taken off get. A monitor taken off an incident that was resolved
+   * already was restored when it resolved, and restoring it again could
+   * overwrite a status set since (maintenance, or a manual monitor's status
+   * set by hand).
+   *
+   * A resolved incident, as it stands after the update, does nothing else
+   * to its monitors. Nothing would clear what it did: an added monitor of a
+   * manual incident would never be probed again, and the incident's status
+   * would sit on its monitors as false downtime.
+   *
+   * An open incident:
+   * - stops probing the monitors of a manual incident when one is added,
+   *   as creating it did (disableActiveMonitoringIfManualIncident);
+   * - puts the monitors added in its status, and all of its monitors only
+   *   when the edit changes that status. Every save used to put all of them
+   *   in it again, overwriting a status a probe had set since.
+   */
+  private async updateMonitorsForIncidentEdit(data: {
+    projectId: ObjectID;
+    incidentId: ObjectID;
+    incidentNumberDisplay: string;
+    carryForward: UpdateCarryForward[string];
+    props: DatabaseCommonInteractionProps;
+  }): Promise<void> {
+    const carryForward: UpdateCarryForward[string] = data.carryForward;
+
+    if (
+      carryForward.monitorsRemoved.length > 0 &&
+      !carryForward.isResolvedBeforeUpdate
+    ) {
+      // change these monitors back to operational state.
+      await this.markMonitorsActiveForMonitoring(
+        data.projectId,
+        carryForward.monitorsRemoved,
+      );
+    }
+
+    const isMonitorStatusChanged: boolean =
+      this.isMonitorStatusChangedByUpdate(carryForward);
+
+    if (carryForward.monitorsAdded.length === 0 && !isMonitorStatusChanged) {
+      return;
+    }
+
+    /*
+     * Read after the update is written, so an edit that resolves the
+     * incident as well counts as resolved: changing the state restored the
+     * monitors it holds, the added ones included.
+     */
+    if (await this.isIncidentResolved({ incidentId: data.incidentId })) {
+      return;
+    }
+
+    if (carryForward.monitorsAdded.length > 0) {
+      await this.disableActiveMonitoringIfManualIncident(data.incidentId);
+    }
+
+    // The status the incident puts its monitors in after the update.
+    const monitorStatusIdToApply: ObjectID | undefined =
+      carryForward.isChangeMonitorStatusToCleared
+        ? undefined
+        : carryForward.newMonitorChangeStatusIdTo ||
+          carryForward.oldChangeMonitorStatusIdTo;
+
+    if (
+      !monitorStatusIdToApply ||
+      (!isMonitorStatusChanged && carryForward.monitorsAdded.length === 0)
+    ) {
+      return;
+    }
+
+    const addedMonitorIds: Array<string> = carryForward.monitorsAdded.map(
+      (monitor: Monitor): string => {
+        return this.getMonitorIdForComparison(monitor);
+      },
+    );
+
+    /*
+     * The monitors the incident holds now, so a status is only ever put on
+     * a monitor the update left on it.
+     */
+    const incident: Model | null = await this.findOneById({
+      id: data.incidentId,
+      select: {
+        monitors: {
+          _id: true,
+        },
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    const monitorIdsToChange: Array<ObjectID> = (incident?.monitors || [])
+      .filter((monitor: Monitor): boolean => {
+        return (
+          isMonitorStatusChanged ||
+          addedMonitorIds.includes(this.getMonitorIdForComparison(monitor))
+        );
+      })
+      .map((monitor: Monitor): ObjectID => {
+        return new ObjectID(monitor._id?.toString() || "");
+      });
+
+    if (monitorIdsToChange.length === 0) {
+      return;
+    }
+
+    await MonitorService.changeMonitorStatus(
+      data.projectId,
+      monitorIdsToChange,
+      monitorStatusIdToApply,
+      true, // notifyMonitorOwners
+      "Status was changed because Incident " +
+        data.incidentNumberDisplay +
+        " was updated.",
+      undefined,
+      data.props,
+    );
+  }
+
   /*
    * The part of an update's feed item that records a change of the status
    * pages the incident is limited to: the pages added and removed, whether

package/Server/Services/IncidentSeverityService.ts

@@ -5,10 +5,12 @@ import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
 import QueryHelper from "../Types/Database/QueryHelper";
 import UpdateBy from "../Types/Database/UpdateBy";
 import DatabaseService from "./DatabaseService";
+import TeamComplianceSettingService from "./TeamComplianceSettingService";
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
 import LIMIT_MAX from "../../Types/Database/LimitMax";
 import BadDataException from "../../Types/Exception/BadDataException";
 import ObjectID from "../../Types/ObjectID";
+import { ComplianceSeverityKind } from "../../Types/Team/ComplianceRule";
 import Model from "../../Models/DatabaseModels/IncidentSeverity";
 import Queue, { QueueName } from "../Infrastructure/Queue";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
@@ -22,6 +24,14 @@ import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 const BACKFILL_NOTIFICATION_RULES_JOB_NAME: string =
   "OnCallDutyPolicy:BackfillNotificationRulesForNewSeverities";
 
+// What a delete carries from onBeforeDelete to onDeleteSuccess.
+interface SeverityDeleteCarryForward {
+  // The deleted severity, for re-ranking the rest; signed-in deletes only.
+  incidentSeverity: Model | null;
+  // Team compliance rules scoped only to severities this delete removes.
+  complianceSettingIds: Array<string>;
+}
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
@@ -126,24 +136,38 @@ export class Service extends DatabaseService<Model> {
       );
     }
 
-    let incidentSeverity: Model | null = null;
+    /*
+     * The severities this delete is about to remove, read while the team
+     * compliance rules scoped to them still say so: the join rows cascade
+     * away with the severity. See TeamComplianceSettingService.
+     */
+    const severities: Array<Model> = await this.findBy({
+      query: deleteBy.query,
+      select: {
+        _id: true,
+        order: true,
+        projectId: true,
+      },
+      limit: deleteBy.limit,
+      skip: deleteBy.skip,
+      props: {
+        isRoot: true,
+      },
+    });
 
-    if (!deleteBy.props.isRoot) {
-      incidentSeverity = await this.findOneBy({
-        query: deleteBy.query,
-        props: {
-          isRoot: true,
-        },
-        select: {
-          order: true,
-          projectId: true,
-        },
-      });
-    }
+    const carryForward: SeverityDeleteCarryForward = {
+      // Only a signed-in delete re-ranks the severities that remain.
+      incidentSeverity: deleteBy.props.isRoot ? null : severities[0] || null,
+      complianceSettingIds:
+        await TeamComplianceSettingService.getRulesScopedToAnyOf({
+          severityKind: ComplianceSeverityKind.Incident,
+          severities: severities,
+        }),
+    };
 
     return {
       deleteBy,
-      carryForward: incidentSeverity,
+      carryForward: carryForward,
     };
   }
 
@@ -153,7 +177,10 @@ export class Service extends DatabaseService<Model> {
     _itemIdsBeforeDelete: ObjectID[],
   ): Promise<OnDelete<Model>> {
     const deleteBy: DeleteBy<Model> = onDelete.deleteBy;
-    const incidentSeverity: Model | null = onDelete.carryForward;
+    const carryForward: SeverityDeleteCarryForward | null =
+      (onDelete.carryForward as SeverityDeleteCarryForward | null) || null;
+    const incidentSeverity: Model | null =
+      carryForward?.incidentSeverity || null;
 
     if (!deleteBy.props.isRoot && incidentSeverity) {
       if (
@@ -169,6 +196,30 @@ export class Service extends DatabaseService<Model> {
       }
     }
 
+    /*
+     * The severity is gone, so a compliance rule that was scoped only to it
+     * (and to severities other deletes have already removed) now reads as a
+     * rule for every incident severity. Pause and mark it instead; a rule with
+     * another severity left is left alone. The delete has already happened,
+     * so a failure here is logged rather than reported as a failed delete.
+     */
+    const complianceSettingIds: Array<string> =
+      carryForward?.complianceSettingIds || [];
+
+    if (complianceSettingIds.length > 0) {
+      try {
+        await TeamComplianceSettingService.pauseRulesLeftWithoutSeverities({
+          severityKind: ComplianceSeverityKind.Incident,
+          settingIds: complianceSettingIds,
+        });
+      } catch (err) {
+        logger.error(
+          "Could not pause the team compliance rules left without a severity by an incident severity delete. They now check every incident severity until they are edited.",
+        );
+        logger.error(err);
+      }
+    }
+
     return {
       deleteBy: deleteBy,
       carryForward: null,