@oneuptime/common 14.0.8 → 14.0.10

package/Server/Services/HostService.ts

@@ -3,7 +3,15 @@ import HostLabelRuleEngineService from "./HostLabelRuleEngineService";
 import HostOwnerRuleEngineService from "./HostOwnerRuleEngineService";
 import Model from "../../Models/DatabaseModels/Host";
 import Label from "../../Models/DatabaseModels/Label";
-import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
+import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
+import CreateBy from "../Types/Database/CreateBy";
+import DeleteBy from "../Types/Database/DeleteBy";
+import UpdateBy from "../Types/Database/UpdateBy";
+import ResourceAiAccessSettings, {
+  ResourceAiAccessFeedItem,
+} from "../Utils/AI/ResourceAccess/ResourceAiAccessSettings";
+import ResourceAiDeleteCleanup from "../Utils/AI/ResourceAccess/ResourceAiDeleteCleanup";
+import AiResourceType from "../../Types/ResourceAiAgent/AiResourceType";
 import HostFeedService from "./HostFeedService";
 import { HostFeedEventType } from "../../Models/DatabaseModels/HostFeed";
 import ResourceFeedUtil from "../Utils/ResourceFeed/ResourceFeedUtil";
@@ -119,6 +127,13 @@ export class Service extends DatabaseService<Model> {
   public async findOrCreateByHostIdentifier(data: {
     projectId: ObjectID;
     hostIdentifier: string;
+    /*
+     * Skip the in-process memo and look the row up (the memo is then
+     * refreshed with what was found or created). For a caller that learned
+     * the memoed id names a host deleted since: the memo is never
+     * invalidated by a delete, so it keeps that id for up to its TTL.
+     */
+    bypassMemo?: boolean | undefined;
   }): Promise<Model> {
     /*
      * Canonicalize the identifier (trim + lowercase, matching
@@ -141,8 +156,9 @@ export class Service extends DatabaseService<Model> {
      * the resolving call performed within the last minute.
      */
     const memoKey: string = `${data.projectId.toString()}:${hostIdentifier}`;
-    const memoedResolution: HostResolution | undefined =
-      hostResolutionInProcessMemo.get(memoKey);
+    const memoedResolution: HostResolution | undefined = data.bypassMemo
+      ? undefined
+      : hostResolutionInProcessMemo.get(memoKey);
     if (memoedResolution) {
       const memoedHost: Model = new Model();
       memoedHost._id = memoedResolution.hostIdStr;
@@ -464,8 +480,14 @@ export class Service extends DatabaseService<Model> {
       cloudAccountId: extra?.cloudAccountId ?? null,
     };
 
+    /*
+     * Change detection only, but the payload carries the cloud account id,
+     * so it is hashed with SHA-256 rather than SHA-1. The value lives in the
+     * heartbeat throttle's cache, so the switch costs each host one extra
+     * metadata write after a deploy, nothing more.
+     */
     return crypto
-      .createHash("sha1")
+      .createHash("sha256")
       .update(JSON.stringify(normalized))
       .digest("hex");
   }
@@ -739,6 +761,79 @@ export class Service extends DatabaseService<Model> {
     });
   }
 
+  /*
+   * Deleting a host settles its in-flight AI remediation rounds and removes its
+   * resource AI agent — read before the delete, done after it for the hosts
+   * actually deleted (ResourceAiDeleteCleanup, shared by every resource AI
+   * agent's resource).
+   */
+  @CaptureSpan()
+  protected override async onBeforeDelete(
+    deleteBy: DeleteBy<Model>,
+  ): Promise<OnDelete<Model>> {
+    return {
+      deleteBy,
+      carryForward: await ResourceAiDeleteCleanup.beforeDelete({
+        resourceType: AiResourceType.Host,
+        service: this,
+        deleteBy,
+      }),
+    };
+  }
+
+  @CaptureSpan()
+  protected override async onDeleteSuccess(
+    onDelete: OnDelete<Model>,
+    deletedItemIds: Array<ObjectID>,
+  ): Promise<OnDelete<Model>> {
+    await ResourceAiDeleteCleanup.afterDelete({
+      resourceType: AiResourceType.Host,
+      onDelete,
+      deletedItemIds,
+    });
+
+    return onDelete;
+  }
+
+  /*
+   * A create is held to the same AI access rules as an update, judged
+   * against the never-configured defaults a new host starts from: an
+   * unusable mode or allowlist is refused for every caller, and the
+   * server-only aiAccess* columns for every caller but root (a master admin
+   * included, whom the create column ACLs never check).
+   */
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    ResourceAiAccessSettings.checkCreate({
+      resourceType: AiResourceType.Host,
+      createBy,
+    });
+
+    return { createBy, carryForward: null };
+  }
+
+  /*
+   * An operator's write of an AI access setting (the investigation switch, the
+   * remediation mode, the command allowlist) is validated and checked against
+   * who may make AI do more on this host — the rules every resource AI agent's
+   * resource shares (ResourceAiAccessSettings).
+   */
+  @CaptureSpan()
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<Model>,
+  ): Promise<OnUpdate<Model>> {
+    return {
+      updateBy,
+      carryForward: await ResourceAiAccessSettings.checkUpdate({
+        resourceType: AiResourceType.Host,
+        service: this,
+        updateBy,
+      }),
+    };
+  }
+
   @CaptureSpan()
   protected override async onUpdateSuccess(
     onUpdate: OnUpdate<Model>,
@@ -750,6 +845,33 @@ export class Service extends DatabaseService<Model> {
       },
     );
 
+    /*
+     * An operator's AI access write: recorded on the feed, and the host marked
+     * AI-configured.
+     */
+    await ResourceAiAccessSettings.afterUpdate({
+      service: this,
+      onUpdate,
+      updatedItemIds,
+      getResourceMarkdownLink: (
+        projectId: ObjectID,
+        hostId: ObjectID,
+      ): Promise<string> => {
+        return this.getHostMarkdownLink(projectId, hostId);
+      },
+      createFeedItem: async (item: ResourceAiAccessFeedItem): Promise<void> => {
+        await HostFeedService.createHostFeedItem({
+          hostId: item.resourceId,
+          projectId: item.projectId,
+          hostFeedEventType: HostFeedEventType.HostUpdated,
+          displayColor: item.displayColor,
+          feedInfoInMarkdown: item.feedInfoInMarkdown,
+          moreInformationInMarkdown: item.moreInformationInMarkdown,
+          userId: item.userId,
+        });
+      },
+    });
+
     return onUpdate;
   }
 

package/Server/Services/IncidentService.ts

@@ -171,10 +171,28 @@ interface AddedStatusPagesDecision {
 
 // key is incidentId for this dictionary.
 type UpdateCarryForward = Dictionary<{
+  /*
+   * The monitors the update takes off and puts on the incident. Both are
+   * empty unless the update sends the monitor list.
+   */
   monitorsRemoved: Array<Monitor>;
   monitorsAdded: Array<Monitor>;
+  /*
+   * Whether the incident was resolved before the update, which decides if
+   * the monitors taken off it are restored. Read before the write, since the
+   * same update may resolve or reopen it, and only when a monitor is taken
+   * off (undefined otherwise).
+   */
+  isResolvedBeforeUpdate?: boolean | undefined;
+  // The monitor status the incident put its monitors in before the update.
   oldChangeMonitorStatusIdTo: ObjectID | undefined;
+  // The monitor status the update writes; undefined when it writes none.
   newMonitorChangeStatusIdTo: ObjectID | undefined;
+  /*
+   * The update sets the incident's monitor status to nothing, so the
+   * incident no longer puts its monitors, old or added, in any status.
+   */
+  isChangeMonitorStatusToCleared?: boolean | undefined;
   statusPageScopeChange?: StatusPageScopeCarryForward | undefined;
 }>;
 
@@ -523,9 +541,10 @@ export class Service extends DatabaseService<Model> {
     updateBy: UpdateBy<Model>,
   ): Promise<OnUpdate<Model>> {
     /*
-     * get monitors for this incident.
-     * if the monitors are removed then change them to operational state.
-     * then change all of the monitors in this incident to the changeMonitorStatusToId.
+     * Records which monitors the update takes off and puts on each incident,
+     * the monitor status it writes, and whether the incident was resolved
+     * before it, for onUpdateSuccess to decide what that does to the
+     * monitors (updateMonitorsForIncidentEdit).
      */
 
     updateBy.query = applyIncidentSelfPrivacyFilter(
@@ -554,8 +573,19 @@ export class Service extends DatabaseService<Model> {
 
     const carryForward: UpdateCarryForward = {};
 
+    const data: Dictionary<unknown> = updateBy.data as Dictionary<unknown>;
+
+    /*
+     * Only an update that sends the monitor list changes which monitors the
+     * incident holds. One that sends just the monitor status (the API, the
+     * CLI or a workflow) takes no monitor off: reading the missing list as
+     * empty would restore every monitor, and walking it threw. A list set to
+     * null is sent: TypeORM clears a many-to-many set to null, like [].
+     */
+    const isMonitorListUpdated: boolean = data["monitors"] !== undefined;
+
     if (
-      updateBy.data.monitors ||
+      isMonitorListUpdated ||
       updateBy.data.changeMonitorStatusTo ||
       updateBy.data.changeMonitorStatusToId
     ) {
@@ -573,64 +603,69 @@ export class Service extends DatabaseService<Model> {
         props: updateBy.props,
       });
 
-      for (const incident of incidentsToUpdate) {
-        carryForward[incident.id!.toString()] = {
-          monitorsRemoved: [],
-          monitorsAdded: [],
-          oldChangeMonitorStatusIdTo: incident.changeMonitorStatusToId,
-          newMonitorChangeStatusIdTo:
-            (updateBy.data.changeMonitorStatusToId as ObjectID) ||
-            (updateBy.data.changeMonitorStatusTo as unknown as MonitorStatus)
-              ?._id ||
-            undefined,
-        };
-
-        for (const monitor of incident.monitors || []) {
-          // check if this monitor is actually removed.
-          let isRemoved: boolean = true;
-
-          for (const updatedMonitor of updateBy.data
-            ?.monitors as unknown as Array<Monitor>) {
-            if (
-              updatedMonitor._id &&
-              updatedMonitor._id.toString() === monitor._id?.toString()
-            ) {
-              isRemoved = false;
-              break;
-            }
-          }
+      /*
+       * The monitor status the update writes, in any shape the API accepts
+       * (an id, a bare uuid string or a relation object), so the status an
+       * update writes is never mistaken for one it leaves alone.
+       */
+      const monitorStatusIdInUpdate: ObjectID | string | undefined =
+        resolveReferenceId(data["changeMonitorStatusToId"]) ||
+        resolveReferenceId(data["changeMonitorStatusTo"]);
+
+      const newMonitorChangeStatusIdTo: ObjectID | undefined =
+        monitorStatusIdInUpdate
+          ? new ObjectID(monitorStatusIdInUpdate.toString())
+          : undefined;
+
+      const isChangeMonitorStatusToCleared: boolean =
+        !newMonitorChangeStatusIdTo &&
+        (data["changeMonitorStatusToId"] === null ||
+          data["changeMonitorStatusTo"] === null);
+
+      const monitorIdsAfterUpdate: Array<string> = isMonitorListUpdated
+        ? this.getMonitorIdsInUpdate(data["monitors"])
+        : [];
 
-          if (isRemoved) {
-            carryForward[incident.id!.toString()]?.monitorsRemoved?.push(
-              monitor,
-            );
-          }
-        }
+      for (const incident of incidentsToUpdate) {
+        const storedMonitors: Array<Monitor> = incident.monitors || [];
 
-        if (updateBy.data.monitors && updateBy.data.monitors.length > 0) {
-          for (const monitor of updateBy.data
-            ?.monitors as unknown as Array<Monitor>) {
-            // check if this monitor is actually added.
-            let isAdded: boolean = true;
-
-            for (const existingMonitor of incident.monitors || []) {
-              if (
-                existingMonitor._id &&
-                existingMonitor._id.toString() === monitor._id?.toString()
-              ) {
-                isAdded = false;
-                break;
-              }
-            }
+        const storedMonitorIds: Array<string> = storedMonitors.map(
+          (monitor: Monitor): string => {
+            return this.getMonitorIdForComparison(monitor);
+          },
+        );
 
-            if (isAdded) {
-              // this monitor is added.
-              carryForward[incident.id!.toString()]?.monitorsAdded?.push(
-                monitor,
+        const monitorsRemoved: Array<Monitor> = isMonitorListUpdated
+          ? storedMonitors.filter((monitor: Monitor): boolean => {
+              return !monitorIdsAfterUpdate.includes(
+                this.getMonitorIdForComparison(monitor),
               );
-            }
-          }
-        }
+            })
+          : [];
+
+        carryForward[incident.id!.toString()] = {
+          monitorsRemoved: monitorsRemoved,
+          /*
+           * Read now, before the write: an update that resolves the incident
+           * and takes a monitor off must still restore that monitor, and one
+           * that reopens a resolved incident must not restore it twice.
+           * After the write, the state no longer tells these cases apart.
+           */
+          isResolvedBeforeUpdate:
+            monitorsRemoved.length > 0
+              ? await this.isIncidentResolved({ incidentId: incident.id! })
+              : undefined,
+          monitorsAdded: monitorIdsAfterUpdate
+            .filter((monitorId: string): boolean => {
+              return !storedMonitorIds.includes(monitorId);
+            })
+            .map((monitorId: string): Monitor => {
+              return new Monitor(new ObjectID(monitorId));
+            }),
+          oldChangeMonitorStatusIdTo: incident.changeMonitorStatusToId,
+          newMonitorChangeStatusIdTo: newMonitorChangeStatusIdTo,
+          isChangeMonitorStatusToCleared: isChangeMonitorStatusToCleared,
+        };
       }
     }
 
@@ -705,6 +740,35 @@ export class Service extends DatabaseService<Model> {
     };
   }
 
+  /*
+   * The ids of the monitors an update's list holds, deduplicated and
+   * lowercased like getMonitorIdForComparison. The list reaches the hook as
+   * models, `{ _id }` objects, ObjectIDs or bare uuid strings (API update),
+   * and comparing only `_id` read every bare id as a monitor taken off and
+   * put back on again.
+   */
+  private getMonitorIdsInUpdate(monitors: unknown): Array<string> {
+    const monitorIds: Array<string> = [];
+
+    for (const monitorId of resolveReferenceIds(monitors)) {
+      const normalizedMonitorId: string = monitorId
+        .toString()
+        .trim()
+        .toLowerCase();
+
+      if (!monitorIds.includes(normalizedMonitorId)) {
+        monitorIds.push(normalizedMonitorId);
+      }
+    }
+
+    return monitorIds;
+  }
+
+  // A monitor's id as getMonitorIdsInUpdate writes it, for comparing the two.
+  private getMonitorIdForComparison(monitor: Monitor): string {
+    return (monitor._id?.toString() || "").trim().toLowerCase();
+  }
+
   /*
    * The incidents an update will write to, as they are stored, for the hooks
    * below that decide what else joins the update.
@@ -4352,12 +4416,6 @@ ${incidentSeverity.name}
                   },
                 });
 
-              // change these monitors back to operational state.
-              await this.markMonitorsActiveForMonitoring(
-                projectId!,
-                incidentCarryForward.monitorsRemoved,
-              );
-
               feedInfoInMarkdown += `\n\n**🗑️ Monitors Removed**:\n`;
 
               for (const monitor of monitorsRemoved) {
@@ -4417,9 +4475,11 @@ ${incidentSeverity.name}
               change: incidentCarryForward.statusPageScopeChange,
             });
 
+            // Saving the status the incident already had changes nothing.
             if (
               incidentCarryForward.oldChangeMonitorStatusIdTo &&
-              incidentCarryForward.newMonitorChangeStatusIdTo
+              incidentCarryForward.newMonitorChangeStatusIdTo &&
+              this.isMonitorStatusChangedByUpdate(incidentCarryForward)
             ) {
               const oldMonitorStatus: MonitorStatus | null =
                 await MonitorStatusService.findOneBy({
@@ -4453,45 +4513,13 @@ ${incidentSeverity.name}
               }
             }
 
-            const changeNewMonitorStatusTo: ObjectID | undefined =
-              incidentCarryForward.newMonitorChangeStatusIdTo ||
-              incidentCarryForward.oldChangeMonitorStatusIdTo;
-
-            if (incidentCarryForward.monitorsAdded?.length > 0) {
-              await this.disableActiveMonitoringIfManualIncident(incidentId);
-            }
-
-            if (changeNewMonitorStatusTo) {
-              const incident: Model | null = await this.findOneById({
-                id: incidentId,
-                select: {
-                  projectId: true,
-                  monitors: {
-                    _id: true,
-                  },
-                },
-                props: {
-                  isRoot: true,
-                },
-              });
-
-              const monitorsForThisIncident: Array<Monitor> =
-                incident?.monitors || [];
-
-              await MonitorService.changeMonitorStatus(
-                projectId!,
-                monitorsForThisIncident.map((monitor: Monitor) => {
-                  return new ObjectID(monitor._id?.toString() || "");
-                }),
-                changeNewMonitorStatusTo,
-                true, // notifyMonitorOwners
-                "Status was changed because Incident " +
-                  incidentNumberDisplay +
-                  " was updated.",
-                undefined,
-                onUpdate.updateBy.props,
-              );
-            }
+            await this.updateMonitorsForIncidentEdit({
+              projectId: projectId,
+              incidentId: incidentId,
+              incidentNumberDisplay: incidentNumberDisplay,
+              carryForward: incidentCarryForward,
+              props: onUpdate.updateBy.props,
+            });
           }
         }
 
@@ -4514,6 +4542,154 @@ ${incidentSeverity.name}
     return onUpdate;
   }
 
+  /*
+   * Whether an update puts the incident's monitors in a different status
+   * from the one it held: it writes a status, and not the same one again.
+   */
+  private isMonitorStatusChangedByUpdate(
+    carryForward: UpdateCarryForward[string],
+  ): boolean {
+    if (!carryForward.newMonitorChangeStatusIdTo) {
+      return false;
+    }
+
+    return (
+      carryForward.newMonitorChangeStatusIdTo.toString().toLowerCase() !==
+      (carryForward.oldChangeMonitorStatusIdTo?.toString() || "").toLowerCase()
+    );
+  }
+
+  /*
+   * What an edit of an incident's monitors, or of the status it puts them
+   * in, does to the monitors themselves. The feed item records the edit
+   * either way; this only decides which monitors change.
+   *
+   * The monitors taken off the incident are restored, unless another open
+   * incident still holds them (markMonitorsActiveForMonitoring), when the
+   * incident was open before the update. That includes an update that
+   * resolves it as well: resolving restores only the monitors it still
+   * holds (IncidentStateTimelineService), so this is the only restore the
+   * ones taken off get. A monitor taken off an incident that was resolved
+   * already was restored when it resolved, and restoring it again could
+   * overwrite a status set since (maintenance, or a manual monitor's status
+   * set by hand).
+   *
+   * A resolved incident, as it stands after the update, does nothing else
+   * to its monitors. Nothing would clear what it did: an added monitor of a
+   * manual incident would never be probed again, and the incident's status
+   * would sit on its monitors as false downtime.
+   *
+   * An open incident:
+   * - stops probing the monitors of a manual incident when one is added,
+   *   as creating it did (disableActiveMonitoringIfManualIncident);
+   * - puts the monitors added in its status, and all of its monitors only
+   *   when the edit changes that status. Every save used to put all of them
+   *   in it again, overwriting a status a probe had set since.
+   */
+  private async updateMonitorsForIncidentEdit(data: {
+    projectId: ObjectID;
+    incidentId: ObjectID;
+    incidentNumberDisplay: string;
+    carryForward: UpdateCarryForward[string];
+    props: DatabaseCommonInteractionProps;
+  }): Promise<void> {
+    const carryForward: UpdateCarryForward[string] = data.carryForward;
+
+    if (
+      carryForward.monitorsRemoved.length > 0 &&
+      !carryForward.isResolvedBeforeUpdate
+    ) {
+      // change these monitors back to operational state.
+      await this.markMonitorsActiveForMonitoring(
+        data.projectId,
+        carryForward.monitorsRemoved,
+      );
+    }
+
+    const isMonitorStatusChanged: boolean =
+      this.isMonitorStatusChangedByUpdate(carryForward);
+
+    if (carryForward.monitorsAdded.length === 0 && !isMonitorStatusChanged) {
+      return;
+    }
+
+    /*
+     * Read after the update is written, so an edit that resolves the
+     * incident as well counts as resolved: changing the state restored the
+     * monitors it holds, the added ones included.
+     */
+    if (await this.isIncidentResolved({ incidentId: data.incidentId })) {
+      return;
+    }
+
+    if (carryForward.monitorsAdded.length > 0) {
+      await this.disableActiveMonitoringIfManualIncident(data.incidentId);
+    }
+
+    // The status the incident puts its monitors in after the update.
+    const monitorStatusIdToApply: ObjectID | undefined =
+      carryForward.isChangeMonitorStatusToCleared
+        ? undefined
+        : carryForward.newMonitorChangeStatusIdTo ||
+          carryForward.oldChangeMonitorStatusIdTo;
+
+    if (
+      !monitorStatusIdToApply ||
+      (!isMonitorStatusChanged && carryForward.monitorsAdded.length === 0)
+    ) {
+      return;
+    }
+
+    const addedMonitorIds: Array<string> = carryForward.monitorsAdded.map(
+      (monitor: Monitor): string => {
+        return this.getMonitorIdForComparison(monitor);
+      },
+    );
+
+    /*
+     * The monitors the incident holds now, so a status is only ever put on
+     * a monitor the update left on it.
+     */
+    const incident: Model | null = await this.findOneById({
+      id: data.incidentId,
+      select: {
+        monitors: {
+          _id: true,
+        },
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    const monitorIdsToChange: Array<ObjectID> = (incident?.monitors || [])
+      .filter((monitor: Monitor): boolean => {
+        return (
+          isMonitorStatusChanged ||
+          addedMonitorIds.includes(this.getMonitorIdForComparison(monitor))
+        );
+      })
+      .map((monitor: Monitor): ObjectID => {
+        return new ObjectID(monitor._id?.toString() || "");
+      });
+
+    if (monitorIdsToChange.length === 0) {
+      return;
+    }
+
+    await MonitorService.changeMonitorStatus(
+      data.projectId,
+      monitorIdsToChange,
+      monitorStatusIdToApply,
+      true, // notifyMonitorOwners
+      "Status was changed because Incident " +
+        data.incidentNumberDisplay +
+        " was updated.",
+      undefined,
+      data.props,
+    );
+  }
+
   /*
    * The part of an update's feed item that records a change of the status
    * pages the incident is limited to: the pages added and removed, whether