@oneuptime/common 7.0.3148 → 7.0.3156

package/Server/Utils/Monitor/MonitorResource.ts

@@ -1,6 +1,3 @@
-import IncidentService from "../../Services/IncidentService";
-import IncidentSeverityService from "../../Services/IncidentSeverityService";
-import IncidentStateTimelineService from "../../Services/IncidentStateTimelineService";
 import MonitorMetricsByMinuteService from "../../Services/MonitorMetricsByMinuteService";
 import MonitorProbeService from "../../Services/MonitorProbeService";
 import MonitorService from "../../Services/MonitorService";
@@ -15,7 +12,6 @@ import ServerMonitorCriteria from "./Criteria/ServerMonitorCriteria";
 import SyntheticMonitoringCriteria from "./Criteria/SyntheticMonitor";
 import DataToProcess from "./DataToProcess";
 import SortOrder from "Common/Types/BaseDatabase/SortOrder";
-import { LIMIT_PER_PROJECT } from "Common/Types/Database/LimitMax";
 import Dictionary from "Common/Types/Dictionary";
 import BadDataException from "Common/Types/Exception/BadDataException";
 import BasicInfrastructureMetrics from "Common/Types/Infrastructure/BasicMetrics";
@@ -41,26 +37,36 @@ import ProbeApiIngestResponse from "Common/Types/Probe/ProbeApiIngestResponse";
 import ProbeMonitorResponse from "Common/Types/Probe/ProbeMonitorResponse";
 import Typeof from "Common/Types/Typeof";
 import MonitorMetricsByMinute from "Common/Models/AnalyticsModels/MonitorMetricsByMinute";
-import Incident, {
-  TelemetryIncidentQuery,
-} from "Common/Models/DatabaseModels/Incident";
-import IncidentSeverity from "Common/Models/DatabaseModels/IncidentSeverity";
-import IncidentStateTimeline from "Common/Models/DatabaseModels/IncidentStateTimeline";
 import Monitor from "Common/Models/DatabaseModels/Monitor";
 import MonitorProbe from "Common/Models/DatabaseModels/MonitorProbe";
 import MonitorStatusTimeline from "Common/Models/DatabaseModels/MonitorStatusTimeline";
-import OnCallDutyPolicy from "Common/Models/DatabaseModels/OnCallDutyPolicy";
 import OneUptimeDate from "Common/Types/Date";
 import LogMonitorCriteria from "./Criteria/LogMonitorCriteria";
 import LogMonitorResponse from "Common/Types/Monitor/LogMonitor/LogMonitorResponse";
 import TelemetryType from "Common/Types/Telemetry/TelemetryType";
 import TraceMonitorResponse from "../../../Types/Monitor/TraceMonitor/TraceMonitorResponse";
 import TraceMonitorCriteria from "./Criteria/TraceMonitorCriteria";
+import { TelemetryQuery } from "../../../Types/Telemetry/TelemetryQuery";
+import MonitorIncident from "./MonitorIncident";
+import MonitorAlert from "./MonitorAlert";
+import MonitorStatusTimelineUtil from "./MonitorStatusTimeline";
+import Semaphore, { SemaphoreMutex } from "../../Infrastructure/Semaphore";
 
 export default class MonitorResourceUtil {
   public static async monitorResource(
     dataToProcess: DataToProcess,
   ): Promise<ProbeApiIngestResponse> {
+    let mutex: SemaphoreMutex | null = null;
+
+    try {
+      mutex = await Semaphore.lock({
+        key: dataToProcess.monitorId.toString(),
+        namespace: "MonitorResourceUtil.monitorResource",
+      });
+    } catch (err) {
+      logger.error(err);
+    }
+
     let response: ProbeApiIngestResponse = {
       monitorId: dataToProcess.monitorId,
       criteriaMetId: undefined,
@@ -284,7 +290,9 @@ export default class MonitorResourceUtil {
     const autoResolveCriteriaInstanceIdIncidentIdsDictionary: Dictionary<
       Array<string>
     > = {};
+
     const criteriaInstanceMap: Dictionary<MonitorCriteriaInstance> = {};
+
     for (const criteriaInstance of criteriaInstances) {
       criteriaInstanceMap[criteriaInstance.data?.id || ""] = criteriaInstance;
 
@@ -312,6 +320,42 @@ export default class MonitorResourceUtil {
       }
     }
 
+    // alerts.
+
+    const autoResolveCriteriaInstanceIdAlertIdsDictionary: Dictionary<
+      Array<string>
+    > = {};
+
+    const criteriaInstanceAlertMap: Dictionary<MonitorCriteriaInstance> = {};
+
+    for (const criteriaInstance of criteriaInstances) {
+      criteriaInstanceAlertMap[criteriaInstance.data?.id || ""] =
+        criteriaInstance;
+
+      if (
+        criteriaInstance.data?.alerts &&
+        criteriaInstance.data?.alerts.length > 0
+      ) {
+        for (const alertTemplate of criteriaInstance.data!.alerts) {
+          if (alertTemplate.autoResolveAlert) {
+            if (
+              !autoResolveCriteriaInstanceIdAlertIdsDictionary[
+                criteriaInstance.data.id.toString()
+              ]
+            ) {
+              autoResolveCriteriaInstanceIdAlertIdsDictionary[
+                criteriaInstance.data.id.toString()
+              ] = [];
+            }
+
+            autoResolveCriteriaInstanceIdAlertIdsDictionary[
+              criteriaInstance.data.id.toString()
+            ]?.push(alertTemplate.id);
+          }
+        }
+      }
+    }
+
     const monitorStep: MonitorStep | undefined =
       monitorSteps.data.monitorStepsInstanceArray[0];
 
@@ -376,7 +420,7 @@ export default class MonitorResourceUtil {
         }`,
       );
 
-      let telemetryQuery: TelemetryIncidentQuery | undefined = undefined;
+      let telemetryQuery: TelemetryQuery | undefined = undefined;
 
       if (dataToProcess && (dataToProcess as LogMonitorResponse).logQuery) {
         telemetryQuery = {
@@ -398,7 +442,17 @@ export default class MonitorResourceUtil {
         );
       }
 
-      await this.criteriaMetCreateIncidentsAndUpdateMonitorStatus({
+      await MonitorStatusTimelineUtil.updateMonitorStatusTimeline({
+        monitor: monitor,
+        rootCause: response.rootCause,
+        dataToProcess: dataToProcess,
+        criteriaInstance: criteriaInstanceMap[response.criteriaMetId!]!,
+        props: {
+          telemetryQuery: telemetryQuery,
+        },
+      });
+
+      await MonitorIncident.criteriaMetCreateIncidentsAndUpdateMonitorStatus({
         monitor: monitor,
         rootCause: response.rootCause,
         dataToProcess: dataToProcess,
@@ -408,6 +462,17 @@ export default class MonitorResourceUtil {
           telemetryQuery: telemetryQuery,
         },
       });
+
+      await MonitorAlert.criteriaMetCreateAlertsAndUpdateMonitorStatus({
+        monitor: monitor,
+        rootCause: response.rootCause,
+        dataToProcess: dataToProcess,
+        autoResolveCriteriaInstanceIdAlertIdsDictionary,
+        criteriaInstance: criteriaInstanceAlertMap[response.criteriaMetId!]!,
+        props: {
+          telemetryQuery: telemetryQuery,
+        },
+      });
     } else if (
       !response.criteriaMetId &&
       monitorSteps.data.defaultMonitorStatusId &&
@@ -418,7 +483,7 @@ export default class MonitorResourceUtil {
         `${dataToProcess.monitorId.toString()} - No criteria met. Change to default status.`,
       );
 
-      await this.checkOpenIncidentsAndCloseIfResolved({
+      await MonitorIncident.checkOpenIncidentsAndCloseIfResolved({
         monitorId: monitor.id!,
         autoResolveCriteriaInstanceIdIncidentIdsDictionary,
         rootCause: "No monitoring criteria met. Change to default status.",
@@ -479,6 +544,14 @@ export default class MonitorResourceUtil {
       }
     }
 
+    if (mutex) {
+      try {
+        await Semaphore.release(mutex);
+      } catch (err) {
+        logger.error(err);
+      }
+    }
+
     return response;
   }
 
@@ -689,379 +762,6 @@ export default class MonitorResourceUtil {
     });
   }
 
-  private static async checkOpenIncidentsAndCloseIfResolved(input: {
-    monitorId: ObjectID;
-    autoResolveCriteriaInstanceIdIncidentIdsDictionary: Dictionary<
-      Array<string>
-    >;
-    rootCause: string;
-    criteriaInstance: MonitorCriteriaInstance | null;
-    dataToProcess: DataToProcess;
-  }): Promise<Array<Incident>> {
-    // check active incidents and if there are open incidents, do not cretae anothr incident.
-    const openIncidents: Array<Incident> = await IncidentService.findBy({
-      query: {
-        monitors: [input.monitorId] as any,
-        currentIncidentState: {
-          isResolvedState: false,
-        },
-      },
-      skip: 0,
-      limit: LIMIT_PER_PROJECT,
-      select: {
-        _id: true,
-        createdCriteriaId: true,
-        createdIncidentTemplateId: true,
-        projectId: true,
-      },
-      props: {
-        isRoot: true,
-      },
-    });
-
-    // check if should close the incident.
-
-    for (const openIncident of openIncidents) {
-      const shouldClose: boolean = MonitorResourceUtil.shouldCloseIncident({
-        openIncident,
-        autoResolveCriteriaInstanceIdIncidentIdsDictionary:
-          input.autoResolveCriteriaInstanceIdIncidentIdsDictionary,
-        criteriaInstance: input.criteriaInstance,
-      });
-
-      if (shouldClose) {
-        // then resolve incident.
-        await MonitorResourceUtil.resolveOpenIncident({
-          openIncident: openIncident,
-          rootCause: input.rootCause,
-          dataToProcess: input.dataToProcess,
-        });
-      }
-    }
-
-    return openIncidents;
-  }
-
-  private static async criteriaMetCreateIncidentsAndUpdateMonitorStatus(input: {
-    criteriaInstance: MonitorCriteriaInstance;
-    monitor: Monitor;
-    dataToProcess: DataToProcess;
-    rootCause: string;
-    autoResolveCriteriaInstanceIdIncidentIdsDictionary: Dictionary<
-      Array<string>
-    >;
-    props: {
-      telemetryQuery?: TelemetryIncidentQuery | undefined;
-    };
-  }): Promise<void> {
-    // criteria filters are met, now process the actions.
-
-    const lastMonitorStatusTimeline: MonitorStatusTimeline | null =
-      await MonitorStatusTimelineService.findOneBy({
-        query: {
-          monitorId: input.monitor.id!,
-          projectId: input.monitor.projectId!,
-        },
-        select: {
-          _id: true,
-          monitorStatusId: true,
-        },
-        sort: {
-          startsAt: SortOrder.Descending,
-        },
-        props: {
-          isRoot: true,
-        },
-      });
-
-    let shouldUpdateStatus: boolean = false;
-
-    if (!lastMonitorStatusTimeline) {
-      // if monitor does not have any status timeline, then create one.
-      shouldUpdateStatus = true;
-    }
-
-    if (
-      input.criteriaInstance.data?.changeMonitorStatus &&
-      input.criteriaInstance.data?.monitorStatusId &&
-      input.criteriaInstance.data?.monitorStatusId.toString() !==
-        lastMonitorStatusTimeline?.id?.toString()
-    ) {
-      // if monitor status is changed, then create a new status timeline.
-      shouldUpdateStatus = true;
-    }
-
-    // check if the current status is same as the last status.
-
-    if (
-      input.criteriaInstance.data?.changeMonitorStatus &&
-      input.criteriaInstance.data?.monitorStatusId &&
-      input.criteriaInstance.data?.monitorStatusId.toString() !==
-        input.monitor.currentMonitorStatusId?.toString()
-    ) {
-      // if monitor status is changed, then create a new status timeline.
-      shouldUpdateStatus = true;
-    }
-
-    if (shouldUpdateStatus) {
-      logger.debug(
-        `${input.monitor.id?.toString()} - Change monitor status to ${input.criteriaInstance.data?.monitorStatusId?.toString()}`,
-      );
-      // change monitor status
-
-      const monitorStatusId: ObjectID | undefined =
-        input.criteriaInstance.data?.monitorStatusId;
-
-      if (!monitorStatusId) {
-        throw new BadDataException("Monitor status is not defined.");
-      }
-
-      //change monitor status.
-
-      // get last status of this monitor.
-
-      // get last monitor status timeline.
-
-      if (
-        lastMonitorStatusTimeline &&
-        lastMonitorStatusTimeline.monitorStatusId &&
-        lastMonitorStatusTimeline.monitorStatusId.toString() ===
-          monitorStatusId.toString()
-      ) {
-        // status is same as last status. do not create new status timeline.
-        return;
-      }
-
-      const monitorStatusTimeline: MonitorStatusTimeline =
-        new MonitorStatusTimeline();
-      monitorStatusTimeline.monitorId = input.monitor.id!;
-      monitorStatusTimeline.monitorStatusId = monitorStatusId;
-      monitorStatusTimeline.projectId = input.monitor.projectId!;
-      monitorStatusTimeline.statusChangeLog = JSON.parse(
-        JSON.stringify(input.dataToProcess),
-      );
-      monitorStatusTimeline.rootCause = input.rootCause;
-
-      await MonitorStatusTimelineService.create({
-        data: monitorStatusTimeline,
-        props: {
-          isRoot: true,
-        },
-      });
-    }
-
-    // check open incidents
-    logger.debug(`${input.monitor.id?.toString()} - Check open incidents.`);
-    // check active incidents and if there are open incidents, do not cretae anothr incident.
-    const openIncidents: Array<Incident> =
-      await this.checkOpenIncidentsAndCloseIfResolved({
-        monitorId: input.monitor.id!,
-        autoResolveCriteriaInstanceIdIncidentIdsDictionary:
-          input.autoResolveCriteriaInstanceIdIncidentIdsDictionary,
-        rootCause: input.rootCause,
-        criteriaInstance: input.criteriaInstance,
-        dataToProcess: input.dataToProcess,
-      });
-
-    if (input.criteriaInstance.data?.createIncidents) {
-      // create incidents
-
-      for (const criteriaIncident of input.criteriaInstance.data?.incidents ||
-        []) {
-        // should create incident.
-
-        const alreadyOpenIncident: Incident | undefined = openIncidents.find(
-          (incident: Incident) => {
-            return (
-              incident.createdCriteriaId ===
-                input.criteriaInstance.data?.id.toString() &&
-              incident.createdIncidentTemplateId ===
-                criteriaIncident.id.toString()
-            );
-          },
-        );
-
-        const hasAlreadyOpenIncident: boolean = Boolean(alreadyOpenIncident);
-
-        logger.debug(
-          `${input.monitor.id?.toString()} - Open Incident ${alreadyOpenIncident?.id?.toString()}`,
-        );
-
-        logger.debug(
-          `${input.monitor.id?.toString()} - Has open incident ${hasAlreadyOpenIncident}`,
-        );
-
-        if (hasAlreadyOpenIncident) {
-          continue;
-        }
-
-        // create incident here.
-
-        logger.debug(`${input.monitor.id?.toString()} - Create incident.`);
-
-        const incident: Incident = new Incident();
-
-        incident.title = criteriaIncident.title;
-        incident.description = criteriaIncident.description;
-
-        if (!criteriaIncident.incidentSeverityId) {
-          // pick the critical criteria.
-
-          const severity: IncidentSeverity | null =
-            await IncidentSeverityService.findOneBy({
-              query: {
-                projectId: input.monitor.projectId!,
-              },
-              sort: {
-                order: SortOrder.Ascending,
-              },
-              props: {
-                isRoot: true,
-              },
-              select: {
-                _id: true,
-              },
-            });
-
-          if (!severity) {
-            throw new BadDataException(
-              "Project does not have incident severity",
-            );
-          } else {
-            incident.incidentSeverityId = severity.id!;
-          }
-        } else {
-          incident.incidentSeverityId = criteriaIncident.incidentSeverityId!;
-        }
-
-        incident.monitors = [input.monitor];
-        incident.projectId = input.monitor.projectId!;
-        incident.rootCause = input.rootCause;
-        incident.createdStateLog = JSON.parse(
-          JSON.stringify(input.dataToProcess, null, 2),
-        );
-
-        incident.createdCriteriaId = input.criteriaInstance.data.id.toString();
-
-        incident.createdIncidentTemplateId = criteriaIncident.id.toString();
-
-        incident.onCallDutyPolicies =
-          criteriaIncident.onCallPolicyIds?.map((id: ObjectID) => {
-            const onCallPolicy: OnCallDutyPolicy = new OnCallDutyPolicy();
-            onCallPolicy._id = id.toString();
-            return onCallPolicy;
-          }) || [];
-
-        incident.isCreatedAutomatically = true;
-
-        if (input.props.telemetryQuery) {
-          incident.telemetryQuery = input.props.telemetryQuery;
-        }
-
-        if (
-          input.dataToProcess &&
-          (input.dataToProcess as ProbeMonitorResponse).probeId
-        ) {
-          incident.createdByProbeId = (
-            input.dataToProcess as ProbeMonitorResponse
-          ).probeId;
-        }
-
-        if (criteriaIncident.remediationNotes) {
-          incident.remediationNotes = criteriaIncident.remediationNotes;
-        }
-
-        await IncidentService.create({
-          data: incident,
-          props: {
-            isRoot: true,
-          },
-        });
-      }
-    }
-  }
-
-  private static async resolveOpenIncident(input: {
-    openIncident: Incident;
-    rootCause: string;
-    dataToProcess:
-      | ProbeMonitorResponse
-      | IncomingMonitorRequest
-      | DataToProcess;
-  }): Promise<void> {
-    const resolvedStateId: ObjectID =
-      await IncidentStateTimelineService.getResolvedStateIdForProject(
-        input.openIncident.projectId!,
-      );
-
-    const incidentStateTimeline: IncidentStateTimeline =
-      new IncidentStateTimeline();
-    incidentStateTimeline.incidentId = input.openIncident.id!;
-    incidentStateTimeline.incidentStateId = resolvedStateId;
-    incidentStateTimeline.projectId = input.openIncident.projectId!;
-
-    if (input.rootCause) {
-      incidentStateTimeline.rootCause =
-        "Incident autoresolved because autoresolve is set to true in monitor criteria. " +
-        input.rootCause;
-    }
-
-    if (input.dataToProcess) {
-      incidentStateTimeline.stateChangeLog = JSON.parse(
-        JSON.stringify(input.dataToProcess),
-      );
-    }
-
-    await IncidentStateTimelineService.create({
-      data: incidentStateTimeline,
-      props: {
-        isRoot: true,
-      },
-    });
-  }
-
-  private static shouldCloseIncident(input: {
-    openIncident: Incident;
-    autoResolveCriteriaInstanceIdIncidentIdsDictionary: Dictionary<
-      Array<string>
-    >;
-    criteriaInstance: MonitorCriteriaInstance | null; // null if no criteia met.
-  }): boolean {
-    if (
-      input.openIncident.createdCriteriaId?.toString() ===
-      input.criteriaInstance?.data?.id.toString()
-    ) {
-      // same incident active. So, do not close.
-      return false;
-    }
-
-    // If antoher criteria is active then, check if the incident id is present in the map.
-
-    if (!input.openIncident.createdCriteriaId?.toString()) {
-      return false;
-    }
-
-    if (!input.openIncident.createdIncidentTemplateId?.toString()) {
-      return false;
-    }
-
-    if (
-      input.autoResolveCriteriaInstanceIdIncidentIdsDictionary[
-        input.openIncident.createdCriteriaId?.toString()
-      ]
-    ) {
-      if (
-        input.autoResolveCriteriaInstanceIdIncidentIdsDictionary[
-          input.openIncident.createdCriteriaId?.toString()
-        ]?.includes(input.openIncident.createdIncidentTemplateId?.toString())
-      ) {
-        return true;
-      }
-    }
-
-    return false;
-  }
-
   private static async processMonitorStep(input: {
     dataToProcess: DataToProcess;
     monitorStep: MonitorStep;
@@ -1091,7 +791,7 @@ export default class MonitorResourceUtil {
       if (rootCause) {
         input.probeApiIngestResponse.criteriaMetId = criteriaInstance.data?.id;
         input.probeApiIngestResponse.rootCause = `
-**This incident is created because the following criteria was met**: 
+**Created because the following criteria was met**: 
 
 **Criteria Name**: ${criteriaInstance.data?.name}
 `;

package/Server/Utils/Monitor/MonitorStatusTimeline.ts

@@ -0,0 +1,120 @@
+import Monitor from "../../../Models/DatabaseModels/Monitor";
+import MonitorStatusTimeline from "../../../Models/DatabaseModels/MonitorStatusTimeline";
+import SortOrder from "../../../Types/BaseDatabase/SortOrder";
+import BadDataException from "../../../Types/Exception/BadDataException";
+import MonitorCriteriaInstance from "../../../Types/Monitor/MonitorCriteriaInstance";
+import ObjectID from "../../../Types/ObjectID";
+import { TelemetryQuery } from "../../../Types/Telemetry/TelemetryQuery";
+import MonitorStatusTimelineService from "../../Services/MonitorStatusTimelineService";
+import logger from "../Logger";
+import DataToProcess from "./DataToProcess";
+
+export default class MonitorStatusTimelineUtil {
+  public static async updateMonitorStatusTimeline(input: {
+    criteriaInstance: MonitorCriteriaInstance;
+    monitor: Monitor;
+    dataToProcess: DataToProcess;
+    rootCause: string;
+    props: {
+      telemetryQuery?: TelemetryQuery | undefined;
+    };
+  }): Promise<MonitorStatusTimeline | null> {
+    // criteria filters are met, now process the actions.
+
+    const lastMonitorStatusTimeline: MonitorStatusTimeline | null =
+      await MonitorStatusTimelineService.findOneBy({
+        query: {
+          monitorId: input.monitor.id!,
+          projectId: input.monitor.projectId!,
+        },
+        select: {
+          _id: true,
+          monitorStatusId: true,
+        },
+        sort: {
+          startsAt: SortOrder.Descending,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+    let shouldUpdateStatus: boolean = false;
+
+    if (!lastMonitorStatusTimeline) {
+      // if monitor does not have any status timeline, then create one.
+      shouldUpdateStatus = true;
+    }
+
+    if (
+      input.criteriaInstance.data?.changeMonitorStatus &&
+      input.criteriaInstance.data?.monitorStatusId &&
+      input.criteriaInstance.data?.monitorStatusId.toString() !==
+        lastMonitorStatusTimeline?.id?.toString()
+    ) {
+      // if monitor status is changed, then create a new status timeline.
+      shouldUpdateStatus = true;
+    }
+
+    // check if the current status is same as the last status.
+
+    if (
+      input.criteriaInstance.data?.changeMonitorStatus &&
+      input.criteriaInstance.data?.monitorStatusId &&
+      input.criteriaInstance.data?.monitorStatusId.toString() !==
+        input.monitor.currentMonitorStatusId?.toString()
+    ) {
+      // if monitor status is changed, then create a new status timeline.
+      shouldUpdateStatus = true;
+    }
+
+    if (shouldUpdateStatus) {
+      logger.debug(
+        `${input.monitor.id?.toString()} - Change monitor status to ${input.criteriaInstance.data?.monitorStatusId?.toString()}`,
+      );
+      // change monitor status
+
+      const monitorStatusId: ObjectID | undefined =
+        input.criteriaInstance.data?.monitorStatusId;
+
+      if (!monitorStatusId) {
+        throw new BadDataException("Monitor status is not defined.");
+      }
+
+      //change monitor status.
+
+      // get last status of this monitor.
+
+      // get last monitor status timeline.
+
+      if (
+        lastMonitorStatusTimeline &&
+        lastMonitorStatusTimeline.monitorStatusId &&
+        lastMonitorStatusTimeline.monitorStatusId.toString() ===
+          monitorStatusId.toString()
+      ) {
+        // status is same as last status. do not create new status timeline.
+        return null;
+      }
+
+      const monitorStatusTimeline: MonitorStatusTimeline =
+        new MonitorStatusTimeline();
+      monitorStatusTimeline.monitorId = input.monitor.id!;
+      monitorStatusTimeline.monitorStatusId = monitorStatusId;
+      monitorStatusTimeline.projectId = input.monitor.projectId!;
+      monitorStatusTimeline.statusChangeLog = JSON.parse(
+        JSON.stringify(input.dataToProcess),
+      );
+      monitorStatusTimeline.rootCause = input.rootCause;
+
+      return await MonitorStatusTimelineService.create({
+        data: monitorStatusTimeline,
+        props: {
+          isRoot: true,
+        },
+      });
+    }
+
+    return null;
+  }
+}