@oneuptime/common 12.0.25 → 12.0.27

package/build/dist/Server/Utils/Monitor/MonitorAlert.js

@@ -58,15 +58,18 @@ export default class MonitorAlert {
             },
         });
         // check if should close the alert.
+        const resolvedAlertIds = new Set();
         for (const openAlert of openAlerts) {
             const shouldClose = this.shouldCloseAlert({
                 openAlert,
                 autoResolveCriteriaInstanceIdAlertIdsDictionary: input.autoResolveCriteriaInstanceIdAlertIdsDictionary,
                 criteriaInstance: input.criteriaInstance,
                 breachingSeriesFingerprints: input.breachingSeriesFingerprints,
+                breachingSeriesFingerprintsByCriteriaId: input.breachingSeriesFingerprintsByCriteriaId,
                 disableSeriesAbsenceResolution: input.disableSeriesAbsenceResolution,
             });
             if (shouldClose) {
+                resolvedAlertIds.add(openAlert.id.toString());
                 // then resolve alert.
                 await this.resolveOpenAlert({
                     openAlert: openAlert,
@@ -85,7 +88,19 @@ export default class MonitorAlert {
                 });
             }
         }
-        return openAlerts;
+        /*
+         * Return the alerts that are still open AFTER this pass. The create
+         * path dedupes against this list, and an alert this pass just
+         * resolved must not count as "already active" — that combination
+         * resolved a stale whole-monitor alert and then refused to create
+         * the per-series alerts meant to replace it.
+         */
+        if (resolvedAlertIds.size === 0) {
+            return openAlerts;
+        }
+        return openAlerts.filter((openAlert) => {
+            return !resolvedAlertIds.has(openAlert.id.toString());
+        });
     }
     /**
      * Event-driven (incoming-request / webhook) resolution: resolve the open
@@ -153,7 +168,7 @@ export default class MonitorAlert {
         }
     }
     static async criteriaMetCreateAlertsAndUpdateMonitorStatus(input) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7, _8;
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, _14, _15, _16, _17;
         const alertLogAttributes = {
             projectId: (_a = input.monitor.projectId) === null || _a === void 0 ? void 0 : _a.toString(),
         };
@@ -165,19 +180,37 @@ export default class MonitorAlert {
             }))
             : undefined;
         // check active alerts and if there are open alerts, do not cretae anothr alert.
-        const openAlerts = await this.checkOpenAlertsAndCloseIfResolved({
-            monitorId: input.monitor.id,
-            autoResolveCriteriaInstanceIdAlertIdsDictionary: input.autoResolveCriteriaInstanceIdAlertIdsDictionary,
-            rootCause: input.rootCause,
-            criteriaInstance: input.criteriaInstance,
-            dataToProcess: input.dataToProcess,
-            evaluationSummary: input.evaluationSummary,
-            breachingSeriesFingerprints,
-            disableSeriesAbsenceResolution: input.disableSeriesAbsenceResolution,
-        });
+        const openAlerts = input.openAlerts !== undefined
+            ? input.openAlerts
+            : await this.checkOpenAlertsAndCloseIfResolved({
+                monitorId: input.monitor.id,
+                autoResolveCriteriaInstanceIdAlertIdsDictionary: input.autoResolveCriteriaInstanceIdAlertIdsDictionary,
+                rootCause: input.rootCause,
+                criteriaInstance: input.criteriaInstance,
+                dataToProcess: input.dataToProcess,
+                evaluationSummary: input.evaluationSummary,
+                breachingSeriesFingerprints,
+                breachingSeriesFingerprintsByCriteriaId: input.breachingSeriesFingerprintsByCriteriaId,
+                disableSeriesAbsenceResolution: input.disableSeriesAbsenceResolution,
+            });
         if (!((_c = input.criteriaInstance.data) === null || _c === void 0 ? void 0 : _c.createAlerts)) {
             return;
         }
+        /*
+         * Checked once, up front. It used to be tested deep inside the
+         * template x series loops with a `return`, which on a grouped
+         * monitor abandoned every series after the first.
+         */
+        if (DisableAutomaticAlertCreation) {
+            (_d = input.evaluationSummary) === null || _d === void 0 ? void 0 : _d.events.push({
+                type: "alert-skipped",
+                title: "Alert creation skipped",
+                message: "Automatic alert creation is disabled by environment configuration.",
+                relatedCriteriaId: (_e = input.criteriaInstance.data) === null || _e === void 0 ? void 0 : _e.id,
+                at: OneUptimeDate.getCurrentDate(),
+            });
+            return;
+        }
         /*
          * Alert-dependency suppression: a parent monitor is in a suppressing
          * status (offline by default), so skip creating any alert for this
@@ -185,14 +218,14 @@ export default class MonitorAlert {
          * purpose — an alert raised before the parent went down must still
          * resolve normally; only new creation is silenced.
          */
-        if ((_d = input.dependencySuppression) === null || _d === void 0 ? void 0 : _d.isSuppressed) {
+        if ((_f = input.dependencySuppression) === null || _f === void 0 ? void 0 : _f.isSuppressed) {
             const suppressionReason = MonitorDependencySuppression.buildSuppressionReason(input.dependencySuppression.suppressingParents);
-            logger.debug(`${(_e = input.monitor.id) === null || _e === void 0 ? void 0 : _e.toString()} - Skipping alert creation: ${suppressionReason}.`, alertLogAttributes);
-            (_f = input.evaluationSummary) === null || _f === void 0 ? void 0 : _f.events.push({
+            logger.debug(`${(_g = input.monitor.id) === null || _g === void 0 ? void 0 : _g.toString()} - Skipping alert creation: ${suppressionReason}.`, alertLogAttributes);
+            (_h = input.evaluationSummary) === null || _h === void 0 ? void 0 : _h.events.push({
                 type: "alert-skipped",
                 title: "Alert suppressed by monitor dependency",
                 message: `Skipped creating alerts because ${suppressionReason}. This monitor's status still updates normally; open alerts still auto-resolve.`,
-                relatedCriteriaId: (_g = input.criteriaInstance.data) === null || _g === void 0 ? void 0 : _g.id,
+                relatedCriteriaId: (_j = input.criteriaInstance.data) === null || _j === void 0 ? void 0 : _j.id,
                 at: OneUptimeDate.getCurrentDate(),
             });
             return;
@@ -228,245 +261,275 @@ export default class MonitorAlert {
          * created alert's owners alongside the criteria-configured ones.
          */
         let networkDeviceOwners = null;
-        for (const criteriaAlert of ((_h = input.criteriaInstance.data) === null || _h === void 0 ? void 0 : _h.alerts) || []) {
+        for (const criteriaAlert of ((_k = input.criteriaInstance.data) === null || _k === void 0 ? void 0 : _k.alerts) || []) {
             for (const seriesMatch of seriesToProcess) {
-                const seriesFingerprint = seriesMatch === null || seriesMatch === void 0 ? void 0 : seriesMatch.fingerprint;
-                const seriesLabels = seriesMatch === null || seriesMatch === void 0 ? void 0 : seriesMatch.labels;
-                const seriesRootCause = (seriesMatch === null || seriesMatch === void 0 ? void 0 : seriesMatch.rootCause) || input.rootCause;
-                /*
-                 * Per-series scheduled-maintenance suppression: skip creating an
-                 * alert for a series whose resource is inside an ongoing
-                 * maintenance window. Other series on the same monitor are
-                 * unaffected. Only *new* creation is suppressed — existing open
-                 * alerts follow the normal resolve path.
-                 */
-                if (seriesFingerprint &&
-                    ((_j = input.suppressedSeriesFingerprints) === null || _j === void 0 ? void 0 : _j.has(seriesFingerprint))) {
-                    logger.debug(`${(_k = input.monitor.id) === null || _k === void 0 ? void 0 : _k.toString()} - Skipping alert for series ${seriesFingerprint}: its resource is under an active scheduled maintenance window.`, alertLogAttributes);
-                    (_l = input.evaluationSummary) === null || _l === void 0 ? void 0 : _l.events.push({
-                        type: "alert-skipped",
-                        title: "Alert suppressed by scheduled maintenance",
-                        message: "Skipped creating an alert because the resource for this series is under an active scheduled maintenance window.",
-                        relatedCriteriaId: (_m = input.criteriaInstance.data) === null || _m === void 0 ? void 0 : _m.id,
-                        at: OneUptimeDate.getCurrentDate(),
+                try {
+                    const seriesFingerprint = seriesMatch === null || seriesMatch === void 0 ? void 0 : seriesMatch.fingerprint;
+                    const seriesLabels = seriesMatch === null || seriesMatch === void 0 ? void 0 : seriesMatch.labels;
+                    const seriesRootCause = (seriesMatch === null || seriesMatch === void 0 ? void 0 : seriesMatch.rootCause) || input.rootCause;
+                    /*
+                     * Per-series scheduled-maintenance suppression: skip creating an
+                     * alert for a series whose resource is inside an ongoing
+                     * maintenance window. Other series on the same monitor are
+                     * unaffected. Only *new* creation is suppressed — existing open
+                     * alerts follow the normal resolve path.
+                     */
+                    if (seriesFingerprint &&
+                        ((_l = input.suppressedSeriesFingerprints) === null || _l === void 0 ? void 0 : _l.has(seriesFingerprint))) {
+                        logger.debug(`${(_m = input.monitor.id) === null || _m === void 0 ? void 0 : _m.toString()} - Skipping alert for series ${seriesFingerprint}: its resource is under an active scheduled maintenance window.`, alertLogAttributes);
+                        (_o = input.evaluationSummary) === null || _o === void 0 ? void 0 : _o.events.push({
+                            type: "alert-skipped",
+                            title: "Alert suppressed by scheduled maintenance",
+                            message: "Skipped creating an alert because the resource for this series is under an active scheduled maintenance window.",
+                            relatedCriteriaId: (_p = input.criteriaInstance.data) === null || _p === void 0 ? void 0 : _p.id,
+                            at: OneUptimeDate.getCurrentDate(),
+                        });
+                        continue;
+                    }
+                    /*
+                     * Mirror the create path: `createdCriteriaId` is only set when the
+                     * criteria has an `id`. Guard the `.toString()` with `?.` (a criteria
+                     * with a missing id otherwise threw "Cannot read properties of
+                     * undefined (reading 'toString')" and failed the queue job every
+                     * cycle) and normalise both sides to `undefined` on missing so dedupe
+                     * stays correct instead of creating a duplicate alert each cycle.
+                     */
+                    const alreadyOpenAlert = openAlerts.find((alert) => {
+                        var _a, _b;
+                        return ((alert.createdCriteriaId || undefined) ===
+                            (((_b = (_a = input.criteriaInstance.data) === null || _a === void 0 ? void 0 : _a.id) === null || _b === void 0 ? void 0 : _b.toString()) || undefined) &&
+                            (alert.seriesFingerprint || undefined) === seriesFingerprint);
                     });
-                    continue;
-                }
-                /*
-                 * Mirror the create path: `createdCriteriaId` is only set when the
-                 * criteria has an `id`. Guard the `.toString()` with `?.` (a criteria
-                 * with a missing id otherwise threw "Cannot read properties of
-                 * undefined (reading 'toString')" and failed the queue job every
-                 * cycle) and normalise both sides to `undefined` on missing so dedupe
-                 * stays correct instead of creating a duplicate alert each cycle.
-                 */
-                const alreadyOpenAlert = openAlerts.find((alert) => {
-                    var _a, _b;
-                    return ((alert.createdCriteriaId || undefined) ===
-                        (((_b = (_a = input.criteriaInstance.data) === null || _a === void 0 ? void 0 : _a.id) === null || _b === void 0 ? void 0 : _b.toString()) || undefined) &&
-                        (alert.seriesFingerprint || undefined) === seriesFingerprint);
-                });
-                const hasAlreadyOpenAlert = Boolean(alreadyOpenAlert);
-                logger.debug(`${(_o = input.monitor.id) === null || _o === void 0 ? void 0 : _o.toString()} - Open Alert ${(_p = alreadyOpenAlert === null || alreadyOpenAlert === void 0 ? void 0 : alreadyOpenAlert.id) === null || _p === void 0 ? void 0 : _p.toString()}`, alertLogAttributes);
-                logger.debug(`${(_q = input.monitor.id) === null || _q === void 0 ? void 0 : _q.toString()} - Has open alert ${hasAlreadyOpenAlert}`, alertLogAttributes);
-                if (hasAlreadyOpenAlert) {
-                    const renderedAlertTitle = (alreadyOpenAlert === null || alreadyOpenAlert === void 0 ? void 0 : alreadyOpenAlert.title) || criteriaAlert.title;
-                    (_r = input.evaluationSummary) === null || _r === void 0 ? void 0 : _r.events.push({
-                        type: "alert-skipped",
-                        title: `Alert already active: ${renderedAlertTitle}`,
-                        message: "Skipped creating a new alert because an active alert exists for this criteria.",
-                        relatedCriteriaId: (_s = input.criteriaInstance.data) === null || _s === void 0 ? void 0 : _s.id,
-                        relatedAlertId: (_t = alreadyOpenAlert === null || alreadyOpenAlert === void 0 ? void 0 : alreadyOpenAlert.id) === null || _t === void 0 ? void 0 : _t.toString(),
-                        relatedAlertNumber: alreadyOpenAlert === null || alreadyOpenAlert === void 0 ? void 0 : alreadyOpenAlert.alertNumber,
-                        relatedAlertNumberWithPrefix: alreadyOpenAlert === null || alreadyOpenAlert === void 0 ? void 0 : alreadyOpenAlert.alertNumberWithPrefix,
-                        at: OneUptimeDate.getCurrentDate(),
+                    const hasAlreadyOpenAlert = Boolean(alreadyOpenAlert);
+                    logger.debug(`${(_q = input.monitor.id) === null || _q === void 0 ? void 0 : _q.toString()} - Open Alert ${(_r = alreadyOpenAlert === null || alreadyOpenAlert === void 0 ? void 0 : alreadyOpenAlert.id) === null || _r === void 0 ? void 0 : _r.toString()}`, alertLogAttributes);
+                    logger.debug(`${(_s = input.monitor.id) === null || _s === void 0 ? void 0 : _s.toString()} - Has open alert ${hasAlreadyOpenAlert}`, alertLogAttributes);
+                    if (hasAlreadyOpenAlert) {
+                        const renderedAlertTitle = (alreadyOpenAlert === null || alreadyOpenAlert === void 0 ? void 0 : alreadyOpenAlert.title) || criteriaAlert.title;
+                        (_t = input.evaluationSummary) === null || _t === void 0 ? void 0 : _t.events.push({
+                            type: "alert-skipped",
+                            title: `Alert already active: ${renderedAlertTitle}`,
+                            message: "Skipped creating a new alert because an active alert exists for this criteria.",
+                            relatedCriteriaId: (_u = input.criteriaInstance.data) === null || _u === void 0 ? void 0 : _u.id,
+                            relatedAlertId: (_v = alreadyOpenAlert === null || alreadyOpenAlert === void 0 ? void 0 : alreadyOpenAlert.id) === null || _v === void 0 ? void 0 : _v.toString(),
+                            relatedAlertNumber: alreadyOpenAlert === null || alreadyOpenAlert === void 0 ? void 0 : alreadyOpenAlert.alertNumber,
+                            relatedAlertNumberWithPrefix: alreadyOpenAlert === null || alreadyOpenAlert === void 0 ? void 0 : alreadyOpenAlert.alertNumberWithPrefix,
+                            at: OneUptimeDate.getCurrentDate(),
+                        });
+                        continue;
+                    }
+                    // create alert here.
+                    logger.debug(`${(_w = input.monitor.id) === null || _w === void 0 ? void 0 : _w.toString()} - Create alert.`, alertLogAttributes);
+                    const alert = new Alert();
+                    const storageMap = MonitorTemplateUtil.buildTemplateStorageMap({
+                        monitorType: input.monitor.monitorType,
+                        dataToProcess: input.dataToProcess,
+                        monitor: input.monitor,
+                        seriesLabels,
                     });
-                    continue;
-                }
-                // create alert here.
-                logger.debug(`${(_u = input.monitor.id) === null || _u === void 0 ? void 0 : _u.toString()} - Create alert.`, alertLogAttributes);
-                const alert = new Alert();
-                const storageMap = MonitorTemplateUtil.buildTemplateStorageMap({
-                    monitorType: input.monitor.monitorType,
-                    dataToProcess: input.dataToProcess,
-                    monitor: input.monitor,
-                    seriesLabels,
-                });
-                alert.title = MonitorTemplateUtil.processTemplateString({
-                    value: criteriaAlert.title,
-                    storageMap,
-                });
-                alert.description = MonitorTemplateUtil.processTemplateString({
-                    value: criteriaAlert.description,
-                    storageMap,
-                });
-                /*
-                 * A criteria severity belonging to *another* project is rejected by
-                 * AlertService on create. Throwing here would fail the whole
-                 * probe/telemetry ingest job for this monitor, so treat it as
-                 * "missing" and fall back to this project's own default — which is
-                 * also what stops the bad id spreading onto new alerts. See the same
-                 * fallback in MonitorIncident.
-                 */
-                const isCriteriaSeverityUsable = await ProjectScopedReferenceValidator.isUsableInProject({
-                    projectId: input.monitor.projectId,
-                    id: criteriaAlert.alertSeverityId,
-                    service: AlertSeverityService,
-                });
-                if (((_v = criteriaAlert.alertSeverityId) === null || _v === void 0 ? void 0 : _v.toString()) &&
-                    !isCriteriaSeverityUsable) {
-                    logger.error(`${(_w = input.monitor.id) === null || _w === void 0 ? void 0 : _w.toString()} - Criteria "${(_x = input.criteriaInstance.data) === null || _x === void 0 ? void 0 : _x.name}" references alert severity ${criteriaAlert.alertSeverityId.toString()}, which does not belong to project ${(_y = input.monitor.projectId) === null || _y === void 0 ? void 0 : _y.toString()}. Falling back to this project's default severity.`);
-                }
-                if (!isCriteriaSeverityUsable) {
-                    // pick the critical criteria.
-                    const severity = await AlertSeverityService.findOneBy({
-                        query: {
-                            projectId: input.monitor.projectId,
-                        },
-                        sort: {
-                            order: SortOrder.Ascending,
-                        },
-                        props: {
-                            isRoot: true,
-                        },
-                        select: {
-                            _id: true,
-                        },
+                    alert.title = MonitorTemplateUtil.processTemplateString({
+                        value: criteriaAlert.title,
+                        storageMap,
                     });
-                    if (!severity) {
-                        throw new BadDataException("Project does not have alert severity");
+                    alert.description = MonitorTemplateUtil.processTemplateString({
+                        value: criteriaAlert.description,
+                        storageMap,
+                    });
+                    /*
+                     * A criteria severity belonging to *another* project is rejected by
+                     * AlertService on create. Throwing here would fail the whole
+                     * probe/telemetry ingest job for this monitor, so treat it as
+                     * "missing" and fall back to this project's own default — which is
+                     * also what stops the bad id spreading onto new alerts. See the same
+                     * fallback in MonitorIncident.
+                     */
+                    const isCriteriaSeverityUsable = await ProjectScopedReferenceValidator.isUsableInProject({
+                        projectId: input.monitor.projectId,
+                        id: criteriaAlert.alertSeverityId,
+                        service: AlertSeverityService,
+                    });
+                    if (((_x = criteriaAlert.alertSeverityId) === null || _x === void 0 ? void 0 : _x.toString()) &&
+                        !isCriteriaSeverityUsable) {
+                        logger.error(`${(_y = input.monitor.id) === null || _y === void 0 ? void 0 : _y.toString()} - Criteria "${(_z = input.criteriaInstance.data) === null || _z === void 0 ? void 0 : _z.name}" references alert severity ${criteriaAlert.alertSeverityId.toString()}, which does not belong to project ${(_0 = input.monitor.projectId) === null || _0 === void 0 ? void 0 : _0.toString()}. Falling back to this project's default severity.`);
                     }
-                    else {
+                    if (!isCriteriaSeverityUsable) {
+                        // pick the critical criteria.
+                        const severity = await AlertSeverityService.findOneBy({
+                            query: {
+                                projectId: input.monitor.projectId,
+                            },
+                            sort: {
+                                order: SortOrder.Ascending,
+                            },
+                            props: {
+                                isRoot: true,
+                            },
+                            select: {
+                                _id: true,
+                            },
+                        });
+                        if (!((_1 = severity === null || severity === void 0 ? void 0 : severity.id) === null || _1 === void 0 ? void 0 : _1.toString())) {
+                            /*
+                             * The project has no alert severity configured. Throwing here
+                             * would fail the entire probe/telemetry ingest job — which
+                             * then retries forever over a misconfiguration the worker
+                             * cannot fix, and takes every other series on this monitor
+                             * down with it. Skip this alert and log instead, exactly as
+                             * the incident path does.
+                             */
+                            logger.error(`${(_2 = input.monitor.id) === null || _2 === void 0 ? void 0 : _2.toString()} - Cannot create alert: project ${(_3 = input.monitor.projectId) === null || _3 === void 0 ? void 0 : _3.toString()} has no alert severity configured. Skipping alert creation for criteria "${(_4 = input.criteriaInstance.data) === null || _4 === void 0 ? void 0 : _4.name}".`);
+                            (_5 = input.evaluationSummary) === null || _5 === void 0 ? void 0 : _5.events.push({
+                                type: "alert-skipped",
+                                title: "Alert creation skipped",
+                                message: "Skipped creating an alert because the project has no alert severity configured.",
+                                relatedCriteriaId: (_6 = input.criteriaInstance.data) === null || _6 === void 0 ? void 0 : _6.id,
+                                at: OneUptimeDate.getCurrentDate(),
+                            });
+                            continue;
+                        }
                         alert.alertSeverityId = severity.id;
                     }
-                }
-                else {
-                    alert.alertSeverityId = criteriaAlert.alertSeverityId;
-                }
-                alert.monitor = input.monitor;
-                alert.projectId = input.monitor.projectId;
-                alert.rootCause = seriesRootCause;
-                alert.createdStateLog = JSON.parse(JSON.stringify(input.dataToProcess, null, 2));
-                /*
-                 * Same capture on every alert this evaluation opens - they all
-                 * came from the one check.
-                 */
-                const serializedMonitorSummary = MonitorSummarySnapshotUtil.serialize(input.monitorSummary);
-                if (serializedMonitorSummary) {
-                    alert.monitorSummary = serializedMonitorSummary;
-                }
-                if ((_z = input.criteriaInstance.data) === null || _z === void 0 ? void 0 : _z.id) {
-                    alert.createdCriteriaId = input.criteriaInstance.data.id.toString();
-                }
-                if (seriesFingerprint) {
-                    alert.seriesFingerprint = seriesFingerprint;
-                }
-                if (seriesLabels && Object.keys(seriesLabels).length > 0) {
-                    alert.seriesLabels = seriesLabels;
+                    else {
+                        alert.alertSeverityId = criteriaAlert.alertSeverityId;
+                    }
+                    alert.monitor = input.monitor;
+                    alert.projectId = input.monitor.projectId;
+                    alert.rootCause = seriesRootCause;
+                    alert.createdStateLog = JSON.parse(JSON.stringify(input.dataToProcess, null, 2));
                     /*
-                     * Attach every resource this series identifies — host, docker
-                     * host, podman host, k8s cluster, service, and the Proxmox /
-                     * Ceph / Swarm / IoT clusters — resolved from the shared label
-                     * key map. Same call the incident path makes, so the two can't
-                     * drift apart again.
+                     * Same capture on every alert this evaluation opens - they all
+                     * came from the one check.
                      */
-                    await SeriesResourceLinker.linkSeriesResourcesToModel({
+                    const serializedMonitorSummary = MonitorSummarySnapshotUtil.serialize(input.monitorSummary);
+                    if (serializedMonitorSummary) {
+                        alert.monitorSummary = serializedMonitorSummary;
+                    }
+                    if ((_7 = input.criteriaInstance.data) === null || _7 === void 0 ? void 0 : _7.id) {
+                        alert.createdCriteriaId = input.criteriaInstance.data.id.toString();
+                    }
+                    if (seriesFingerprint) {
+                        alert.seriesFingerprint = seriesFingerprint;
+                    }
+                    if (seriesLabels && Object.keys(seriesLabels).length > 0) {
+                        alert.seriesLabels = seriesLabels;
+                        /*
+                         * Attach every resource this series identifies — host, docker
+                         * host, podman host, k8s cluster, service, and the Proxmox /
+                         * Ceph / Swarm / IoT clusters — resolved from the shared label
+                         * key map. Same call the incident path makes, so the two can't
+                         * drift apart again.
+                         */
+                        await SeriesResourceLinker.linkSeriesResourcesToModel({
+                            model: alert,
+                            seriesLabels,
+                            projectId: input.monitor.projectId,
+                            monitorType: input.monitor.monitorType,
+                        });
+                    }
+                    /*
+                     * Deterministic resource link from the monitor's step config
+                     * (resolved once above). For most monitor types this — not the
+                     * label path above — is what makes the per-resource Activity
+                     * tabs and badge counts see monitor-created alerts, because
+                     * their criteria are ungrouped and emit no series labels at
+                     * all. Runs for both grouped and ungrouped alerts, and merges
+                     * with whatever the labels resolved.
+                     */
+                    SeriesResourceLinker.attachResolvedResources({
                         model: alert,
-                        seriesLabels,
-                        projectId: input.monitor.projectId,
-                        monitorType: input.monitor.monitorType,
+                        resolved: resourceContext,
                     });
-                }
-                /*
-                 * Deterministic resource link from the monitor's step config
-                 * (resolved once above). For most monitor types this — not the
-                 * label path above — is what makes the per-resource Activity
-                 * tabs and badge counts see monitor-created alerts, because
-                 * their criteria are ungrouped and emit no series labels at
-                 * all. Runs for both grouped and ungrouped alerts, and merges
-                 * with whatever the labels resolved.
-                 */
-                SeriesResourceLinker.attachResolvedResources({
-                    model: alert,
-                    resolved: resourceContext,
-                });
-                alert.onCallDutyPolicies =
-                    ((_0 = criteriaAlert.onCallPolicyIds) === null || _0 === void 0 ? void 0 : _0.map((id) => {
-                        const onCallPolicy = new OnCallDutyPolicy();
-                        onCallPolicy._id = id.toString();
-                        return onCallPolicy;
-                    })) || [];
-                // Set labels from criteria
-                alert.labels =
-                    ((_1 = criteriaAlert.labelIds) === null || _1 === void 0 ? void 0 : _1.map((id) => {
-                        const label = new Label();
-                        label._id = id.toString();
-                        return label;
-                    })) || [];
-                alert.isCreatedAutomatically = true;
-                if (criteriaAlert.isPrivate === true) {
-                    alert.isPrivate = true;
-                }
-                if (input.props.telemetryQuery) {
-                    alert.telemetryQuery = input.props.telemetryQuery;
-                }
-                if (input.dataToProcess &&
-                    input.dataToProcess.probeId) {
-                    alert.createdByProbeId = input.dataToProcess.probeId;
-                }
-                if (criteriaAlert.remediationNotes) {
-                    alert.remediationNotes = MonitorTemplateUtil.processTemplateString({
-                        value: criteriaAlert.remediationNotes,
-                        storageMap,
+                    alert.onCallDutyPolicies =
+                        ((_8 = criteriaAlert.onCallPolicyIds) === null || _8 === void 0 ? void 0 : _8.map((id) => {
+                            const onCallPolicy = new OnCallDutyPolicy();
+                            onCallPolicy._id = id.toString();
+                            return onCallPolicy;
+                        })) || [];
+                    // Set labels from criteria
+                    alert.labels =
+                        ((_9 = criteriaAlert.labelIds) === null || _9 === void 0 ? void 0 : _9.map((id) => {
+                            const label = new Label();
+                            label._id = id.toString();
+                            return label;
+                        })) || [];
+                    alert.isCreatedAutomatically = true;
+                    if (criteriaAlert.isPrivate === true) {
+                        alert.isPrivate = true;
+                    }
+                    if (input.props.telemetryQuery) {
+                        alert.telemetryQuery = input.props.telemetryQuery;
+                    }
+                    if (input.dataToProcess &&
+                        input.dataToProcess.probeId) {
+                        alert.createdByProbeId = input.dataToProcess.probeId;
+                    }
+                    if (criteriaAlert.remediationNotes) {
+                        alert.remediationNotes = MonitorTemplateUtil.processTemplateString({
+                            value: criteriaAlert.remediationNotes,
+                            storageMap,
+                        });
+                    }
+                    const createdAlert = await AlertService.create({
+                        data: alert,
+                        props: {
+                            isRoot: true,
+                        },
                     });
-                }
-                if (DisableAutomaticAlertCreation) {
-                    (_2 = input.evaluationSummary) === null || _2 === void 0 ? void 0 : _2.events.push({
-                        type: "alert-skipped",
-                        title: "Alert creation skipped",
-                        message: "Automatic alert creation is disabled by environment configuration.",
-                        relatedCriteriaId: (_3 = input.criteriaInstance.data) === null || _3 === void 0 ? void 0 : _3.id,
+                    /*
+                     * Add owner teams and users after alert creation. Owners configured
+                     * on the criteria template are merged (deduped) with the owners of
+                     * the network device this monitor watches, so device ownership flows
+                     * into the alerts its monitor raises.
+                     */
+                    if (networkDeviceOwners === null) {
+                        networkDeviceOwners =
+                            await NetworkDeviceOwnerUserService.getDeviceOwnersForMonitor({
+                                monitor: input.monitor,
+                                monitorStepId: (_10 = input.dataToProcess.monitorStepId) === null || _10 === void 0 ? void 0 : _10.toString(),
+                            });
+                    }
+                    const ownerUserIds = this.mergeOwnerIds(criteriaAlert.ownerUserIds || [], networkDeviceOwners.ownerUserIds);
+                    const ownerTeamIds = this.mergeOwnerIds(criteriaAlert.ownerTeamIds || [], networkDeviceOwners.ownerTeamIds);
+                    if (ownerTeamIds.length || ownerUserIds.length) {
+                        await AlertService.addOwners(input.monitor.projectId, createdAlert.id, ownerUserIds, ownerTeamIds, true, // notify owners
+                        {
+                            isRoot: true,
+                        });
+                    }
+                    (_11 = input.evaluationSummary) === null || _11 === void 0 ? void 0 : _11.events.push({
+                        type: "alert-created",
+                        title: `Alert created: ${createdAlert.title || criteriaAlert.title}`,
+                        message: `Alert triggered from criteria "${((_12 = input.criteriaInstance.data) === null || _12 === void 0 ? void 0 : _12.name) || "Unnamed criteria"}".`,
+                        relatedCriteriaId: (_13 = input.criteriaInstance.data) === null || _13 === void 0 ? void 0 : _13.id,
+                        relatedAlertId: (_14 = createdAlert.id) === null || _14 === void 0 ? void 0 : _14.toString(),
+                        relatedAlertNumber: createdAlert.alertNumber,
+                        relatedAlertNumberWithPrefix: createdAlert.alertNumberWithPrefix,
                         at: OneUptimeDate.getCurrentDate(),
                     });
-                    return;
-                }
-                const createdAlert = await AlertService.create({
-                    data: alert,
-                    props: {
-                        isRoot: true,
-                    },
-                });
-                /*
-                 * Add owner teams and users after alert creation. Owners configured
-                 * on the criteria template are merged (deduped) with the owners of
-                 * the network device this monitor watches, so device ownership flows
-                 * into the alerts its monitor raises.
-                 */
-                if (networkDeviceOwners === null) {
-                    networkDeviceOwners =
-                        await NetworkDeviceOwnerUserService.getDeviceOwnersForMonitor({
-                            monitor: input.monitor,
-                            monitorStepId: (_4 = input.dataToProcess.monitorStepId) === null || _4 === void 0 ? void 0 : _4.toString(),
-                        });
                 }
-                const ownerUserIds = this.mergeOwnerIds(criteriaAlert.ownerUserIds || [], networkDeviceOwners.ownerUserIds);
-                const ownerTeamIds = this.mergeOwnerIds(criteriaAlert.ownerTeamIds || [], networkDeviceOwners.ownerTeamIds);
-                if (ownerTeamIds.length || ownerUserIds.length) {
-                    await AlertService.addOwners(input.monitor.projectId, createdAlert.id, ownerUserIds, ownerTeamIds, true, // notify owners
-                    {
-                        isRoot: true,
+                catch (err) {
+                    /*
+                     * One series must not take the rest of the fleet down with it.
+                     * Everything below this loop happens per host/container/key and
+                     * touches the database; an error thrown from here used to
+                     * propagate out of MonitorResource (which has no catch, only a
+                     * `finally` that releases the lock) and abandon every series
+                     * after the failing one, plus the monitor log for the tick.
+                     */
+                    logger.error(`${(_15 = input.monitor.id) === null || _15 === void 0 ? void 0 : _15.toString()} - Failed to create alert${(seriesMatch === null || seriesMatch === void 0 ? void 0 : seriesMatch.fingerprint)
+                        ? ` for series ${seriesMatch.fingerprint}`
+                        : ""}.`);
+                    logger.error(err);
+                    (_16 = input.evaluationSummary) === null || _16 === void 0 ? void 0 : _16.events.push({
+                        type: "alert-skipped",
+                        title: "Alert creation failed",
+                        message: `Could not create an alert${(seriesMatch === null || seriesMatch === void 0 ? void 0 : seriesMatch.fingerprint)
+                            ? ` for series ${seriesMatch.fingerprint}`
+                            : ""}: ${err instanceof Error ? err.message : String(err)}`,
+                        relatedCriteriaId: (_17 = input.criteriaInstance.data) === null || _17 === void 0 ? void 0 : _17.id,
+                        at: OneUptimeDate.getCurrentDate(),
                     });
+                    continue;
                 }
-                (_5 = input.evaluationSummary) === null || _5 === void 0 ? void 0 : _5.events.push({
-                    type: "alert-created",
-                    title: `Alert created: ${createdAlert.title || criteriaAlert.title}`,
-                    message: `Alert triggered from criteria "${((_6 = input.criteriaInstance.data) === null || _6 === void 0 ? void 0 : _6.name) || "Unnamed criteria"}".`,
-                    relatedCriteriaId: (_7 = input.criteriaInstance.data) === null || _7 === void 0 ? void 0 : _7.id,
-                    relatedAlertId: (_8 = createdAlert.id) === null || _8 === void 0 ? void 0 : _8.toString(),
-                    relatedAlertNumber: createdAlert.alertNumber,
-                    relatedAlertNumberWithPrefix: createdAlert.alertNumberWithPrefix,
-                    at: OneUptimeDate.getCurrentDate(),
-                });
             }
         }
     }
@@ -532,8 +595,51 @@ export default class MonitorAlert {
             }
         }
     }
+    /**
+     * Is `openAlert`'s series still breaching the criteria that raised it?
+     *
+     * Prefers the per-criteria dictionary, which answers the question for
+     * the criteria that actually owns this alert. The single-set fallback
+     * only knows the winning criteria's breaches, so it has to approximate
+     * with `createAlerts` — a criteria that creates no alerts is a
+     * recovery criteria whose "matches" are healthy series, and counting
+     * those as breaches would pin an offline alert open forever.
+     */
+    static isSeriesStillBreaching(input) {
+        var _a, _b, _c, _d;
+        const owningCriteriaId = ((_a = input.openAlert.createdCriteriaId) === null || _a === void 0 ? void 0 : _a.toString()) || undefined;
+        if (input.breachingSeriesFingerprintsByCriteriaId && owningCriteriaId) {
+            return Boolean((_b = input.breachingSeriesFingerprintsByCriteriaId[owningCriteriaId]) === null || _b === void 0 ? void 0 : _b.has(input.openSeriesFingerprint));
+        }
+        const matchedCriteriaCreatesAlerts = ((_d = (_c = input.criteriaInstance) === null || _c === void 0 ? void 0 : _c.data) === null || _d === void 0 ? void 0 : _d.createAlerts) === true;
+        return (matchedCriteriaCreatesAlerts &&
+            input.breachingSeriesFingerprints.has(input.openSeriesFingerprint));
+    }
+    static wasCriteriaEvaluated(input) {
+        var _a;
+        const owningCriteriaId = ((_a = input.openAlert.createdCriteriaId) === null || _a === void 0 ? void 0 : _a.toString()) || undefined;
+        if (!owningCriteriaId) {
+            return true;
+        }
+        return (input.breachingSeriesFingerprintsByCriteriaId[owningCriteriaId] !==
+            undefined);
+    }
+    /**
+     * Alert auto-resolve lists templates by criteria; the presence of any
+     * template for a criteria means "this criteria's alerts are configured
+     * to auto-resolve".
+     */
+    static isAutoResolveConfiguredForAlert(input) {
+        var _a;
+        const createdCriteriaId = ((_a = input.openAlert.createdCriteriaId) === null || _a === void 0 ? void 0 : _a.toString()) || undefined;
+        if (!createdCriteriaId) {
+            return false;
+        }
+        const autoResolveTemplates = input.autoResolveCriteriaInstanceIdAlertIdsDictionary[createdCriteriaId];
+        return Boolean(autoResolveTemplates && autoResolveTemplates.length > 0);
+    }
     static shouldCloseAlert(input) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j;
+        var _a, _b, _c, _d, _e, _f;
         const openSeriesFingerprint = input.openAlert.seriesFingerprint || undefined;
         /*
          * Event-driven (incoming-request / webhook) per-key alerts must NEVER
@@ -546,53 +652,62 @@ export default class MonitorAlert {
         if (input.disableSeriesAbsenceResolution && openSeriesFingerprint) {
             return false;
         }
-        /*
-         * Per-series auto-resolve: when a breaching-series set is given and
-         * this alert has a fingerprint, resolve whenever the fingerprint is
-         * no longer in the set, independent of whether other series on the
-         * same monitor are still breaching.
-         */
-        if (input.breachingSeriesFingerprints !== undefined &&
-            openSeriesFingerprint) {
+        // Per-series mode: this evaluation knows which series are breaching.
+        if (input.breachingSeriesFingerprints !== undefined) {
             /*
-             * The breaching set is the matched criteria's per-series matches.
-             * On a recovery tick the matched criteria is the RECOVERY criteria
-             * and its matches are healthy series, not breaches; counting them
-             * would pin the open offline alert open forever. Membership only
-             * counts as still-breaching when the matched criteria actually
-             * creates alerts (createAlerts=false recovery criteria contributes
-             * no breaches, letting the per-series alert auto-resolve).
+             * A whole-monitor alert (no fingerprint) on a monitor that now
+             * alerts per series. It was raised before the monitor was grouped,
+             * and nothing can ever dedupe against it again — every alert from
+             * here on carries a fingerprint. Left alone it stays open forever
+             * while its replacements come and go. Resolve it, on the same
+             * auto-resolve terms as any other alert from its criteria.
              */
-            const matchedCriteriaCreatesAlerts = ((_b = (_a = input.criteriaInstance) === null || _a === void 0 ? void 0 : _a.data) === null || _b === void 0 ? void 0 : _b.createAlerts) === true;
-            const stillBreaching = matchedCriteriaCreatesAlerts &&
-                input.breachingSeriesFingerprints.has(openSeriesFingerprint);
-            if (stillBreaching) {
-                return false;
+            if (!openSeriesFingerprint) {
+                if (input.disableSeriesAbsenceResolution) {
+                    return false;
+                }
+                return MonitorAlert.isAutoResolveConfiguredForAlert({
+                    openAlert: input.openAlert,
+                    autoResolveCriteriaInstanceIdAlertIdsDictionary: input.autoResolveCriteriaInstanceIdAlertIdsDictionary,
+                });
             }
-            if (!((_c = input.openAlert.createdCriteriaId) === null || _c === void 0 ? void 0 : _c.toString())) {
+            if (MonitorAlert.isSeriesStillBreaching({
+                openAlert: input.openAlert,
+                criteriaInstance: input.criteriaInstance,
+                openSeriesFingerprint,
+                breachingSeriesFingerprints: input.breachingSeriesFingerprints,
+                breachingSeriesFingerprintsByCriteriaId: input.breachingSeriesFingerprintsByCriteriaId,
+            })) {
                 return false;
             }
-            const autoResolveTemplates = input.autoResolveCriteriaInstanceIdAlertIdsDictionary[input.openAlert.createdCriteriaId.toString()];
             /*
-             * Alert auto-resolve lists templates by criteria; presence of any
-             * template for this criteria means "this criteria's alerts are
-             * configured to auto-resolve", so resolve this series.
+             * The criteria that raised this alert was not evaluated on this
+             * tick at all (disabled, deleted, or the monitor stops at its
+             * first match). Absence of a breaching set for it is not evidence
+             * of recovery, so leave the alert alone.
              */
-            if (autoResolveTemplates && autoResolveTemplates.length > 0) {
-                return true;
+            if (input.breachingSeriesFingerprintsByCriteriaId &&
+                !MonitorAlert.wasCriteriaEvaluated({
+                    openAlert: input.openAlert,
+                    breachingSeriesFingerprintsByCriteriaId: input.breachingSeriesFingerprintsByCriteriaId,
+                })) {
+                return false;
             }
-            return false;
+            return MonitorAlert.isAutoResolveConfiguredForAlert({
+                openAlert: input.openAlert,
+                autoResolveCriteriaInstanceIdAlertIdsDictionary: input.autoResolveCriteriaInstanceIdAlertIdsDictionary,
+            });
         }
-        if (((_d = input.openAlert.createdCriteriaId) === null || _d === void 0 ? void 0 : _d.toString()) ===
-            ((_g = (_f = (_e = input.criteriaInstance) === null || _e === void 0 ? void 0 : _e.data) === null || _f === void 0 ? void 0 : _f.id) === null || _g === void 0 ? void 0 : _g.toString())) {
+        if (((_a = input.openAlert.createdCriteriaId) === null || _a === void 0 ? void 0 : _a.toString()) ===
+            ((_d = (_c = (_b = input.criteriaInstance) === null || _b === void 0 ? void 0 : _b.data) === null || _c === void 0 ? void 0 : _c.id) === null || _d === void 0 ? void 0 : _d.toString())) {
             // same alert active. So, do not close.
             return false;
         }
         // If antoher criteria is active then, check if the alert id is present in the map.
-        if (!((_h = input.openAlert.createdCriteriaId) === null || _h === void 0 ? void 0 : _h.toString())) {
+        if (!((_e = input.openAlert.createdCriteriaId) === null || _e === void 0 ? void 0 : _e.toString())) {
             return false;
         }
-        if (input.autoResolveCriteriaInstanceIdAlertIdsDictionary[(_j = input.openAlert.createdCriteriaId) === null || _j === void 0 ? void 0 : _j.toString()]) {
+        if (input.autoResolveCriteriaInstanceIdAlertIdsDictionary[(_f = input.openAlert.createdCriteriaId) === null || _f === void 0 ? void 0 : _f.toString()]) {
             return true;
         }
         return false;