@oneuptime/common 11.3.6 → 11.3.8

package/build/dist/Server/Services/AlertService.js

@@ -31,6 +31,7 @@ import AlertOwnerUser from "../../Models/DatabaseModels/AlertOwnerUser";
 import AlertStateTimeline from "../../Models/DatabaseModels/AlertStateTimeline";
 import { IsBillingEnabled } from "../EnvironmentConfig";
 import logger from "../Utils/Logger";
+import Semaphore from "../Infrastructure/Semaphore";
 import TelemetryUtil from "../Utils/Telemetry/Telemetry";
 import MetricService from "./MetricService";
 import GlobalConfigService from "./GlobalConfigService";
@@ -979,7 +980,7 @@ ${alertSeverity.name}
         return Service.DEFAULT_METRIC_RETENTION_DAYS;
     }
     async refreshAlertMetrics(data) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7;
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v, _w, _x, _y, _z, _0, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11;
         const alert = await this.findOneById({
             id: data.alertId,
             select: {
@@ -1028,181 +1029,274 @@ ${alertSeverity.name}
             },
         });
         const firstAlertStateTimeline = alertStateTimelines[0];
-        // delete all the alert metrics with this alert id because it's a refresh
-        await MetricService.deleteBy({
-            query: {
-                primaryEntityId: data.alertId,
-            },
-            props: {
-                isRoot: true,
-            },
-        });
-        const itemsToSave = [];
-        const metricTypesMap = {};
-        const metricRetentionDays = await this.getMetricRetentionDays();
-        const alertMetricRetentionDate = OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), metricRetentionDays);
-        // now we need to create new metrics for this alert - TimeToAcknowledge, TimeToResolve, AlertCount, AlertDuration
-        const alertStartsAt = (firstAlertStateTimeline === null || firstAlertStateTimeline === void 0 ? void 0 : firstAlertStateTimeline.startsAt) ||
-            alert.createdAt ||
-            OneUptimeDate.getCurrentDate();
-        const alertCountMetric = new Metric();
-        alertCountMetric.projectId = alert.projectId;
-        alertCountMetric.primaryEntityId = alert.id;
-        alertCountMetric.primaryEntityType = ServiceType.Alert;
-        alertCountMetric.name = AlertMetricType.AlertCount;
-        alertCountMetric.value = 1;
-        alertCountMetric.attributes = {
-            alertId: data.alertId.toString(),
-            projectId: alert.projectId.toString(),
-            monitorId: (_b = (_a = alert.monitor) === null || _a === void 0 ? void 0 : _a._id) === null || _b === void 0 ? void 0 : _b.toString(),
-            monitorName: (_d = (_c = alert.monitor) === null || _c === void 0 ? void 0 : _c.name) === null || _d === void 0 ? void 0 : _d.toString(),
-            alertSeverityId: (_f = (_e = alert.alertSeverity) === null || _e === void 0 ? void 0 : _e._id) === null || _f === void 0 ? void 0 : _f.toString(),
-            alertSeverityName: (_h = (_g = alert.alertSeverity) === null || _g === void 0 ? void 0 : _g.name) === null || _h === void 0 ? void 0 : _h.toString(),
-        };
-        alertCountMetric.attributeKeys = TelemetryUtil.getAttributeKeys(alertCountMetric.attributes);
-        alertCountMetric.time = alertStartsAt;
-        alertCountMetric.timeUnixNano = OneUptimeDate.toUnixNano(alertCountMetric.time);
-        alertCountMetric.metricPointType = MetricPointType.Sum;
-        alertCountMetric.retentionDate = alertMetricRetentionDate;
-        itemsToSave.push(alertCountMetric);
-        const metricType = new MetricType();
-        metricType.name = alertCountMetric.name;
-        metricType.description = "Number of alerts created";
-        metricType.unit = "";
-        metricType.services = [];
-        metricTypesMap[alertCountMetric.name] = metricType;
-        // is the alert acknowledged?
-        const isAlertAcknowledged = alertStateTimelines.some((timeline) => {
-            var _a;
-            return (_a = timeline.alertState) === null || _a === void 0 ? void 0 : _a.isAcknowledgedState;
-        });
-        if (isAlertAcknowledged) {
-            const ackAlertStateTimeline = alertStateTimelines.find((timeline) => {
-                var _a;
-                return (_a = timeline.alertState) === null || _a === void 0 ? void 0 : _a.isAcknowledgedState;
+        /*
+         * Serialize concurrent refreshes for this alert across pods. The
+         * write-once guard below is read-then-insert (findBy existing, then a
+         * conditional createMany). Both call sites fire refresh fire-and-forget
+         * after releasing the per-alert state-timeline mutex, so two
+         * close-together transitions (e.g. auto-acknowledge then auto-resolve)
+         * could interleave: both read a snapshot missing a metric and both
+         * insert it. The Metric table is a plain MergeTree with no row collapse,
+         * so those duplicates would be permanent and re-inflate the very Sum/Avg
+         * widgets this change fixes. A best-effort distributed lock keyed on the
+         * alert makes the read+insert atomic; if the lock can't be taken we still
+         * proceed (the existence check alone covers the common sequential case).
+         */
+        let metricRefreshMutex = null;
+        try {
+            metricRefreshMutex = await Semaphore.lock({
+                key: data.alertId.toString(),
+                namespace: "AlertService.refreshAlertMetrics",
+                lockTimeout: 30000,
+            });
+        }
+        catch (err) {
+            logger.error(err, {
+                projectId: (_a = alert.projectId) === null || _a === void 0 ? void 0 : _a.toString(),
+                alertId: (_b = alert.id) === null || _b === void 0 ? void 0 : _b.toString(),
             });
-            if (ackAlertStateTimeline) {
-                const timeToAcknowledgeMetric = new Metric();
-                timeToAcknowledgeMetric.projectId = alert.projectId;
-                timeToAcknowledgeMetric.primaryEntityId = alert.id;
-                timeToAcknowledgeMetric.primaryEntityType = ServiceType.Alert;
-                timeToAcknowledgeMetric.name = AlertMetricType.TimeToAcknowledge;
-                timeToAcknowledgeMetric.value = OneUptimeDate.getDifferenceInSeconds((ackAlertStateTimeline === null || ackAlertStateTimeline === void 0 ? void 0 : ackAlertStateTimeline.startsAt) || OneUptimeDate.getCurrentDate(), alertStartsAt);
-                timeToAcknowledgeMetric.attributes = {
+        }
+        try {
+            /*
+             * Write-once metrics — no deletes.
+             *
+             * The raw Metric table (`MetricItemV3`) feeds an AggregatingMergeTree
+             * materialized view (`MetricItemAggMV1m_mv`) that accumulates
+             * `sumState`/`countState` per (projectId, name, primaryEntityId,
+             * minute). The MV trigger fires only on INSERT — an `ALTER ... DELETE`
+             * mutation on the source rolls back nothing in the MV. So the old
+             * "delete every metric for this alert, then re-insert" refresh both
+             * (a) inflated those rollups and (b) issued one heavy `ALTER ... DELETE`
+             * mutation per state transition per alert — a mutation storm.
+             *
+             * Every alert metric has a single eventually-final value: AlertCount is
+             * constant `value = 1`; MTTA/MTTR/duration are each fixed once the
+             * relevant state transition has happened. So we load what we've already
+             * emitted for this alert and insert each metric exactly once, when it
+             * becomes final. Repeat refreshes that find nothing new are no-ops.
+             */
+            const existingMetricNames = new Set((await MetricService.findBy({
+                query: {
+                    projectId: alert.projectId,
+                    primaryEntityId: data.alertId,
+                },
+                select: {
+                    name: true,
+                },
+                skip: 0,
+                limit: LIMIT_PER_PROJECT,
+                props: {
+                    isRoot: true,
+                },
+            }))
+                .map((metric) => {
+                return metric.name;
+            })
+                .filter(Boolean));
+            const itemsToSave = [];
+            const metricTypesMap = {};
+            const metricRetentionDays = await this.getMetricRetentionDays();
+            const alertMetricRetentionDate = OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), metricRetentionDays);
+            // now we need to create new metrics for this alert - TimeToAcknowledge, TimeToResolve, AlertCount, AlertDuration
+            const alertStartsAt = (firstAlertStateTimeline === null || firstAlertStateTimeline === void 0 ? void 0 : firstAlertStateTimeline.startsAt) ||
+                alert.createdAt ||
+                OneUptimeDate.getCurrentDate();
+            // register the metric type so the catalog stays complete across refreshes.
+            const alertCountMetricType = new MetricType();
+            alertCountMetricType.name = AlertMetricType.AlertCount;
+            alertCountMetricType.description = "Number of alerts created";
+            alertCountMetricType.unit = "";
+            alertCountMetricType.services = [];
+            metricTypesMap[AlertMetricType.AlertCount] = alertCountMetricType;
+            // write-once: AlertCount is a constant value=1 keyed by the alert.
+            if (!existingMetricNames.has(AlertMetricType.AlertCount)) {
+                const alertCountMetric = new Metric();
+                alertCountMetric.projectId = alert.projectId;
+                alertCountMetric.primaryEntityId = alert.id;
+                alertCountMetric.primaryEntityType = ServiceType.Alert;
+                alertCountMetric.name = AlertMetricType.AlertCount;
+                alertCountMetric.value = 1;
+                alertCountMetric.attributes = {
                     alertId: data.alertId.toString(),
                     projectId: alert.projectId.toString(),
-                    monitorId: (_k = (_j = alert.monitor) === null || _j === void 0 ? void 0 : _j._id) === null || _k === void 0 ? void 0 : _k.toString(),
-                    monitorName: (_m = (_l = alert.monitor) === null || _l === void 0 ? void 0 : _l.name) === null || _m === void 0 ? void 0 : _m.toString(),
-                    alertSeverityId: (_p = (_o = alert.alertSeverity) === null || _o === void 0 ? void 0 : _o._id) === null || _p === void 0 ? void 0 : _p.toString(),
-                    alertSeverityName: (_r = (_q = alert.alertSeverity) === null || _q === void 0 ? void 0 : _q.name) === null || _r === void 0 ? void 0 : _r.toString(),
+                    monitorId: (_d = (_c = alert.monitor) === null || _c === void 0 ? void 0 : _c._id) === null || _d === void 0 ? void 0 : _d.toString(),
+                    monitorName: (_f = (_e = alert.monitor) === null || _e === void 0 ? void 0 : _e.name) === null || _f === void 0 ? void 0 : _f.toString(),
+                    alertSeverityId: (_h = (_g = alert.alertSeverity) === null || _g === void 0 ? void 0 : _g._id) === null || _h === void 0 ? void 0 : _h.toString(),
+                    alertSeverityName: (_k = (_j = alert.alertSeverity) === null || _j === void 0 ? void 0 : _j.name) === null || _k === void 0 ? void 0 : _k.toString(),
                 };
-                timeToAcknowledgeMetric.attributeKeys = TelemetryUtil.getAttributeKeys(timeToAcknowledgeMetric.attributes);
-                timeToAcknowledgeMetric.time =
-                    (ackAlertStateTimeline === null || ackAlertStateTimeline === void 0 ? void 0 : ackAlertStateTimeline.startsAt) ||
-                        alert.createdAt ||
-                        OneUptimeDate.getCurrentDate();
-                timeToAcknowledgeMetric.timeUnixNano = OneUptimeDate.toUnixNano(timeToAcknowledgeMetric.time);
-                timeToAcknowledgeMetric.metricPointType = MetricPointType.Sum;
-                timeToAcknowledgeMetric.retentionDate = alertMetricRetentionDate;
-                itemsToSave.push(timeToAcknowledgeMetric);
-                const metricType = new MetricType();
-                metricType.name = timeToAcknowledgeMetric.name;
-                metricType.description = "Time taken to acknowledge the alert";
-                metricType.unit = "seconds";
-                metricTypesMap[timeToAcknowledgeMetric.name] = metricType;
+                alertCountMetric.attributeKeys = TelemetryUtil.getAttributeKeys(alertCountMetric.attributes);
+                alertCountMetric.time = alertStartsAt;
+                alertCountMetric.timeUnixNano = OneUptimeDate.toUnixNano(alertCountMetric.time);
+                alertCountMetric.metricPointType = MetricPointType.Sum;
+                alertCountMetric.retentionDate = alertMetricRetentionDate;
+                itemsToSave.push(alertCountMetric);
             }
-        }
-        // time to resolve
-        const isAlertResolved = alertStateTimelines.some((timeline) => {
-            var _a;
-            return (_a = timeline.alertState) === null || _a === void 0 ? void 0 : _a.isResolvedState;
-        });
-        if (isAlertResolved) {
+            // is the alert acknowledged?
+            const isAlertAcknowledged = alertStateTimelines.some((timeline) => {
+                var _a;
+                return (_a = timeline.alertState) === null || _a === void 0 ? void 0 : _a.isAcknowledgedState;
+            });
+            if (isAlertAcknowledged) {
+                const ackAlertStateTimeline = alertStateTimelines.find((timeline) => {
+                    var _a;
+                    return (_a = timeline.alertState) === null || _a === void 0 ? void 0 : _a.isAcknowledgedState;
+                });
+                if (ackAlertStateTimeline) {
+                    // register the metric type so the catalog stays complete across refreshes.
+                    const metricType = new MetricType();
+                    metricType.name = AlertMetricType.TimeToAcknowledge;
+                    metricType.description = "Time taken to acknowledge the alert";
+                    metricType.unit = "seconds";
+                    metricTypesMap[AlertMetricType.TimeToAcknowledge] = metricType;
+                    // write-once: MTTA is fixed once the alert is first acknowledged.
+                    if (!existingMetricNames.has(AlertMetricType.TimeToAcknowledge)) {
+                        const timeToAcknowledgeMetric = new Metric();
+                        timeToAcknowledgeMetric.projectId = alert.projectId;
+                        timeToAcknowledgeMetric.primaryEntityId = alert.id;
+                        timeToAcknowledgeMetric.primaryEntityType = ServiceType.Alert;
+                        timeToAcknowledgeMetric.name = AlertMetricType.TimeToAcknowledge;
+                        timeToAcknowledgeMetric.value =
+                            OneUptimeDate.getDifferenceInSeconds((ackAlertStateTimeline === null || ackAlertStateTimeline === void 0 ? void 0 : ackAlertStateTimeline.startsAt) ||
+                                OneUptimeDate.getCurrentDate(), alertStartsAt);
+                        timeToAcknowledgeMetric.attributes = {
+                            alertId: data.alertId.toString(),
+                            projectId: alert.projectId.toString(),
+                            monitorId: (_m = (_l = alert.monitor) === null || _l === void 0 ? void 0 : _l._id) === null || _m === void 0 ? void 0 : _m.toString(),
+                            monitorName: (_p = (_o = alert.monitor) === null || _o === void 0 ? void 0 : _o.name) === null || _p === void 0 ? void 0 : _p.toString(),
+                            alertSeverityId: (_r = (_q = alert.alertSeverity) === null || _q === void 0 ? void 0 : _q._id) === null || _r === void 0 ? void 0 : _r.toString(),
+                            alertSeverityName: (_t = (_s = alert.alertSeverity) === null || _s === void 0 ? void 0 : _s.name) === null || _t === void 0 ? void 0 : _t.toString(),
+                        };
+                        timeToAcknowledgeMetric.attributeKeys =
+                            TelemetryUtil.getAttributeKeys(timeToAcknowledgeMetric.attributes);
+                        timeToAcknowledgeMetric.time =
+                            (ackAlertStateTimeline === null || ackAlertStateTimeline === void 0 ? void 0 : ackAlertStateTimeline.startsAt) ||
+                                alert.createdAt ||
+                                OneUptimeDate.getCurrentDate();
+                        timeToAcknowledgeMetric.timeUnixNano = OneUptimeDate.toUnixNano(timeToAcknowledgeMetric.time);
+                        timeToAcknowledgeMetric.metricPointType = MetricPointType.Sum;
+                        timeToAcknowledgeMetric.retentionDate = alertMetricRetentionDate;
+                        itemsToSave.push(timeToAcknowledgeMetric);
+                    }
+                }
+            }
+            // time to resolve
+            const isAlertResolved = alertStateTimelines.some((timeline) => {
+                var _a;
+                return (_a = timeline.alertState) === null || _a === void 0 ? void 0 : _a.isResolvedState;
+            });
             const resolvedAlertStateTimeline = alertStateTimelines.find((timeline) => {
                 var _a;
                 return (_a = timeline.alertState) === null || _a === void 0 ? void 0 : _a.isResolvedState;
             });
-            if (resolvedAlertStateTimeline) {
-                const timeToResolveMetric = new Metric();
-                timeToResolveMetric.projectId = alert.projectId;
-                timeToResolveMetric.primaryEntityId = alert.id;
-                timeToResolveMetric.primaryEntityType = ServiceType.Alert;
-                timeToResolveMetric.name = AlertMetricType.TimeToResolve;
-                timeToResolveMetric.value = OneUptimeDate.getDifferenceInSeconds((resolvedAlertStateTimeline === null || resolvedAlertStateTimeline === void 0 ? void 0 : resolvedAlertStateTimeline.startsAt) ||
-                    OneUptimeDate.getCurrentDate(), alertStartsAt);
-                timeToResolveMetric.attributes = {
-                    alertId: data.alertId.toString(),
-                    projectId: alert.projectId.toString(),
-                    monitorId: (_t = (_s = alert.monitor) === null || _s === void 0 ? void 0 : _s._id) === null || _t === void 0 ? void 0 : _t.toString(),
-                    monitorName: (_v = (_u = alert.monitor) === null || _u === void 0 ? void 0 : _u.name) === null || _v === void 0 ? void 0 : _v.toString(),
-                    alertSeverityId: (_x = (_w = alert.alertSeverity) === null || _w === void 0 ? void 0 : _w._id) === null || _x === void 0 ? void 0 : _x.toString(),
-                    alertSeverityName: (_z = (_y = alert.alertSeverity) === null || _y === void 0 ? void 0 : _y.name) === null || _z === void 0 ? void 0 : _z.toString(),
-                };
-                timeToResolveMetric.attributeKeys = TelemetryUtil.getAttributeKeys(timeToResolveMetric.attributes);
-                timeToResolveMetric.time =
-                    (resolvedAlertStateTimeline === null || resolvedAlertStateTimeline === void 0 ? void 0 : resolvedAlertStateTimeline.startsAt) ||
-                        alert.createdAt ||
-                        OneUptimeDate.getCurrentDate();
-                timeToResolveMetric.timeUnixNano = OneUptimeDate.toUnixNano(timeToResolveMetric.time);
-                timeToResolveMetric.metricPointType = MetricPointType.Sum;
-                timeToResolveMetric.retentionDate = alertMetricRetentionDate;
-                itemsToSave.push(timeToResolveMetric);
+            if (isAlertResolved && resolvedAlertStateTimeline) {
+                // register the metric type so the catalog stays complete across refreshes.
                 const metricType = new MetricType();
-                metricType.name = timeToResolveMetric.name;
+                metricType.name = AlertMetricType.TimeToResolve;
                 metricType.description = "Time taken to resolve the alert";
                 metricType.unit = "seconds";
-                metricTypesMap[timeToResolveMetric.name] = metricType;
+                metricTypesMap[AlertMetricType.TimeToResolve] = metricType;
+                // write-once: MTTR is fixed once the alert is first resolved.
+                if (!existingMetricNames.has(AlertMetricType.TimeToResolve)) {
+                    const timeToResolveMetric = new Metric();
+                    timeToResolveMetric.projectId = alert.projectId;
+                    timeToResolveMetric.primaryEntityId = alert.id;
+                    timeToResolveMetric.primaryEntityType = ServiceType.Alert;
+                    timeToResolveMetric.name = AlertMetricType.TimeToResolve;
+                    timeToResolveMetric.value = OneUptimeDate.getDifferenceInSeconds((resolvedAlertStateTimeline === null || resolvedAlertStateTimeline === void 0 ? void 0 : resolvedAlertStateTimeline.startsAt) ||
+                        OneUptimeDate.getCurrentDate(), alertStartsAt);
+                    timeToResolveMetric.attributes = {
+                        alertId: data.alertId.toString(),
+                        projectId: alert.projectId.toString(),
+                        monitorId: (_v = (_u = alert.monitor) === null || _u === void 0 ? void 0 : _u._id) === null || _v === void 0 ? void 0 : _v.toString(),
+                        monitorName: (_x = (_w = alert.monitor) === null || _w === void 0 ? void 0 : _w.name) === null || _x === void 0 ? void 0 : _x.toString(),
+                        alertSeverityId: (_z = (_y = alert.alertSeverity) === null || _y === void 0 ? void 0 : _y._id) === null || _z === void 0 ? void 0 : _z.toString(),
+                        alertSeverityName: (_1 = (_0 = alert.alertSeverity) === null || _0 === void 0 ? void 0 : _0.name) === null || _1 === void 0 ? void 0 : _1.toString(),
+                    };
+                    timeToResolveMetric.attributeKeys = TelemetryUtil.getAttributeKeys(timeToResolveMetric.attributes);
+                    timeToResolveMetric.time =
+                        (resolvedAlertStateTimeline === null || resolvedAlertStateTimeline === void 0 ? void 0 : resolvedAlertStateTimeline.startsAt) ||
+                            alert.createdAt ||
+                            OneUptimeDate.getCurrentDate();
+                    timeToResolveMetric.timeUnixNano = OneUptimeDate.toUnixNano(timeToResolveMetric.time);
+                    timeToResolveMetric.metricPointType = MetricPointType.Sum;
+                    timeToResolveMetric.retentionDate = alertMetricRetentionDate;
+                    itemsToSave.push(timeToResolveMetric);
+                }
+            }
+            /*
+             * Alert duration — write-once, finalized at resolution.
+             *
+             * Previously recomputed as (latest state transition − start) on every
+             * refresh, which grew for open alerts and required delete + re-insert.
+             * With no deletes we emit a single final duration once the alert reaches
+             * a resolved state: (first resolution − start). An alert that is never
+             * resolved has no duration row (its lifetime is not yet final).
+             */
+            if (isAlertResolved && resolvedAlertStateTimeline) {
+                // register the metric type so the catalog stays complete across refreshes.
+                const metricType = new MetricType();
+                metricType.name = AlertMetricType.AlertDuration;
+                metricType.description = "Duration of the alert";
+                metricType.unit = "seconds";
+                metricTypesMap[AlertMetricType.AlertDuration] = metricType;
+                if (!existingMetricNames.has(AlertMetricType.AlertDuration)) {
+                    const alertEndsAt = resolvedAlertStateTimeline.startsAt ||
+                        OneUptimeDate.getCurrentDate();
+                    const alertDurationMetric = new Metric();
+                    alertDurationMetric.projectId = alert.projectId;
+                    alertDurationMetric.primaryEntityId = alert.id;
+                    alertDurationMetric.primaryEntityType = ServiceType.Alert;
+                    alertDurationMetric.name = AlertMetricType.AlertDuration;
+                    alertDurationMetric.value = OneUptimeDate.getDifferenceInSeconds(alertEndsAt, alertStartsAt);
+                    alertDurationMetric.attributes = {
+                        alertId: data.alertId.toString(),
+                        projectId: alert.projectId.toString(),
+                        monitorId: (_3 = (_2 = alert.monitor) === null || _2 === void 0 ? void 0 : _2._id) === null || _3 === void 0 ? void 0 : _3.toString(),
+                        monitorName: (_5 = (_4 = alert.monitor) === null || _4 === void 0 ? void 0 : _4.name) === null || _5 === void 0 ? void 0 : _5.toString(),
+                        alertSeverityId: (_7 = (_6 = alert.alertSeverity) === null || _6 === void 0 ? void 0 : _6._id) === null || _7 === void 0 ? void 0 : _7.toString(),
+                        alertSeverityName: (_9 = (_8 = alert.alertSeverity) === null || _8 === void 0 ? void 0 : _8.name) === null || _9 === void 0 ? void 0 : _9.toString(),
+                    };
+                    alertDurationMetric.attributeKeys = TelemetryUtil.getAttributeKeys(alertDurationMetric.attributes);
+                    alertDurationMetric.time = alertEndsAt;
+                    alertDurationMetric.timeUnixNano = OneUptimeDate.toUnixNano(alertDurationMetric.time);
+                    alertDurationMetric.metricPointType = MetricPointType.Sum;
+                    alertDurationMetric.retentionDate = alertMetricRetentionDate;
+                    itemsToSave.push(alertDurationMetric);
+                }
+            }
+            // write-once: a refresh that finds nothing new inserts nothing.
+            if (itemsToSave.length > 0) {
+                await MetricService.createMany({
+                    items: itemsToSave,
+                    props: {
+                        isRoot: true,
+                    },
+                });
             }
+            TelemetryUtil.indexMetricNameServiceNameMap({
+                metricNameServiceNameMap: metricTypesMap,
+                projectId: alert.projectId,
+            }).catch((err) => {
+                var _a, _b;
+                logger.error(err, {
+                    projectId: (_a = alert.projectId) === null || _a === void 0 ? void 0 : _a.toString(),
+                    alertId: (_b = alert.id) === null || _b === void 0 ? void 0 : _b.toString(),
+                });
+            });
         }
-        // alert duration
-        const alertDurationMetric = new Metric();
-        const lastAlertStateTimeline = alertStateTimelines[alertStateTimelines.length - 1];
-        if (lastAlertStateTimeline) {
-            const alertEndsAt = lastAlertStateTimeline.startsAt || OneUptimeDate.getCurrentDate();
-            alertDurationMetric.projectId = alert.projectId;
-            alertDurationMetric.primaryEntityId = alert.id;
-            alertDurationMetric.primaryEntityType = ServiceType.Alert;
-            alertDurationMetric.name = AlertMetricType.AlertDuration;
-            alertDurationMetric.value = OneUptimeDate.getDifferenceInSeconds(alertEndsAt, alertStartsAt);
-            alertDurationMetric.attributes = {
-                alertId: data.alertId.toString(),
-                projectId: alert.projectId.toString(),
-                monitorId: (_1 = (_0 = alert.monitor) === null || _0 === void 0 ? void 0 : _0._id) === null || _1 === void 0 ? void 0 : _1.toString(),
-                monitorName: (_3 = (_2 = alert.monitor) === null || _2 === void 0 ? void 0 : _2.name) === null || _3 === void 0 ? void 0 : _3.toString(),
-                alertSeverityId: (_5 = (_4 = alert.alertSeverity) === null || _4 === void 0 ? void 0 : _4._id) === null || _5 === void 0 ? void 0 : _5.toString(),
-                alertSeverityName: (_7 = (_6 = alert.alertSeverity) === null || _6 === void 0 ? void 0 : _6.name) === null || _7 === void 0 ? void 0 : _7.toString(),
-            };
-            alertDurationMetric.attributeKeys = TelemetryUtil.getAttributeKeys(alertDurationMetric.attributes);
-            alertDurationMetric.time =
-                (lastAlertStateTimeline === null || lastAlertStateTimeline === void 0 ? void 0 : lastAlertStateTimeline.startsAt) ||
-                    alert.createdAt ||
-                    OneUptimeDate.getCurrentDate();
-            alertDurationMetric.timeUnixNano = OneUptimeDate.toUnixNano(alertDurationMetric.time);
-            alertDurationMetric.metricPointType = MetricPointType.Sum;
-            alertDurationMetric.retentionDate = alertMetricRetentionDate;
-            itemsToSave.push(alertDurationMetric);
-            const metricType = new MetricType();
-            metricType.name = alertDurationMetric.name;
-            metricType.description = "Duration of the alert";
-            metricType.unit = "seconds";
-            metricTypesMap[alertDurationMetric.name] = metricType;
+        finally {
+            if (metricRefreshMutex) {
+                try {
+                    await Semaphore.release(metricRefreshMutex);
+                }
+                catch (err) {
+                    logger.error(err, {
+                        projectId: (_10 = alert.projectId) === null || _10 === void 0 ? void 0 : _10.toString(),
+                        alertId: (_11 = alert.id) === null || _11 === void 0 ? void 0 : _11.toString(),
+                    });
+                }
+            }
         }
-        await MetricService.createMany({
-            items: itemsToSave,
-            props: {
-                isRoot: true,
-            },
-        });
-        TelemetryUtil.indexMetricNameServiceNameMap({
-            metricNameServiceNameMap: metricTypesMap,
-            projectId: alert.projectId,
-        }).catch((err) => {
-            var _a, _b;
-            logger.error(err, {
-                projectId: (_a = alert.projectId) === null || _a === void 0 ? void 0 : _a.toString(),
-                alertId: (_b = alert.id) === null || _b === void 0 ? void 0 : _b.toString(),
-            });
-        });
     }
     async isAlertResolved(data) {
         const alert = await this.findOneBy({