@oneuptime/common 11.3.6 → 11.3.8

package/build/dist/Server/Services/IncidentService.js

@@ -49,7 +49,7 @@ import ServiceType from "../../Types/Telemetry/ServiceType";
 import OneUptimeDate from "../../Types/Date";
 import TelemetryUtil from "../Utils/Telemetry/Telemetry";
 import logger from "../Utils/Logger";
-import NotEqual from "../../Types/BaseDatabase/NotEqual";
+import Semaphore from "../Infrastructure/Semaphore";
 import IncidentFeedService from "./IncidentFeedService";
 import IncidentSlaService from "./IncidentSlaService";
 import { setIsPublicForMarkdownImages } from "../Utils/InlineImageAccessTokenSync";
@@ -1941,7 +1941,7 @@ ${incidentSeverity.name}
         return Service.DEFAULT_METRIC_RETENTION_DAYS;
     }
     async refreshIncidentMetrics(data) {
-        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
+        var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q, _r, _s, _t, _u, _v;
         const incident = await this.findOneById({
             id: data.incidentId,
             select: {
@@ -2061,254 +2061,340 @@ ${incidentSeverity.name}
         });
         const firstIncidentStateTimeline = incidentStateTimelines[0];
         /*
-         * Delete the existing metrics for this incident so the time-varying
-         * ones (TimeToAcknowledge / TimeToResolve / IncidentDuration /
-         * TimeInState) get rewritten with the latest state-timeline values
-         * on this refresh. IncidentCount is excluded from the delete: it is
-         * a constant `value = 1` keyed by `primaryEntityId + bucketTime` that
-         * never changes. Re-emitting it across refreshes inflated the
-         * 1-minute aggregating materialized view (`MetricItemAggMV1m_mv`),
-         * because the MV trigger only fires on inserts — ALTER DELETE
-         * mutations don't roll back the previously-accumulated
-         * `sumState` / `countState`. That's why the Incident Dashboard
-         * sum-of-IncidentCount widget read ~33% higher than the actual
-         * unique-incident count.
+         * Serialize concurrent refreshes for this incident across pods. The
+         * write-once guard below is read-then-insert (findBy existing, then a
+         * conditional createMany). Both call sites fire refresh fire-and-forget
+         * after releasing the per-incident state-timeline mutex, so two
+         * close-together transitions (e.g. auto-acknowledge then auto-resolve)
+         * could interleave: both read a snapshot missing a metric and both
+         * insert it. The Metric table is a plain MergeTree with no row collapse,
+         * so those duplicates would be permanent and re-inflate the very Sum/Avg
+         * widgets this change fixes. A best-effort distributed lock keyed on the
+         * incident makes the read+insert atomic; if the lock can't be taken we
+         * still proceed (the existence check alone covers the common sequential
+         * case).
          */
-        await MetricService.deleteBy({
-            query: {
-                projectId: incident.projectId,
-                primaryEntityId: data.incidentId,
-                name: new NotEqual(IncidentMetricType.IncidentCount),
-            },
-            props: {
-                isRoot: true,
-            },
-        });
-        const itemsToSave = [];
-        const metricRetentionDays = await this.getMetricRetentionDays();
-        const incidentMetricRetentionDate = OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), metricRetentionDays);
-        // now we need to create new metrics for this incident - TimeToAcknowledge, TimeToResolve, IncidentCount, IncidentDuration
-        const incidentStartsAt = (firstIncidentStateTimeline === null || firstIncidentStateTimeline === void 0 ? void 0 : firstIncidentStateTimeline.startsAt) ||
-            incident.declaredAt ||
-            incident.createdAt ||
-            OneUptimeDate.getCurrentDate();
-        const metricTypesMap = {};
-        /*
-         * common attributes shared by all incident metrics
-         * All values must be strings for ClickHouse Map(String, String) storage.
-         * Arrays are joined as comma-separated strings.
-         */
-        const baseMetricAttributes = {
-            incidentId: data.incidentId.toString(),
-            projectId: incident.projectId.toString(),
-            monitorIds: (((_a = incident.monitors) === null || _a === void 0 ? void 0 : _a.map((monitor) => {
-                var _a;
-                return (_a = monitor._id) === null || _a === void 0 ? void 0 : _a.toString();
-            }).filter(Boolean)) || []).join(", "),
-            monitorNames: (((_b = incident.monitors) === null || _b === void 0 ? void 0 : _b.map((monitor) => {
+        let metricRefreshMutex = null;
+        try {
+            metricRefreshMutex = await Semaphore.lock({
+                key: data.incidentId.toString(),
+                namespace: "IncidentService.refreshIncidentMetrics",
+                lockTimeout: 30000,
+            });
+        }
+        catch (err) {
+            logger.error(err, {
+                projectId: (_a = incident.projectId) === null || _a === void 0 ? void 0 : _a.toString(),
+                incidentId: (_b = incident.id) === null || _b === void 0 ? void 0 : _b.toString(),
+            });
+        }
+        try {
+            /*
+             * Write-once metrics — no deletes.
+             *
+             * The raw Metric table (`MetricItemV3`) feeds an AggregatingMergeTree
+             * materialized view (`MetricItemAggMV1m_mv`) that accumulates
+             * `sumState`/`countState` per (projectId, name, primaryEntityId,
+             * minute). The MV trigger fires only on INSERT — an `ALTER ... DELETE`
+             * mutation on the source rolls back nothing in the MV. So the old
+             * "delete every metric for this incident, then re-insert" refresh both
+             * (a) inflated those rollups (the Sum-of-IncidentCount widget read high)
+             * and (b) issued one heavy `ALTER ... DELETE` mutation per state
+             * transition per incident — a mutation storm.
+             *
+             * Every incident metric has a single eventually-final value:
+             * IncidentCount is constant `value = 1`; MTTA/MTTR/duration/time-in-state
+             * are each fixed once the relevant state transition has happened. So we
+             * load what we've already emitted for this incident and insert each
+             * metric exactly once, when it becomes final. Repeat refreshes that find
+             * nothing new are pure no-ops (no insert, no mutation).
+             */
+            const existingMetrics = await MetricService.findBy({
+                query: {
+                    projectId: incident.projectId,
+                    primaryEntityId: data.incidentId,
+                },
+                select: {
+                    name: true,
+                    time: true,
+                    attributes: true,
+                },
+                skip: 0,
+                limit: LIMIT_PER_PROJECT,
+                props: {
+                    isRoot: true,
+                },
+            });
+            const existingMetricNames = new Set(existingMetrics
+                .map((metric) => {
+                return metric.name;
+            })
+                .filter(Boolean));
+            /*
+             * Identity of a TimeInState row: the state plus the instant that state
+             * began (a state can be entered more than once over a re-opened
+             * incident). Second granularity is robust against DateTime64 round-trip
+             * jitter and two distinct entries into the same state within one second
+             * never happen.
+             */
+            const buildTimeInStateKey = (stateId, startsAt) => {
+                return `${stateId || ""}|${startsAt ? OneUptimeDate.toUnixTimestamp(startsAt) : ""}`;
+            };
+            const existingTimeInStateKeys = new Set(existingMetrics
+                .filter((metric) => {
+                return metric.name === IncidentMetricType.TimeInState;
+            })
+                .map((metric) => {
                 var _a;
-                return (_a = monitor.name) === null || _a === void 0 ? void 0 : _a.toString();
-            }).filter(Boolean)) || []).join(", "),
-            incidentSeverityId: (_d = (_c = incident.incidentSeverity) === null || _c === void 0 ? void 0 : _c._id) === null || _d === void 0 ? void 0 : _d.toString(),
-            incidentSeverityName: (_f = (_e = incident.incidentSeverity) === null || _e === void 0 ? void 0 : _e.name) === null || _f === void 0 ? void 0 : _f.toString(),
-            ownerUserIds: ownerUserIds.join(", "),
-            ownerUserNames: ownerUserNames.join(", "),
-            ownerTeamIds: ownerTeamIds.join(", "),
-            ownerTeamNames: ownerTeamNames.join(", "),
-        };
-        /*
-         * Only emit IncidentCount on the very first refresh (i.e. when no
-         * existing IncidentCount row is present for this primaryEntityId). See
-         * the delete comment above — emitting it on every refresh would
-         * accumulate phantom `sumState` entries in the MV that ALTER
-         * DELETE can't undo. By keeping the original row alive and never
-         * re-emitting, the dashboard Sum stays equal to the true count of
-         * distinct incidents.
-         */
-        const incidentCountMetricExists = await MetricService.existsBy({
-            query: {
-                projectId: incident.projectId,
-                primaryEntityId: data.incidentId,
-                name: IncidentMetricType.IncidentCount,
-            },
-            props: {
-                isRoot: true,
-            },
-        });
-        if (!incidentCountMetricExists) {
-            const incidentCountMetric = new Metric();
-            incidentCountMetric.projectId = incident.projectId;
-            incidentCountMetric.primaryEntityId = incident.id;
-            incidentCountMetric.primaryEntityType = ServiceType.Incident;
-            incidentCountMetric.name = IncidentMetricType.IncidentCount;
-            incidentCountMetric.value = 1;
-            incidentCountMetric.attributes = Object.assign({}, baseMetricAttributes);
-            incidentCountMetric.attributeKeys = TelemetryUtil.getAttributeKeys(incidentCountMetric.attributes);
-            incidentCountMetric.time = incidentStartsAt;
-            incidentCountMetric.timeUnixNano = OneUptimeDate.toUnixNano(incidentCountMetric.time);
-            incidentCountMetric.metricPointType = MetricPointType.Sum;
-            incidentCountMetric.retentionDate = incidentMetricRetentionDate;
-            itemsToSave.push(incidentCountMetric);
-        }
-        // Always register the metric type so it shows up in the type catalog.
-        const metricType = new MetricType();
-        metricType.name = IncidentMetricType.IncidentCount;
-        metricType.description = "Number of incidents created";
-        metricType.unit = "";
-        metricType.services = [];
-        metricTypesMap[IncidentMetricType.IncidentCount] = metricType;
-        // is the incident acknowledged?
-        const isIncidentAcknowledged = incidentStateTimelines.some((timeline) => {
-            var _a;
-            return (_a = timeline.incidentState) === null || _a === void 0 ? void 0 : _a.isAcknowledgedState;
-        });
-        if (isIncidentAcknowledged) {
-            const ackIncidentStateTimeline = incidentStateTimelines.find((timeline) => {
+                return buildTimeInStateKey((_a = metric.attributes) === null || _a === void 0 ? void 0 : _a["incidentStateId"], metric.time);
+            }));
+            const itemsToSave = [];
+            const metricRetentionDays = await this.getMetricRetentionDays();
+            const incidentMetricRetentionDate = OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), metricRetentionDays);
+            // now we need to create new metrics for this incident - TimeToAcknowledge, TimeToResolve, IncidentCount, IncidentDuration
+            const incidentStartsAt = (firstIncidentStateTimeline === null || firstIncidentStateTimeline === void 0 ? void 0 : firstIncidentStateTimeline.startsAt) ||
+                incident.declaredAt ||
+                incident.createdAt ||
+                OneUptimeDate.getCurrentDate();
+            const metricTypesMap = {};
+            /*
+             * common attributes shared by all incident metrics
+             * All values must be strings for ClickHouse Map(String, String) storage.
+             * Arrays are joined as comma-separated strings.
+             */
+            const baseMetricAttributes = {
+                incidentId: data.incidentId.toString(),
+                projectId: incident.projectId.toString(),
+                monitorIds: (((_c = incident.monitors) === null || _c === void 0 ? void 0 : _c.map((monitor) => {
+                    var _a;
+                    return (_a = monitor._id) === null || _a === void 0 ? void 0 : _a.toString();
+                }).filter(Boolean)) || []).join(", "),
+                monitorNames: (((_d = incident.monitors) === null || _d === void 0 ? void 0 : _d.map((monitor) => {
+                    var _a;
+                    return (_a = monitor.name) === null || _a === void 0 ? void 0 : _a.toString();
+                }).filter(Boolean)) || []).join(", "),
+                incidentSeverityId: (_f = (_e = incident.incidentSeverity) === null || _e === void 0 ? void 0 : _e._id) === null || _f === void 0 ? void 0 : _f.toString(),
+                incidentSeverityName: (_h = (_g = incident.incidentSeverity) === null || _g === void 0 ? void 0 : _g.name) === null || _h === void 0 ? void 0 : _h.toString(),
+                ownerUserIds: ownerUserIds.join(", "),
+                ownerUserNames: ownerUserNames.join(", "),
+                ownerTeamIds: ownerTeamIds.join(", "),
+                ownerTeamNames: ownerTeamNames.join(", "),
+            };
+            /*
+             * Only emit IncidentCount on the very first refresh (i.e. when no
+             * existing IncidentCount row is present for this primaryEntityId). See
+             * the write-once note above — emitting it on every refresh would
+             * accumulate phantom `sumState` entries in the MV. By keeping the
+             * original row alive and never re-emitting, the dashboard Sum stays
+             * equal to the true count of distinct incidents.
+             */
+            if (!existingMetricNames.has(IncidentMetricType.IncidentCount)) {
+                const incidentCountMetric = new Metric();
+                incidentCountMetric.projectId = incident.projectId;
+                incidentCountMetric.primaryEntityId = incident.id;
+                incidentCountMetric.primaryEntityType = ServiceType.Incident;
+                incidentCountMetric.name = IncidentMetricType.IncidentCount;
+                incidentCountMetric.value = 1;
+                incidentCountMetric.attributes = Object.assign({}, baseMetricAttributes);
+                incidentCountMetric.attributeKeys = TelemetryUtil.getAttributeKeys(incidentCountMetric.attributes);
+                incidentCountMetric.time = incidentStartsAt;
+                incidentCountMetric.timeUnixNano = OneUptimeDate.toUnixNano(incidentCountMetric.time);
+                incidentCountMetric.metricPointType = MetricPointType.Sum;
+                incidentCountMetric.retentionDate = incidentMetricRetentionDate;
+                itemsToSave.push(incidentCountMetric);
+            }
+            // Always register the metric type so it shows up in the type catalog.
+            const metricType = new MetricType();
+            metricType.name = IncidentMetricType.IncidentCount;
+            metricType.description = "Number of incidents created";
+            metricType.unit = "";
+            metricType.services = [];
+            metricTypesMap[IncidentMetricType.IncidentCount] = metricType;
+            // is the incident acknowledged?
+            const isIncidentAcknowledged = incidentStateTimelines.some((timeline) => {
                 var _a;
                 return (_a = timeline.incidentState) === null || _a === void 0 ? void 0 : _a.isAcknowledgedState;
             });
-            if (ackIncidentStateTimeline) {
-                const timeToAcknowledgeMetric = new Metric();
-                timeToAcknowledgeMetric.projectId = incident.projectId;
-                timeToAcknowledgeMetric.primaryEntityId = incident.id;
-                timeToAcknowledgeMetric.primaryEntityType = ServiceType.Incident;
-                timeToAcknowledgeMetric.name = IncidentMetricType.TimeToAcknowledge;
-                timeToAcknowledgeMetric.value = OneUptimeDate.getDifferenceInSeconds((ackIncidentStateTimeline === null || ackIncidentStateTimeline === void 0 ? void 0 : ackIncidentStateTimeline.startsAt) || OneUptimeDate.getCurrentDate(), incidentStartsAt);
-                timeToAcknowledgeMetric.attributes = Object.assign({}, baseMetricAttributes);
-                timeToAcknowledgeMetric.attributeKeys = TelemetryUtil.getAttributeKeys(timeToAcknowledgeMetric.attributes);
-                timeToAcknowledgeMetric.time =
-                    (ackIncidentStateTimeline === null || ackIncidentStateTimeline === void 0 ? void 0 : ackIncidentStateTimeline.startsAt) ||
-                        incident.declaredAt ||
-                        incident.createdAt ||
-                        OneUptimeDate.getCurrentDate();
-                timeToAcknowledgeMetric.timeUnixNano = OneUptimeDate.toUnixNano(timeToAcknowledgeMetric.time);
-                timeToAcknowledgeMetric.metricPointType = MetricPointType.Sum;
-                timeToAcknowledgeMetric.retentionDate = incidentMetricRetentionDate;
-                itemsToSave.push(timeToAcknowledgeMetric);
-                // add metric type for this to map.
-                const metricType = new MetricType();
-                metricType.name = timeToAcknowledgeMetric.name;
-                metricType.description = "Time taken to acknowledge the incident";
-                metricType.unit = "seconds";
-                // add to map.
-                metricTypesMap[timeToAcknowledgeMetric.name] = metricType;
+            if (isIncidentAcknowledged) {
+                const ackIncidentStateTimeline = incidentStateTimelines.find((timeline) => {
+                    var _a;
+                    return (_a = timeline.incidentState) === null || _a === void 0 ? void 0 : _a.isAcknowledgedState;
+                });
+                if (ackIncidentStateTimeline) {
+                    // register the metric type so the catalog stays complete across refreshes.
+                    const metricType = new MetricType();
+                    metricType.name = IncidentMetricType.TimeToAcknowledge;
+                    metricType.description = "Time taken to acknowledge the incident";
+                    metricType.unit = "seconds";
+                    metricTypesMap[IncidentMetricType.TimeToAcknowledge] = metricType;
+                    // write-once: MTTA is fixed once the incident is first acknowledged.
+                    if (!existingMetricNames.has(IncidentMetricType.TimeToAcknowledge)) {
+                        const timeToAcknowledgeMetric = new Metric();
+                        timeToAcknowledgeMetric.projectId = incident.projectId;
+                        timeToAcknowledgeMetric.primaryEntityId = incident.id;
+                        timeToAcknowledgeMetric.primaryEntityType = ServiceType.Incident;
+                        timeToAcknowledgeMetric.name = IncidentMetricType.TimeToAcknowledge;
+                        timeToAcknowledgeMetric.value =
+                            OneUptimeDate.getDifferenceInSeconds((ackIncidentStateTimeline === null || ackIncidentStateTimeline === void 0 ? void 0 : ackIncidentStateTimeline.startsAt) ||
+                                OneUptimeDate.getCurrentDate(), incidentStartsAt);
+                        timeToAcknowledgeMetric.attributes = Object.assign({}, baseMetricAttributes);
+                        timeToAcknowledgeMetric.attributeKeys =
+                            TelemetryUtil.getAttributeKeys(timeToAcknowledgeMetric.attributes);
+                        timeToAcknowledgeMetric.time =
+                            (ackIncidentStateTimeline === null || ackIncidentStateTimeline === void 0 ? void 0 : ackIncidentStateTimeline.startsAt) ||
+                                incident.declaredAt ||
+                                incident.createdAt ||
+                                OneUptimeDate.getCurrentDate();
+                        timeToAcknowledgeMetric.timeUnixNano = OneUptimeDate.toUnixNano(timeToAcknowledgeMetric.time);
+                        timeToAcknowledgeMetric.metricPointType = MetricPointType.Sum;
+                        timeToAcknowledgeMetric.retentionDate = incidentMetricRetentionDate;
+                        itemsToSave.push(timeToAcknowledgeMetric);
+                    }
+                }
             }
-        }
-        // time to resolve
-        const isIncidentResolved = incidentStateTimelines.some((timeline) => {
-            var _a;
-            return (_a = timeline.incidentState) === null || _a === void 0 ? void 0 : _a.isResolvedState;
-        });
-        if (isIncidentResolved) {
+            // time to resolve
+            const isIncidentResolved = incidentStateTimelines.some((timeline) => {
+                var _a;
+                return (_a = timeline.incidentState) === null || _a === void 0 ? void 0 : _a.isResolvedState;
+            });
             const resolvedIncidentStateTimeline = incidentStateTimelines.find((timeline) => {
                 var _a;
                 return (_a = timeline.incidentState) === null || _a === void 0 ? void 0 : _a.isResolvedState;
             });
-            if (resolvedIncidentStateTimeline) {
-                const timeToResolveMetric = new Metric();
-                timeToResolveMetric.projectId = incident.projectId;
-                timeToResolveMetric.primaryEntityId = incident.id;
-                timeToResolveMetric.primaryEntityType = ServiceType.Incident;
-                timeToResolveMetric.name = IncidentMetricType.TimeToResolve;
-                timeToResolveMetric.value = OneUptimeDate.getDifferenceInSeconds((resolvedIncidentStateTimeline === null || resolvedIncidentStateTimeline === void 0 ? void 0 : resolvedIncidentStateTimeline.startsAt) ||
-                    OneUptimeDate.getCurrentDate(), incidentStartsAt);
-                timeToResolveMetric.attributes = Object.assign({}, baseMetricAttributes);
-                timeToResolveMetric.attributeKeys = TelemetryUtil.getAttributeKeys(timeToResolveMetric.attributes);
-                timeToResolveMetric.time =
-                    (resolvedIncidentStateTimeline === null || resolvedIncidentStateTimeline === void 0 ? void 0 : resolvedIncidentStateTimeline.startsAt) ||
-                        incident.declaredAt ||
-                        incident.createdAt ||
-                        OneUptimeDate.getCurrentDate();
-                timeToResolveMetric.timeUnixNano = OneUptimeDate.toUnixNano(timeToResolveMetric.time);
-                timeToResolveMetric.metricPointType = MetricPointType.Sum;
-                timeToResolveMetric.retentionDate = incidentMetricRetentionDate;
-                itemsToSave.push(timeToResolveMetric);
-                // add metric type for this to map.
+            if (isIncidentResolved && resolvedIncidentStateTimeline) {
+                // register the metric type so the catalog stays complete across refreshes.
                 const metricType = new MetricType();
-                metricType.name = timeToResolveMetric.name;
+                metricType.name = IncidentMetricType.TimeToResolve;
                 metricType.description = "Time taken to resolve the incident";
                 metricType.unit = "seconds";
-                // add to map.
-                metricTypesMap[timeToResolveMetric.name] = metricType;
-            }
-        }
-        // incident duration
-        const incidentDurationMetric = new Metric();
-        const lastIncidentStateTimeline = incidentStateTimelines[incidentStateTimelines.length - 1];
-        if (lastIncidentStateTimeline) {
-            const incidentEndsAt = lastIncidentStateTimeline.startsAt || OneUptimeDate.getCurrentDate();
-            // save metric.
-            incidentDurationMetric.projectId = incident.projectId;
-            incidentDurationMetric.primaryEntityId = incident.id;
-            incidentDurationMetric.primaryEntityType = ServiceType.Incident;
-            incidentDurationMetric.name = IncidentMetricType.IncidentDuration;
-            incidentDurationMetric.value = OneUptimeDate.getDifferenceInSeconds(incidentEndsAt, incidentStartsAt);
-            incidentDurationMetric.attributes = Object.assign({}, baseMetricAttributes);
-            incidentDurationMetric.attributeKeys = TelemetryUtil.getAttributeKeys(incidentDurationMetric.attributes);
-            incidentDurationMetric.time =
-                (lastIncidentStateTimeline === null || lastIncidentStateTimeline === void 0 ? void 0 : lastIncidentStateTimeline.startsAt) ||
-                    incident.declaredAt ||
-                    incident.createdAt ||
-                    OneUptimeDate.getCurrentDate();
-            incidentDurationMetric.timeUnixNano = OneUptimeDate.toUnixNano(incidentDurationMetric.time);
-            incidentDurationMetric.metricPointType = MetricPointType.Sum;
-            incidentDurationMetric.retentionDate = incidentMetricRetentionDate;
-            itemsToSave.push(incidentDurationMetric);
-            // add metric type for this to map.
-            const metricType = new MetricType();
-            metricType.name = incidentDurationMetric.name;
-            metricType.description = "Duration of the incident";
-            metricType.unit = "seconds";
-            // add to map.
-            metricTypesMap[incidentDurationMetric.name] = metricType;
-        }
-        // time-in-state metrics — emit one metric per state transition that has a completed duration
-        for (const timeline of incidentStateTimelines) {
-            if (!timeline.startsAt || !timeline.endsAt) {
-                continue;
-            }
-            const stateName = ((_h = (_g = timeline.incidentState) === null || _g === void 0 ? void 0 : _g.name) === null || _h === void 0 ? void 0 : _h.toString()) || "Unknown";
-            const timeInStateMetric = new Metric();
-            timeInStateMetric.projectId = incident.projectId;
-            timeInStateMetric.primaryEntityId = incident.id;
-            timeInStateMetric.primaryEntityType = ServiceType.Incident;
-            timeInStateMetric.name = IncidentMetricType.TimeInState;
-            timeInStateMetric.value = OneUptimeDate.getDifferenceInSeconds(timeline.endsAt, timeline.startsAt);
-            timeInStateMetric.attributes = Object.assign(Object.assign({}, baseMetricAttributes), { incidentStateName: stateName, incidentStateId: (_j = timeline.incidentStateId) === null || _j === void 0 ? void 0 : _j.toString(), isCreatedState: ((_l = (_k = timeline.incidentState) === null || _k === void 0 ? void 0 : _k.isCreatedState) === null || _l === void 0 ? void 0 : _l.toString()) || "false", isAcknowledgedState: ((_o = (_m = timeline.incidentState) === null || _m === void 0 ? void 0 : _m.isAcknowledgedState) === null || _o === void 0 ? void 0 : _o.toString()) || "false", isResolvedState: ((_q = (_p = timeline.incidentState) === null || _p === void 0 ? void 0 : _p.isResolvedState) === null || _q === void 0 ? void 0 : _q.toString()) || "false" });
-            timeInStateMetric.attributeKeys = TelemetryUtil.getAttributeKeys(timeInStateMetric.attributes);
-            timeInStateMetric.time = timeline.startsAt;
-            timeInStateMetric.timeUnixNano = OneUptimeDate.toUnixNano(timeInStateMetric.time);
-            timeInStateMetric.metricPointType = MetricPointType.Sum;
-            timeInStateMetric.retentionDate = incidentMetricRetentionDate;
-            itemsToSave.push(timeInStateMetric);
-        }
-        // add metric type for time-in-state to map (only once)
-        if (incidentStateTimelines.some((t) => {
-            return t.startsAt && t.endsAt;
-        })) {
-            const timeInStateMetricType = new MetricType();
-            timeInStateMetricType.name = IncidentMetricType.TimeInState;
-            timeInStateMetricType.description =
-                "Time spent in each incident state (e.g. Created, Investigating, Acknowledged)";
-            timeInStateMetricType.unit = "seconds";
-            metricTypesMap[timeInStateMetricType.name] = timeInStateMetricType;
-        }
-        await MetricService.createMany({
-            items: itemsToSave,
-            props: {
-                isRoot: true,
-            },
-        });
-        TelemetryUtil.indexMetricNameServiceNameMap({
-            metricNameServiceNameMap: metricTypesMap,
-            projectId: incident.projectId,
-        }).catch((err) => {
-            var _a, _b;
-            logger.error(err, {
-                projectId: (_a = incident.projectId) === null || _a === void 0 ? void 0 : _a.toString(),
-                incidentId: (_b = incident.id) === null || _b === void 0 ? void 0 : _b.toString(),
+                metricTypesMap[IncidentMetricType.TimeToResolve] = metricType;
+                // write-once: MTTR is fixed once the incident is first resolved.
+                if (!existingMetricNames.has(IncidentMetricType.TimeToResolve)) {
+                    const timeToResolveMetric = new Metric();
+                    timeToResolveMetric.projectId = incident.projectId;
+                    timeToResolveMetric.primaryEntityId = incident.id;
+                    timeToResolveMetric.primaryEntityType = ServiceType.Incident;
+                    timeToResolveMetric.name = IncidentMetricType.TimeToResolve;
+                    timeToResolveMetric.value = OneUptimeDate.getDifferenceInSeconds((resolvedIncidentStateTimeline === null || resolvedIncidentStateTimeline === void 0 ? void 0 : resolvedIncidentStateTimeline.startsAt) ||
+                        OneUptimeDate.getCurrentDate(), incidentStartsAt);
+                    timeToResolveMetric.attributes = Object.assign({}, baseMetricAttributes);
+                    timeToResolveMetric.attributeKeys = TelemetryUtil.getAttributeKeys(timeToResolveMetric.attributes);
+                    timeToResolveMetric.time =
+                        (resolvedIncidentStateTimeline === null || resolvedIncidentStateTimeline === void 0 ? void 0 : resolvedIncidentStateTimeline.startsAt) ||
+                            incident.declaredAt ||
+                            incident.createdAt ||
+                            OneUptimeDate.getCurrentDate();
+                    timeToResolveMetric.timeUnixNano = OneUptimeDate.toUnixNano(timeToResolveMetric.time);
+                    timeToResolveMetric.metricPointType = MetricPointType.Sum;
+                    timeToResolveMetric.retentionDate = incidentMetricRetentionDate;
+                    itemsToSave.push(timeToResolveMetric);
+                }
+            }
+            /*
+             * Incident duration — write-once, finalized at resolution.
+             *
+             * The previous implementation recomputed duration as
+             * (latest state transition − start) on every refresh, so an open
+             * incident's duration grew and had to be deleted + re-inserted each
+             * time. With no deletes we emit a single final duration once the
+             * incident reaches a resolved state: (first resolution − start). An
+             * incident that is never resolved has no duration row (its lifetime is
+             * not yet final), which keeps the "Avg Duration" widget meaningful.
+             */
+            if (isIncidentResolved && resolvedIncidentStateTimeline) {
+                // register the metric type so the catalog stays complete across refreshes.
+                const metricType = new MetricType();
+                metricType.name = IncidentMetricType.IncidentDuration;
+                metricType.description = "Duration of the incident";
+                metricType.unit = "seconds";
+                metricTypesMap[IncidentMetricType.IncidentDuration] = metricType;
+                if (!existingMetricNames.has(IncidentMetricType.IncidentDuration)) {
+                    const incidentEndsAt = resolvedIncidentStateTimeline.startsAt ||
+                        OneUptimeDate.getCurrentDate();
+                    const incidentDurationMetric = new Metric();
+                    incidentDurationMetric.projectId = incident.projectId;
+                    incidentDurationMetric.primaryEntityId = incident.id;
+                    incidentDurationMetric.primaryEntityType = ServiceType.Incident;
+                    incidentDurationMetric.name = IncidentMetricType.IncidentDuration;
+                    incidentDurationMetric.value = OneUptimeDate.getDifferenceInSeconds(incidentEndsAt, incidentStartsAt);
+                    incidentDurationMetric.attributes = Object.assign({}, baseMetricAttributes);
+                    incidentDurationMetric.attributeKeys = TelemetryUtil.getAttributeKeys(incidentDurationMetric.attributes);
+                    incidentDurationMetric.time = incidentEndsAt;
+                    incidentDurationMetric.timeUnixNano = OneUptimeDate.toUnixNano(incidentDurationMetric.time);
+                    incidentDurationMetric.metricPointType = MetricPointType.Sum;
+                    incidentDurationMetric.retentionDate = incidentMetricRetentionDate;
+                    itemsToSave.push(incidentDurationMetric);
+                }
+            }
+            // time-in-state metrics — emit one metric per state transition that has a completed duration
+            for (const timeline of incidentStateTimelines) {
+                if (!timeline.startsAt || !timeline.endsAt) {
+                    continue;
+                }
+                // write-once: each completed state transition is emitted exactly once.
+                const timeInStateKey = buildTimeInStateKey((_j = timeline.incidentStateId) === null || _j === void 0 ? void 0 : _j.toString(), timeline.startsAt);
+                if (existingTimeInStateKeys.has(timeInStateKey)) {
+                    continue;
+                }
+                const stateName = ((_l = (_k = timeline.incidentState) === null || _k === void 0 ? void 0 : _k.name) === null || _l === void 0 ? void 0 : _l.toString()) || "Unknown";
+                const timeInStateMetric = new Metric();
+                timeInStateMetric.projectId = incident.projectId;
+                timeInStateMetric.primaryEntityId = incident.id;
+                timeInStateMetric.primaryEntityType = ServiceType.Incident;
+                timeInStateMetric.name = IncidentMetricType.TimeInState;
+                timeInStateMetric.value = OneUptimeDate.getDifferenceInSeconds(timeline.endsAt, timeline.startsAt);
+                timeInStateMetric.attributes = Object.assign(Object.assign({}, baseMetricAttributes), { incidentStateName: stateName, incidentStateId: (_m = timeline.incidentStateId) === null || _m === void 0 ? void 0 : _m.toString(), isCreatedState: ((_p = (_o = timeline.incidentState) === null || _o === void 0 ? void 0 : _o.isCreatedState) === null || _p === void 0 ? void 0 : _p.toString()) || "false", isAcknowledgedState: ((_r = (_q = timeline.incidentState) === null || _q === void 0 ? void 0 : _q.isAcknowledgedState) === null || _r === void 0 ? void 0 : _r.toString()) || "false", isResolvedState: ((_t = (_s = timeline.incidentState) === null || _s === void 0 ? void 0 : _s.isResolvedState) === null || _t === void 0 ? void 0 : _t.toString()) || "false" });
+                timeInStateMetric.attributeKeys = TelemetryUtil.getAttributeKeys(timeInStateMetric.attributes);
+                timeInStateMetric.time = timeline.startsAt;
+                timeInStateMetric.timeUnixNano = OneUptimeDate.toUnixNano(timeInStateMetric.time);
+                timeInStateMetric.metricPointType = MetricPointType.Sum;
+                timeInStateMetric.retentionDate = incidentMetricRetentionDate;
+                itemsToSave.push(timeInStateMetric);
+            }
+            // add metric type for time-in-state to map (only once)
+            if (incidentStateTimelines.some((t) => {
+                return t.startsAt && t.endsAt;
+            })) {
+                const timeInStateMetricType = new MetricType();
+                timeInStateMetricType.name = IncidentMetricType.TimeInState;
+                timeInStateMetricType.description =
+                    "Time spent in each incident state (e.g. Created, Investigating, Acknowledged)";
+                timeInStateMetricType.unit = "seconds";
+                metricTypesMap[timeInStateMetricType.name] = timeInStateMetricType;
+            }
+            // write-once: a refresh that finds nothing new inserts nothing.
+            if (itemsToSave.length > 0) {
+                await MetricService.createMany({
+                    items: itemsToSave,
+                    props: {
+                        isRoot: true,
+                    },
+                });
+            }
+            TelemetryUtil.indexMetricNameServiceNameMap({
+                metricNameServiceNameMap: metricTypesMap,
+                projectId: incident.projectId,
+            }).catch((err) => {
+                var _a, _b;
+                logger.error(err, {
+                    projectId: (_a = incident.projectId) === null || _a === void 0 ? void 0 : _a.toString(),
+                    incidentId: (_b = incident.id) === null || _b === void 0 ? void 0 : _b.toString(),
+                });
             });
-        });
+        }
+        finally {
+            if (metricRefreshMutex) {
+                try {
+                    await Semaphore.release(metricRefreshMutex);
+                }
+                catch (err) {
+                    logger.error(err, {
+                        projectId: (_u = incident.projectId) === null || _u === void 0 ? void 0 : _u.toString(),
+                        incidentId: (_v = incident.id) === null || _v === void 0 ? void 0 : _v.toString(),
+                    });
+                }
+            }
+        }
     }
     async getWorkspaceChannelForIncident(data) {
         const incident = await this.findOneById({