@oneuptime/common 14.0.5 → 14.0.7

package/build/dist/Server/Services/IncidentAlertService.js

@@ -22,6 +22,7 @@ import ProjectService from "./ProjectService";
 import { IsBillingEnabled } from "../EnvironmentConfig";
 import { applyAlertRelatedRecordPrivacyFilter } from "../Utils/Alert/AlertPrivacyFilter";
 import { applyIncidentRelatedRecordPrivacyFilter } from "../Utils/Incident/IncidentPrivacyFilter";
+import AlertStateChangeAuthorization from "../Utils/Alert/AlertStateChangeAuthorization";
 import PostgresErrorTranslator from "../Utils/Database/PostgresErrorTranslator";
 import ProjectScopedReferenceValidator from "../Utils/Database/ProjectScopedReferenceValidator";
 import RelationIdUtil from "../Utils/Database/RelationIdUtil";
@@ -36,7 +37,7 @@ import BadDataException from "../../Types/Exception/BadDataException";
 import ForbiddenException from "../../Types/Exception/ForbiddenException";
 import NotAuthorizedException from "../../Types/Exception/NotAuthorizedException";
 import NotFoundException from "../../Types/Exception/NotFoundException";
-import { INCIDENT_ALERT_ALREADY_LINKED_MESSAGE, INCIDENT_ALERT_IDS_TO_LINK_KEY, MAX_ALERTS_PER_INCIDENT_LINK_ACTION, } from "../../Types/Incident/IncidentAlertLink";
+import { INCIDENT_ACKNOWLEDGE_ALERTS_TO_LINK_KEY, INCIDENT_ALERT_ALREADY_LINKED_MESSAGE, INCIDENT_ALERT_IDS_TO_LINK_KEY, MAX_ALERTS_PER_INCIDENT_LINK_ACTION, } from "../../Types/Incident/IncidentAlertLink";
 import ObjectID from "../../Types/ObjectID";
 /*
  * The decision half of the linked-alert state sync, shared by the cascade
@@ -86,6 +87,25 @@ export function chooseLinkedAlertTargetState(data) {
     }
     return null;
 }
+/*
+ * Alerts acknowledged at once when an incident is declared from many. Each
+ * acknowledgement is a full state change (and waits on its Slack / Microsoft
+ * Teams post), so one at a time could leave the last of 50 alerts escalating
+ * for minutes. Distinct alerts never share a timeline lock.
+ */
+const DECLARED_ALERT_ACKNOWLEDGE_CONCURRENCY = 5;
+/*
+ * The "why" on an alert acknowledged because an incident was declared from
+ * it. It goes to the alert's feed, the alert's Slack / Microsoft Teams
+ * channels and its owners' notifications - places that must not name a
+ * private incident - so a private incident is not named at all.
+ */
+export function getDeclaredAlertAcknowledgementCause(data) {
+    if (data.isIncidentPrivate) {
+        return "Acknowledged because a private incident was declared from this alert.";
+    }
+    return `Acknowledged because ${withNumber("Incident", data.incidentNumber)} was declared from this alert.`;
+}
 /*
  * miscDataProps key on an IncidentAlert create: the link is one of the links
  * written while an incident is being declared from alerts. The incident then
@@ -844,6 +864,413 @@ export class Service extends DatabaseService {
         }
         return result;
     }
+    /*
+     * Reads miscDataProps[INCIDENT_ACKNOWLEDGE_ALERTS_TO_LINK_KEY] on an
+     * incident create: whether to acknowledge the alerts the incident is
+     * declared from once they are linked, and which. Null when not asked to
+     * (absent, null or false). Otherwise the project's Acknowledged alert
+     * state and the alerts that are not acknowledged yet - the only ones that
+     * will be written, and so the only ones the caller must be allowed to
+     * change: an alert that is already acknowledged or resolved is left alone,
+     * so being unable to change it must not refuse the declaration.
+     *
+     * Called from IncidentService's onBeforeCreate after the alert ids were
+     * validated - before the incident number is taken - so an impossible
+     * request is refused instead of the incident being declared while its
+     * alerts keep paging: the project has no Acknowledged alert state, or the
+     * caller may not change the state of every alert that would be
+     * acknowledged (see AlertStateChangeAuthorization; the acknowledgements
+     * themselves are written as root, for exactly these alerts).
+     */
+    async validateAcknowledgeAlertsForNewIncident(data) {
+        var _a, _b;
+        if (data.acknowledgeAlerts === undefined ||
+            data.acknowledgeAlerts === null ||
+            data.acknowledgeAlerts === false) {
+            return null;
+        }
+        if (data.acknowledgeAlerts !== true) {
+            throw new BadDataException(`${INCIDENT_ACKNOWLEDGE_ALERTS_TO_LINK_KEY} must be true or false.`);
+        }
+        if (data.alertIds.length === 0) {
+            throw new BadDataException(`${INCIDENT_ACKNOWLEDGE_ALERTS_TO_LINK_KEY} only applies when the incident is declared from alerts: send the alert ids in ${INCIDENT_ALERT_IDS_TO_LINK_KEY}.`);
+        }
+        if (!data.projectId) {
+            throw new BadDataException("projectId is required to acknowledge alerts.");
+        }
+        const projectId = data.projectId;
+        const acknowledgedState = await AlertStateService.findOneBy({
+            query: {
+                projectId: projectId,
+                isAcknowledgedState: true,
+            },
+            select: {
+                _id: true,
+                order: true,
+            },
+            props: {
+                isRoot: true,
+            },
+        });
+        if (!acknowledgedState ||
+            !acknowledgedState._id ||
+            acknowledgedState.order === undefined ||
+            acknowledgedState.order === null) {
+            throw new BadDataException("This project has no Acknowledged alert state, so the alerts cannot be acknowledged. Declare the incident without acknowledging them, or add an Acknowledged state in the alert settings.");
+        }
+        const acknowledgedOrder = acknowledgedState.order;
+        const alerts = await AlertService.findBy({
+            query: {
+                _id: QueryHelper.any(data.alertIds),
+                projectId: projectId,
+            },
+            select: {
+                _id: true,
+                currentAlertState: {
+                    order: true,
+                },
+            },
+            limit: LIMIT_PER_PROJECT,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        const stateOrderByAlertId = new Map();
+        for (const alert of alerts) {
+            if (alert._id) {
+                stateOrderByAlertId.set(normalizeId(alert._id), (_b = (_a = alert.currentAlertState) === null || _a === void 0 ? void 0 : _a.order) !== null && _b !== void 0 ? _b : undefined);
+            }
+        }
+        // Same rule as acknowledgeAlertsDeclaredWithIncident: by order.
+        const alertIdsToAcknowledge = data.alertIds.filter((alertId) => {
+            const key = normalizeId(alertId);
+            if (!stateOrderByAlertId.has(key)) {
+                return false;
+            }
+            const order = stateOrderByAlertId.get(key);
+            return order === undefined || order < acknowledgedOrder;
+        });
+        if (alertIdsToAcknowledge.length > 0 &&
+            !data.props.isRoot &&
+            !data.props.isMasterAdmin) {
+            try {
+                await AlertStateChangeAuthorization.assertCanChangeStateOfAlerts({
+                    projectId: projectId,
+                    alertIds: alertIdsToAcknowledge,
+                    props: data.props,
+                });
+            }
+            catch (error) {
+                // A lapsed session or an unpaid project keeps its own answer.
+                if (error instanceof NotAuthorizedException ||
+                    error instanceof BadDataException) {
+                    throw new BadDataException("You do not have permission to acknowledge one or more of these alerts. Declare the incident without acknowledging them, or ask a project admin for permission.");
+                }
+                throw error;
+            }
+        }
+        return {
+            acknowledgedAlertStateId: new ObjectID(acknowledgedState._id.toString()),
+            alertIdsToAcknowledge: alertIdsToAcknowledge,
+        };
+    }
+    /*
+     * Acknowledges the alerts an incident was just declared from, when the
+     * declaration asked for it (INCIDENT_ACKNOWLEDGE_ALERTS_TO_LINK_KEY). An
+     * alert's own on-call escalation and its per-user notifications stop once
+     * the alert is acknowledged (the workers check at their next run), so this
+     * is what makes declaring an incident stop the alerts paging.
+     *
+     * Written as root - the caller's right to change the state of every one of
+     * these alerts was checked before the incident was created - and credited
+     * to the declaring user, as if they had pressed Acknowledge on each alert
+     * themselves: the alert's owners are told, and the alert's feed says who
+     * and why. A private incident is not named in that "why", because the
+     * alert's feed entry is posted to the alert's Slack / Microsoft Teams
+     * channels and its owners' notifications.
+     *
+     * Left as they are:
+     * - alerts already acknowledged, resolved or in any later state - nothing
+     *   is ever moved backwards;
+     * - linked alerts the project's linked-alert sync is about to move anyway
+     *   (the incident was declared straight into an acknowledged or resolved
+     *   state with the project switches on): one writer per alert, so the two
+     *   never race on its timeline.
+     *
+     * IncidentService runs this after the links are written and does not wait
+     * for it. It never throws: one alert failing never stops the others.
+     */
+    async acknowledgeAlertsDeclaredWithIncident(data) {
+        var _a, _b;
+        const result = {
+            acknowledgedAlertIds: [],
+            alreadyAcknowledgedAlertIds: [],
+            leftToLinkedAlertSyncAlertIds: [],
+            failed: [],
+        };
+        const alertIds = uniqueIds(data.alertIds);
+        if (alertIds.length === 0) {
+            return result;
+        }
+        const logAttributes = {
+            projectId: data.projectId.toString(),
+            incidentId: data.incidentId.toString(),
+        };
+        try {
+            const acknowledgedState = await AlertStateService.findOneBy({
+                query: {
+                    projectId: data.projectId,
+                    isAcknowledgedState: true,
+                },
+                select: {
+                    _id: true,
+                    order: true,
+                },
+                props: {
+                    isRoot: true,
+                },
+            });
+            if (!acknowledgedState ||
+                !acknowledgedState._id ||
+                acknowledgedState.order === undefined ||
+                acknowledgedState.order === null) {
+                const message = "This project has no Acknowledged alert state, so the alerts could not be acknowledged.";
+                logger.error(`IncidentAlertService: ${message}`, logAttributes);
+                for (const alertId of alertIds) {
+                    result.failed.push({ alertId: alertId, message: message });
+                }
+                return result;
+            }
+            const acknowledgedStateId = new ObjectID(acknowledgedState._id.toString());
+            const acknowledgedOrder = acknowledgedState.order;
+            const incident = await IncidentService.findOneById({
+                id: data.incidentId,
+                select: {
+                    incidentNumber: true,
+                    incidentNumberWithPrefix: true,
+                    isPrivate: true,
+                    currentIncidentStateId: true,
+                },
+                props: {
+                    isRoot: true,
+                },
+            });
+            const rootCause = getDeclaredAlertAcknowledgementCause({
+                incidentNumber: formatNumber(incident === null || incident === void 0 ? void 0 : incident.incidentNumberWithPrefix, incident === null || incident === void 0 ? void 0 : incident.incidentNumber),
+                isIncidentPrivate: (incident === null || incident === void 0 ? void 0 : incident.isPrivate) === true,
+            });
+            const alerts = await AlertService.findBy({
+                query: {
+                    _id: QueryHelper.any(alertIds),
+                    projectId: data.projectId,
+                },
+                select: {
+                    _id: true,
+                    currentAlertStateId: true,
+                    currentAlertState: {
+                        order: true,
+                    },
+                },
+                limit: LIMIT_PER_PROJECT,
+                skip: 0,
+                props: {
+                    isRoot: true,
+                },
+            });
+            const alertById = new Map();
+            for (const alert of alerts) {
+                if (alert._id) {
+                    alertById.set(normalizeId(alert._id), alert);
+                }
+            }
+            const ownedBySync = await this.getLinkedAlertsTheSyncWillMove({
+                projectId: data.projectId,
+                incidentId: data.incidentId,
+                incidentStateId: (incident === null || incident === void 0 ? void 0 : incident.currentIncidentStateId) || undefined,
+                linkedAlerts: uniqueIds(data.linkedAlertIds)
+                    .map((alertId) => {
+                    return alertById.get(normalizeId(alertId));
+                })
+                    .filter((alert) => {
+                    return Boolean(alert);
+                }),
+            });
+            const alertIdsToWrite = [];
+            for (const alertId of alertIds) {
+                const key = normalizeId(alertId);
+                const alert = alertById.get(key);
+                if (!alert) {
+                    result.failed.push({
+                        alertId: alertId,
+                        message: "The alert could not be found in this project.",
+                    });
+                    continue;
+                }
+                const stateOrder = (_b = (_a = alert.currentAlertState) === null || _a === void 0 ? void 0 : _a.order) !== null && _b !== void 0 ? _b : undefined;
+                if (stateOrder !== undefined && stateOrder >= acknowledgedOrder) {
+                    result.alreadyAcknowledgedAlertIds.push(alertId);
+                    continue;
+                }
+                if (ownedBySync.has(key)) {
+                    result.leftToLinkedAlertSyncAlertIds.push(alertId);
+                    continue;
+                }
+                alertIdsToWrite.push(alertId);
+            }
+            for (let index = 0; index < alertIdsToWrite.length; index += DECLARED_ALERT_ACKNOWLEDGE_CONCURRENCY) {
+                const batch = alertIdsToWrite.slice(index, index + DECLARED_ALERT_ACKNOWLEDGE_CONCURRENCY);
+                const outcomes = await Promise.all(batch.map((alertId) => {
+                    return this.acknowledgeDeclaredAlert({
+                        projectId: data.projectId,
+                        alertId: alertId,
+                        acknowledgedStateId: acknowledgedStateId,
+                        acknowledgedOrder: acknowledgedOrder,
+                        rootCause: rootCause,
+                        acknowledgedByUserId: data.acknowledgedByUserId,
+                    });
+                }));
+                batch.forEach((alertId, batchIndex) => {
+                    var _a;
+                    const failure = (_a = outcomes[batchIndex]) !== null && _a !== void 0 ? _a : null;
+                    if (failure === null) {
+                        result.acknowledgedAlertIds.push(alertId);
+                        return;
+                    }
+                    result.failed.push({ alertId: alertId, message: failure });
+                    logger.error(`IncidentAlertService could not acknowledge alert ${alertId.toString()} for the incident it was declared with: ${failure}`, Object.assign(Object.assign({}, logAttributes), { alertId: alertId.toString() }));
+                });
+            }
+        }
+        catch (error) {
+            logger.error(`IncidentAlertService could not acknowledge the alerts an incident was declared from: ${error}`, logAttributes);
+            const settled = new Set([
+                ...result.acknowledgedAlertIds,
+                ...result.alreadyAcknowledgedAlertIds,
+                ...result.leftToLinkedAlertSyncAlertIds,
+                ...result.failed.map((failure) => {
+                    return failure.alertId;
+                }),
+            ].map((alertId) => {
+                return normalizeId(alertId);
+            }));
+            for (const alertId of alertIds) {
+                if (!settled.has(normalizeId(alertId))) {
+                    result.failed.push({
+                        alertId: alertId,
+                        message: error instanceof Error ? error.message : String(error),
+                    });
+                }
+            }
+        }
+        return result;
+    }
+    /*
+     * The linked alerts the project's linked-alert sync (and the incident
+     * state cascade) will move by themselves, because the incident is already
+     * in a state their switches act on: the same plan, the same per-alert
+     * choice. Empty when both switches are off - the default - or the incident
+     * is before Acknowledged, which is the usual declaration.
+     */
+    async getLinkedAlertsTheSyncWillMove(data) {
+        const willMove = new Set();
+        if (!data.incidentStateId || data.linkedAlerts.length === 0) {
+            return willMove;
+        }
+        const switches = await this.getLinkedAlertSwitches(data.projectId);
+        if (!switches) {
+            return willMove;
+        }
+        const plan = await this.buildLinkedAlertStatePlan({
+            projectId: data.projectId,
+            incidentStateId: data.incidentStateId,
+            switches: switches,
+        });
+        if (!plan) {
+            return willMove;
+        }
+        const blockedFromResolve = plan.resolvedAlertState
+            ? await this.getAlertsHeldOpenByAnotherIncident({
+                plan: plan,
+                projectId: data.projectId,
+                incidentId: data.incidentId,
+                alerts: data.linkedAlerts,
+            })
+            : new Set();
+        for (const alert of data.linkedAlerts) {
+            const alertId = alert._id ? normalizeId(alert._id) : "";
+            const alertStateOrder = alert.currentAlertStateId
+                ? plan.alertStateOrderById.get(normalizeId(alert.currentAlertStateId))
+                : undefined;
+            if (!alertId || alertStateOrder === undefined) {
+                continue;
+            }
+            const target = chooseLinkedAlertTargetState({
+                alertStateOrder: alertStateOrder,
+                acknowledgedAlertState: plan.acknowledgedAlertState,
+                resolvedAlertState: plan.resolvedAlertState,
+                isBlockedFromResolve: blockedFromResolve.has(alertId),
+            });
+            if (target) {
+                willMove.add(alertId);
+            }
+        }
+        return willMove;
+    }
+    /*
+     * Acknowledges one declared alert. Null when the alert ended up
+     * acknowledged (or past it), otherwise why not. Never throws.
+     *
+     * The outcome is read back from the alert rather than taken from the
+     * write: the write may be refused because somebody acknowledged or
+     * resolved the alert in the meantime (the outcome asked for), and a write
+     * that "succeeded" only counts if the alert's current state moved.
+     */
+    async acknowledgeDeclaredAlert(data) {
+        var _a, _b;
+        let failure = "The alert's state did not change to Acknowledged.";
+        try {
+            await AlertService.changeAlertState({
+                projectId: data.projectId,
+                alertId: data.alertId,
+                alertStateId: data.acknowledgedStateId,
+                notifyOwners: true,
+                rootCause: data.rootCause,
+                stateChangeLog: undefined,
+                createdByUserId: data.acknowledgedByUserId,
+                props: {
+                    isRoot: true,
+                },
+            });
+        }
+        catch (error) {
+            failure = error instanceof Error ? error.message : String(error);
+        }
+        try {
+            const alert = await AlertService.findOneBy({
+                query: {
+                    _id: data.alertId,
+                    projectId: data.projectId,
+                },
+                select: {
+                    currentAlertState: {
+                        order: true,
+                    },
+                },
+                props: {
+                    isRoot: true,
+                },
+            });
+            const order = (_b = (_a = alert === null || alert === void 0 ? void 0 : alert.currentAlertState) === null || _a === void 0 ? void 0 : _a.order) !== null && _b !== void 0 ? _b : undefined;
+            if (order !== undefined && order >= data.acknowledgedOrder) {
+                return null;
+            }
+        }
+        catch (_c) {
+            // The alert cannot be read back, so it cannot be counted as done.
+        }
+        return failure;
+    }
     /*
      * An incident changed state: bring its linked alerts along, when the
      * project has opted in. Called fire-and-forget from
@@ -919,8 +1346,11 @@ export class Service extends DatabaseService {
     /*
      * The link-time half: an alert was just linked to an incident that may
      * already be acknowledged or resolved. Same rules as the cascade, applied to
-     * the one alert. A newly declared incident is in its created state, so
-     * this is a no-op for "declare incident from alerts". Never throws.
+     * the one alert. An incident declared from alerts usually starts in its
+     * created state, which calls for nothing; one declared straight into an
+     * acknowledged or resolved state brings its alerts along here (and
+     * acknowledgeAlertsDeclaredWithIncident leaves those alerts to this).
+     * Never throws.
      */
     async syncAlertWithLinkedIncidentState(data) {
         try {
@@ -1330,6 +1760,30 @@ __decorate([
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "linkAlertsToIncident", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "validateAcknowledgeAlertsForNewIncident", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "acknowledgeAlertsDeclaredWithIncident", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getLinkedAlertsTheSyncWillMove", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "acknowledgeDeclaredAlert", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),