@oneuptime/common 14.0.1 → 14.0.2

package/build/dist/Server/Services/MonitorService.js

@@ -8,7 +8,7 @@ var __metadata = (this && this.__metadata) || function (k, v) {
     if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
 };
 import DatabaseConfig from "../DatabaseConfig";
-import { AllowedActiveMonitorCountInFreePlan, IsBillingEnabled, } from "../EnvironmentConfig";
+import { AllowedActiveMonitorCountInFreePlan, InboundEmailDomain, IsBillingEnabled, } from "../EnvironmentConfig";
 import { ActiveMonitoringMeteredPlan } from "../Types/Billing/MeteredPlan/AllMeteredPlans";
 import QueryHelper from "../Types/Database/QueryHelper";
 import DatabaseService from "./DatabaseService";
@@ -69,10 +69,18 @@ import RelationIdUtil from "../Utils/Database/RelationIdUtil";
 import OwnerRuleAssignment from "../Utils/Rules/OwnerRuleAssignment";
 import HostAddressUtil from "../../Utils/HostAddressUtil";
 import NetworkDeviceMonitorTemplateUtil from "../../Utils/Monitor/NetworkDeviceMonitorTemplateUtil";
+import IncomingEmailMonitorAddress from "../../Utils/Monitor/IncomingEmailMonitorAddress";
+import ProbeMonitorsNotification from "../Utils/Monitor/ProbeMonitorsNotification";
 const MONITOR_TEMPLATE_RELATION_KEYS = [
     "monitorTemplateId",
     "monitorTemplate",
 ];
+/*
+ * A global or shared probe can be attached to thousands of monitors. Work
+ * that fans out per monitor (flag syncs, owner lookups) runs in batches of
+ * this size instead of all at once, so it cannot exhaust the connection pool.
+ */
+const PROBE_FAN_OUT_CONCURRENCY = 50;
 export class Service extends DatabaseService {
     constructor() {
         super(Model);
@@ -244,7 +252,16 @@ export class Service extends DatabaseService {
                 ignoreHooks: true,
             },
         });
-        const lastMonitorStatus = await MonitorStatusTimelineService.findOneBy({
+        /*
+         * Exactly one open row is the healthy shape, and only then is that row
+         * the monitor's current status. Two or more is the orphan case the
+         * KeepCurrentStateConsistent reconciler repairs first. The unsorted
+         * findOneBy this replaces took whichever open row the database happened
+         * to return, so an old orphan could be written back over the real
+         * current status. Two rows, newest first, is enough to tell one from
+         * several.
+         */
+        const openRows = await MonitorStatusTimelineService.findBy({
             query: {
                 monitorId: monitorId,
                 endsAt: QueryHelper.isNull(),
@@ -254,13 +271,19 @@ export class Service extends DatabaseService {
                 monitorStatusId: true,
                 projectId: true,
             },
+            sort: {
+                startsAt: SortOrder.Descending,
+            },
+            limit: 2,
+            skip: 0,
             props: {
                 isRoot: true,
             },
         });
-        if (!lastMonitorStatus) {
+        if (openRows.length !== 1) {
             return;
         }
+        const lastMonitorStatus = openRows[0];
         if (!lastMonitorStatus.monitorStatusId) {
             return;
         }
@@ -481,6 +504,9 @@ export class Service extends DatabaseService {
         if (updateDataKeys.includes("monitoringInterval")) {
             updateBy.data["monitoringInterval"] = MonitoringIntervalValidator.validateAndNormalize(updateBy.data.monitoringInterval);
         }
+        if (updateDataKeys.includes("incomingEmailCustomLocalPart")) {
+            await this.validateIncomingEmailCustomLocalPartUpdate(updateBy);
+        }
         const isMonitorStepsWritten = updateDataKeys.includes("monitorSteps");
         const isMonitorTemplateWritten = RelationIdUtil.isWritten(updateDataKeys, MONITOR_TEMPLATE_RELATION_KEYS);
         if (isMonitorStepsWritten || isMonitorTemplateWritten) {
@@ -817,6 +843,91 @@ export class Service extends DatabaseService {
         }
         return ids;
     }
+    /*
+     * Guards a write to incomingEmailCustomLocalPart, the custom name of an
+     * Incoming Email monitor's inbound address (see IncomingEmailMonitorAddress).
+     *
+     * Normalizes the value in place -- the stored name is always the bare,
+     * lowercased local part, whatever the caller typed -- and refuses a name
+     * that is malformed, reserved, or already another monitor's address. Every
+     * project shares the one inbound domain, so "taken" is checked across ALL
+     * monitors, not just this project's. The column's unique index is the
+     * backstop for two writes racing past this check; this is what gives the
+     * person a readable message in the common case.
+     *
+     * null or "" clears the custom name, which puts the generated
+     * monitor-{secretKey} address back in service.
+     */
+    async validateIncomingEmailCustomLocalPartUpdate(updateBy) {
+        const data = updateBy.data;
+        const rawValue = data["incomingEmailCustomLocalPart"];
+        if (rawValue === null || rawValue === undefined || rawValue === "") {
+            data["incomingEmailCustomLocalPart"] = null;
+            return;
+        }
+        if (typeof rawValue !== "string") {
+            throw new BadDataException("Incoming email custom address must be a string.");
+        }
+        const localPart = IncomingEmailMonitorAddress.normalizeCustomLocalPart({
+            value: rawValue,
+            inboundDomain: InboundEmailDomain,
+        });
+        data["incomingEmailCustomLocalPart"] = localPart;
+        /*
+         * Two rows are enough to know the write targets more than one monitor,
+         * which is itself the error: one address cannot route to two monitors.
+         */
+        const targets = await this.findBy({
+            query: !updateBy.props.isRoot && updateBy.props.tenantId
+                ? Object.assign(Object.assign({}, updateBy.query), { projectId: updateBy.props.tenantId }) : updateBy.query,
+            select: {
+                _id: true,
+                monitorType: true,
+            },
+            limit: 2,
+            skip: 0,
+            props: {
+                isRoot: true,
+                ignoreHooks: true,
+            },
+        });
+        if (targets.length > 1) {
+            throw new BadDataException("A custom email address can only be set on one monitor at a time.");
+        }
+        const target = targets[0];
+        // Nothing matched: the update writes nothing, so there is nothing to guard.
+        if (!target) {
+            return;
+        }
+        if (target.monitorType !== MonitorType.IncomingEmail) {
+            throw new BadDataException("Custom email addresses can only be set on Incoming Email monitors.");
+        }
+        const holders = await this.findBy({
+            query: {
+                incomingEmailCustomLocalPart: localPart,
+            },
+            select: {
+                _id: true,
+            },
+            limit: 2,
+            skip: 0,
+            props: {
+                isRoot: true,
+                ignoreHooks: true,
+            },
+        });
+        const isTakenByAnotherMonitor = holders.some((holder) => {
+            var _a, _b;
+            return ((_a = holder.id) === null || _a === void 0 ? void 0 : _a.toString()) !== ((_b = target.id) === null || _b === void 0 ? void 0 : _b.toString());
+        });
+        if (isTakenByAnotherMonitor) {
+            const address = IncomingEmailMonitorAddress.getAddress({
+                customLocalPart: localPart,
+                inboundDomain: InboundEmailDomain,
+            }) || localPart;
+            throw new BadDataException(`The email address ${address} is already used by another monitor. Please choose a different name.`);
+        }
+    }
     async getProjectIdsForUpdateQuery(updateBy) {
         const monitors = await this.findBy({
             query: updateBy.query,
@@ -1476,6 +1587,7 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
         return URL.fromString(dashboardUrl.toString()).addRoute(`/${projectId.toString()}/monitors/${monitorId.toString()}`);
     }
     async findOwners(monitorId) {
+        var _a, _b;
         if (!monitorId) {
             throw new BadDataException("monitorId is required");
         }
@@ -1485,6 +1597,7 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
             },
             select: {
                 _id: true,
+                projectId: true,
                 user: {
                     _id: true,
                     email: true,
@@ -1504,6 +1617,7 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
             },
             select: {
                 _id: true,
+                projectId: true,
                 teamId: true,
             },
             skip: 0,
@@ -1530,7 +1644,15 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
                 }
             }
         }
-        return users;
+        const projectId = ((_a = ownerUsers[0]) === null || _a === void 0 ? void 0 : _a.projectId) || ((_b = ownerTeams[0]) === null || _b === void 0 ? void 0 : _b.projectId);
+        if (!projectId) {
+            return [];
+        }
+        // Owners who left the project are not notified, nor listed as notified.
+        return await TeamMemberService.filterUsersToProjectMembers({
+            projectId: projectId,
+            users: users,
+        });
     }
     async addOwners(projectId, monitorId, userIds, teamIds, notifyOwners, props) {
         // Owners already on the monitor are skipped, not added a second time.
@@ -1682,7 +1804,40 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
         // Execute all creates in parallel
         await Promise.all(createPromises);
     }
+    /*
+     * Re-derives a monitor's two probe flags and announces what changed, one
+     * notification per monitor. This is the path for changes that concern one
+     * monitor: a probe attached, enabled, disabled or removed, or a monitor
+     * created. A probe connecting or disconnecting goes through
+     * refreshProbeStatus instead, which announces all of that probe's monitors
+     * in one message per person.
+     */
     async refreshMonitorProbeStatus(monitorId) {
+        const changes = await this.syncMonitorProbeFlags(monitorId);
+        if (!changes) {
+            return;
+        }
+        if (changes.isNoProbeEnabledOnThisMonitor !== undefined) {
+            await this.notifyOwnersWhenNoProbeIsEnabled({
+                monitorId: monitorId,
+                isNoProbesEnabled: changes.isNoProbeEnabledOnThisMonitor,
+            });
+        }
+        if (changes.isAllProbesDisconnectedFromThisMonitor !== undefined) {
+            await this.notifyOwnersProbesDisconnected({
+                monitorId: monitorId,
+                isProbeDisconnected: changes.isAllProbesDisconnectedFromThisMonitor,
+            });
+        }
+    }
+    /*
+     * Re-derives isNoProbeEnabledOnThisMonitor and
+     * isAllProbesDisconnectedFromThisMonitor from the monitor's MonitorProbe
+     * rows, writes the flags that changed, and reports them. Sends nothing:
+     * the caller decides how to announce the changes. Returns null when the
+     * monitor is gone or is not checked by probes.
+     */
+    async syncMonitorProbeFlags(monitorId) {
         const monitor = await this.findOneById({
             id: monitorId,
             select: {
@@ -1696,19 +1851,22 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
             },
         });
         if (!monitor) {
-            return;
+            return null;
         }
         if (!monitor.id) {
-            return;
+            return null;
         }
         const monitorType = monitor === null || monitor === void 0 ? void 0 : monitor.monitorType;
         if (!monitorType) {
-            return;
+            return null;
         }
         const isProbeableMonitor = MonitorTypeHelper.isProbableMonitor(monitorType);
         if (!isProbeableMonitor) {
-            return;
+            return null;
         }
+        const changes = {
+            monitorId: monitorId,
+        };
         // get all the probes for this monitor.
         const probesForMonitor = await MonitorProbeService.findBy({
             query: {
@@ -1746,11 +1904,7 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
                         isRoot: true,
                     },
                 });
-                // notify owners that no probe is enabled.
-                await this.notifyOwnersWhenNoProbeIsEnabled({
-                    monitorId: monitorId,
-                    isNoProbesEnabled: true,
-                });
+                changes.isNoProbeEnabledOnThisMonitor = true;
             }
         }
         else if (monitor.isNoProbeEnabledOnThisMonitor) {
@@ -1763,11 +1917,28 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
                     isRoot: true,
                 },
             });
-            // notify owners that probes are now enabled.
-            await this.notifyOwnersWhenNoProbeIsEnabled({
-                monitorId: monitorId,
-                isNoProbesEnabled: false,
-            });
+            changes.isNoProbeEnabledOnThisMonitor = false;
+        }
+        /*
+         * With no enabled probe left there is nothing to be connected or
+         * disconnected. Clear a stale "all disconnected" flag quietly: the "no
+         * probes enabled" notice above already tells the owners this monitor is
+         * not being checked, and announcing "Probes ... are Connected" here
+         * would say the opposite.
+         */
+        if (enabledProbes.length === 0) {
+            if (monitor.isAllProbesDisconnectedFromThisMonitor) {
+                await this.updateOneById({
+                    id: monitorId,
+                    data: {
+                        isAllProbesDisconnectedFromThisMonitor: false,
+                    },
+                    props: {
+                        isRoot: true,
+                    },
+                });
+            }
+            return changes;
         }
         const disconnectedProbes = probesForMonitor.filter((monitorProbe) => {
             var _a;
@@ -1782,11 +1953,10 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
             });
             if (anyGlobalProbe) {
                 // do not notify if any global probe is disconnected.
-                return;
+                return changes;
             }
         }
-        if (disconnectedProbes.length === enabledProbes.length &&
-            enabledProbes.length > 0) {
+        if (disconnectedProbes.length === enabledProbes.length) {
             if (!monitor.isAllProbesDisconnectedFromThisMonitor) {
                 // all probes are disconnected.
                 await this.updateOneById({
@@ -1798,10 +1968,7 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
                         isRoot: true,
                     },
                 });
-                await this.notifyOwnersProbesDisconnected({
-                    monitorId: monitorId,
-                    isProbeDisconnected: true,
-                });
+                changes.isAllProbesDisconnectedFromThisMonitor = true;
             }
         }
         else if (monitor.isAllProbesDisconnectedFromThisMonitor) {
@@ -1814,11 +1981,9 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
                     isRoot: true,
                 },
             });
-            await this.notifyOwnersProbesDisconnected({
-                monitorId: monitorId,
-                isProbeDisconnected: false,
-            });
+            changes.isAllProbesDisconnectedFromThisMonitor = false;
         }
+        return changes;
     }
     async getLabelsForMonitors(data) {
         if (data.monitorIds.length === 0) {
@@ -1916,6 +2081,7 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
                 templateType: EmailTemplateType.MonitorProbesStatus,
                 vars: vars,
                 subject: `[${enabledStatus} Monitor Probes] ${monitor.name}`,
+                isSubjectLiteral: true,
             };
             const sms = {
                 message: `This is a message from OneUptime. Probes for monitor ${monitor.name} is ${enabledStatus}. To unsubscribe from this notification go to User Settings in OneUptime Dashboard.`,
@@ -2014,14 +2180,15 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
                 templateType: EmailTemplateType.MonitorProbesStatus,
                 vars: vars,
                 subject: `[${status} Monitor Probes] ${monitor.name}`,
+                isSubjectLiteral: true,
             };
             const sms = {
-                message: `This is a message from OneUptime. Probes for monitor ${monitor.name} is ${status}. To unsubscribe from this notification go to User Settings in OneUptime Dashboard.`,
+                message: `This is a message from OneUptime. Probes for monitor ${monitor.name} are ${status}. To unsubscribe from this notification go to User Settings in OneUptime Dashboard.`,
             };
             const callMessage = {
                 data: [
                     {
-                        sayMessage: `This is a message from OneUptime. New monitor was created ${monitor.name}. To unsubscribe from this notification go to User Settings in OneUptime Dashboard. Good bye.`,
+                        sayMessage: `This is a message from OneUptime. Probes for monitor ${monitor.name} are ${status}. To unsubscribe from this notification go to User Settings in OneUptime Dashboard. Good bye.`,
                     },
                 ],
             };
@@ -2040,9 +2207,12 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
                 emailEnvelope: emailMessage,
                 smsMessage: sms,
                 callRequestMessage: callMessage,
-                pushNotificationMessage: PushNotificationUtil.createMonitorCreatedNotification({
-                    monitorName: monitor.name,
+                pushNotificationMessage: PushNotificationUtil.createMonitorProbeStatusNotification({
+                    title: "OneUptime: Monitor Probe Status",
+                    body: `Probes for monitor ${monitor.name} are ${status}`,
+                    tag: "monitor-probe-status",
                     monitorId: monitor.id.toString(),
+                    monitorName: monitor.name,
                 }),
                 whatsAppMessage,
                 eventType,
@@ -2050,6 +2220,18 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
             });
         }
     }
+    /*
+     * A probe connected or disconnected: re-derive the flags of every monitor
+     * it checks, then announce the monitors it took down (or brought back) in
+     * ONE message per person per project, instead of one message per monitor.
+     *
+     * Only monitors that use this probe (their row for it is enabled) and whose
+     * flag moved the same way as the probe are grouped under its name. Any
+     * other flag change seen during this refresh (a stale flag being corrected,
+     * on a monitor that moved the other way or does not use this probe) did not
+     * happen BECAUSE of this probe, so it keeps the per-monitor message, which
+     * does not name a probe.
+     */
     async refreshProbeStatus(probeId) {
         // get all the monitors for this probe.
         const monitorProbes = await MonitorProbeService.findBy({
@@ -2058,12 +2240,8 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
             },
             select: {
                 _id: true,
-                isEnabled: true,
-                projectId: true,
                 monitorId: true,
-                monitor: {
-                    monitorType: true,
-                },
+                isEnabled: true,
             },
             skip: 0,
             limit: LIMIT_PER_PROJECT,
@@ -2074,21 +2252,258 @@ ${((_d = createdItem.description) === null || _d === void 0 ? void 0 : _d.trim()
         if (monitorProbes.length === 0) {
             return;
         }
+        const probe = await ProbeService.findOneById({
+            id: probeId,
+            select: {
+                _id: true,
+                name: true,
+                connectionStatus: true,
+            },
+            props: {
+                isRoot: true,
+            },
+        });
         /*
          * Each monitor appears at most once for a given probeId (composite
-         * unique on MonitorProbe), so concurrent refreshes operate on disjoint
-         * rows and are safe to run in parallel. A global/shared probe can be
-         * attached to thousands of monitors, though, so refresh in bounded
-         * batches instead of firing every refresh at once — an unbounded
-         * Promise.all here would exhaust the database connection pool.
+         * unique on MonitorProbe); dedupe anyway so a monitor can never be
+         * listed, or announced, twice.
+         */
+        const monitorIds = [];
+        const seenMonitorIds = new Set();
+        /*
+         * Monitors that actually use this probe. A monitor whose row for it is
+         * disabled is still synced (its flag may be stale), but whatever moved
+         * its flag, it was not this probe.
+         */
+        const monitorIdsUsingProbe = new Set();
+        for (const monitorProbe of monitorProbes) {
+            if (!monitorProbe.monitorId) {
+                continue;
+            }
+            const key = monitorProbe.monitorId.toString();
+            if (monitorProbe.isEnabled) {
+                monitorIdsUsingProbe.add(key);
+            }
+            if (seenMonitorIds.has(key)) {
+                continue;
+            }
+            seenMonitorIds.add(key);
+            monitorIds.push(monitorProbe.monitorId);
+        }
+        /*
+         * Concurrent syncs operate on disjoint monitors and are safe in
+         * parallel, but run in bounded batches (see PROBE_FAN_OUT_CONCURRENCY).
+         * One monitor failing is logged and does not stop the others.
+         */
+        const allChanges = [];
+        for (let i = 0; i < monitorIds.length; i += PROBE_FAN_OUT_CONCURRENCY) {
+            const batch = monitorIds.slice(i, i + PROBE_FAN_OUT_CONCURRENCY);
+            const batchChanges = await Promise.all(batch.map(async (monitorId) => {
+                try {
+                    return await this.syncMonitorProbeFlags(monitorId);
+                }
+                catch (err) {
+                    logger.error(`Error refreshing probe flags of monitor ${monitorId.toString()} after probe ${probeId.toString()} changed status`);
+                    logger.error(err);
+                    return null;
+                }
+            }));
+            for (const changes of batchChanges) {
+                if (changes) {
+                    allChanges.push(changes);
+                }
+            }
+        }
+        /*
+         * The direction this probe moved. Without a probe row or a status (the
+         * probe was deleted mid-refresh) nothing can be attributed to it, and
+         * every change falls back to the per-monitor message.
          */
-        const refreshConcurrency = 50;
-        for (let i = 0; i < monitorProbes.length; i += refreshConcurrency) {
-            const batch = monitorProbes.slice(i, i + refreshConcurrency);
-            await Promise.all(batch.map((monitorProbe) => {
-                return this.refreshMonitorProbeStatus(monitorProbe.monitorId);
+        let isProbeDisconnected = null;
+        if ((probe === null || probe === void 0 ? void 0 : probe.connectionStatus) === ProbeConnectionStatus.Disconnected) {
+            isProbeDisconnected = true;
+        }
+        else if ((probe === null || probe === void 0 ? void 0 : probe.connectionStatus) === ProbeConnectionStatus.Connected) {
+            isProbeDisconnected = false;
+        }
+        const groupedMonitorIds = [];
+        for (const changes of allChanges) {
+            if (changes.isNoProbeEnabledOnThisMonitor !== undefined) {
+                await this.notifySafely(`no-probe-enabled notification for monitor ${changes.monitorId.toString()}`, () => {
+                    return this.notifyOwnersWhenNoProbeIsEnabled({
+                        monitorId: changes.monitorId,
+                        isNoProbesEnabled: changes.isNoProbeEnabledOnThisMonitor,
+                    });
+                });
+            }
+            if (changes.isAllProbesDisconnectedFromThisMonitor === undefined) {
+                continue;
+            }
+            if (probe &&
+                isProbeDisconnected !== null &&
+                monitorIdsUsingProbe.has(changes.monitorId.toString()) &&
+                changes.isAllProbesDisconnectedFromThisMonitor === isProbeDisconnected) {
+                groupedMonitorIds.push(changes.monitorId);
+                continue;
+            }
+            await this.notifySafely(`probe status notification for monitor ${changes.monitorId.toString()}`, () => {
+                return this.notifyOwnersProbesDisconnected({
+                    monitorId: changes.monitorId,
+                    isProbeDisconnected: changes.isAllProbesDisconnectedFromThisMonitor,
+                });
+            });
+        }
+        if (probe && isProbeDisconnected !== null && groupedMonitorIds.length > 0) {
+            await this.notifyOwnersOfMonitorsAffectedByProbeStatusChange({
+                probeId: probeId,
+                probeName: probe.name || "Probe",
+                isProbeDisconnected: isProbeDisconnected,
+                monitorIds: groupedMonitorIds,
+            });
+        }
+    }
+    // Runs one notification, logging instead of throwing if it fails.
+    async notifySafely(description, notify) {
+        try {
+            await notify();
+        }
+        catch (err) {
+            logger.error(`Error sending ${description}`);
+            logger.error(err);
+        }
+    }
+    /*
+     * One message per person per project for all the monitors a probe took
+     * down (isProbeDisconnected) or brought back. Recipients are exactly the
+     * people the per-monitor message would have reached: each monitor's
+     * owners, or the project owners for a monitor that has no owners.
+     */
+    async notifyOwnersOfMonitorsAffectedByProbeStatusChange(data) {
+        if (data.monitorIds.length === 0) {
+            return;
+        }
+        const monitors = await this.findBy({
+            query: {
+                _id: QueryHelper.any(data.monitorIds),
+            },
+            select: {
+                _id: true,
+                name: true,
+                projectId: true,
+                project: {
+                    name: true,
+                },
+            },
+            skip: 0,
+            limit: LIMIT_PER_PROJECT,
+            props: {
+                isRoot: true,
+            },
+        });
+        // A shared probe can serve monitors in many projects: one pass each.
+        const monitorsByProjectId = new Map();
+        for (const monitor of monitors) {
+            if (!monitor.id || !monitor.projectId) {
+                continue;
+            }
+            const projectKey = monitor.projectId.toString();
+            const projectMonitors = monitorsByProjectId.get(projectKey) || [];
+            projectMonitors.push(monitor);
+            monitorsByProjectId.set(projectKey, projectMonitors);
+        }
+        for (const projectMonitors of monitorsByProjectId.values()) {
+            await this.notifySafely(`grouped probe status notification for project ${projectMonitors[0].projectId.toString()}`, () => {
+                return this.notifyProjectOfMonitorsAffectedByProbeStatusChange({
+                    probeId: data.probeId,
+                    probeName: data.probeName,
+                    isProbeDisconnected: data.isProbeDisconnected,
+                    monitors: projectMonitors,
+                });
+            });
+        }
+    }
+    async notifyProjectOfMonitorsAffectedByProbeStatusChange(data) {
+        var _a, _b;
+        const projectId = data.monitors[0].projectId;
+        const projectName = ((_a = data.monitors[0].project) === null || _a === void 0 ? void 0 : _a.name) || "Project";
+        const affectedMonitors = [];
+        const ownersByMonitorId = {};
+        for (let i = 0; i < data.monitors.length; i += PROBE_FAN_OUT_CONCURRENCY) {
+            const batch = data.monitors.slice(i, i + PROBE_FAN_OUT_CONCURRENCY);
+            await Promise.all(batch.map(async (monitor) => {
+                try {
+                    const owners = await this.findOwners(monitor.id);
+                    affectedMonitors.push({
+                        monitorId: monitor.id,
+                        monitorName: monitor.name || "Monitor",
+                        monitorViewLink: (await this.getMonitorLinkInDashboard(projectId, monitor.id)).toString(),
+                    });
+                    ownersByMonitorId[monitor.id.toString()] = owners;
+                }
+                catch (err) {
+                    /*
+                     * Without its owners there is no telling who should hear about
+                     * this monitor. Leave it out rather than mail it to the wrong
+                     * people.
+                     */
+                    logger.error(`Error resolving owners of monitor ${monitor.id.toString()} for a grouped probe status notification`);
+                    logger.error(err);
+                }
             }));
         }
+        if (affectedMonitors.length === 0) {
+            return;
+        }
+        const needsProjectOwners = affectedMonitors.some((monitor) => {
+            return ((ownersByMonitorId[monitor.monitorId.toString()] || []).length === 0);
+        });
+        const projectOwners = needsProjectOwners
+            ? await ProjectService.getOwners(projectId)
+            : [];
+        const recipients = ProbeMonitorsNotification.groupMonitorsByRecipient({
+            // Owners resolve concurrently; sort so recipients come out stable.
+            monitors: ProbeMonitorsNotification.sortMonitors(affectedMonitors),
+            ownersByMonitorId: ownersByMonitorId,
+            projectOwners: projectOwners,
+        });
+        if (recipients.length === 0) {
+            return;
+        }
+        /*
+         * The probe id rides along as a query parameter the page ignores. It
+         * keeps the link unique per probe, and the email rollup folds deferred
+         * emails that share a link into one row: without it, two probes going
+         * down in one project would collapse into a single digest row naming
+         * only the last one.
+         */
+        const viewMonitorsLink = (data.isProbeDisconnected
+            ? await this.getMonitorsWithDisconnectedProbesLinkInDashboard(projectId)
+            : await this.getMonitorsLinkInDashboard(projectId))
+            .addQueryParam("probeId", data.probeId.toString())
+            .toString();
+        for (const recipient of recipients) {
+            await this.notifySafely(`grouped probe status notification to user ${(_b = recipient.user.id) === null || _b === void 0 ? void 0 : _b.toString()}`, async () => {
+                const content = ProbeMonitorsNotification.buildContent({
+                    probeName: data.probeName,
+                    projectName: projectName,
+                    isProbeDisconnected: data.isProbeDisconnected,
+                    recipient: recipient,
+                    viewMonitorsLink: viewMonitorsLink,
+                });
+                await UserNotificationSettingService.sendUserNotification(Object.assign({ userId: recipient.user.id, projectId: projectId, emailEnvelope: content.emailEnvelope, smsMessage: content.smsMessage, callRequestMessage: content.callRequestMessage, pushNotificationMessage: content.pushNotificationMessage, whatsAppMessage: content.whatsAppMessage, eventType: ProbeMonitorsNotification.EVENT_TYPE }, (recipient.monitors.length === 1
+                    ? { monitorId: recipient.monitors[0].monitorId }
+                    : {})));
+            });
+        }
+    }
+    async getMonitorsLinkInDashboard(projectId) {
+        const dashboardUrl = await DatabaseConfig.getDashboardUrl();
+        return URL.fromString(dashboardUrl.toString()).addRoute(`/${projectId.toString()}/monitors`);
+    }
+    // The "Probe Disconnected" list: every monitor whose probes are all down.
+    async getMonitorsWithDisconnectedProbesLinkInDashboard(projectId) {
+        const dashboardUrl = await DatabaseConfig.getDashboardUrl();
+        return URL.fromString(dashboardUrl.toString()).addRoute(`/${projectId.toString()}/monitors/probe-disconnected`);
     }
     async changeMonitorStatus(projectId, monitorIds, monitorStatusId, notifyOwners, rootCause, statusChangeLog, props, startsAt) {
         for (const monitorId of monitorIds) {
@@ -2379,6 +2794,12 @@ __decorate([
     __metadata("design:paramtypes", [ObjectID]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "refreshMonitorProbeStatus", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [ObjectID]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "syncMonitorProbeFlags", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),
@@ -2403,6 +2824,24 @@ __decorate([
     __metadata("design:paramtypes", [ObjectID]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "refreshProbeStatus", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "notifyOwnersOfMonitorsAffectedByProbeStatusChange", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [ObjectID]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getMonitorsLinkInDashboard", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [ObjectID]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getMonitorsWithDisconnectedProbesLinkInDashboard", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),