@oneuptime/common 12.0.24 → 12.0.26

package/build/dist/Server/Services/NetworkSiteService.js

@@ -35,6 +35,10 @@ import { Raw } from "typeorm";
 import RelationIdUtil from "../Utils/Database/RelationIdUtil";
 import MaterializedPathUtil from "../../Utils/NetworkSite/MaterializedPathUtil";
 import SiteStatusRollupUtil from "../../Utils/NetworkSite/SiteStatusRollupUtil";
+import { parseSiteHealthRollupPolicy } from "../../Types/NetworkSite/SiteHealthRollupPolicy";
+import NetworkSiteMaintenanceSuppression from "../Utils/NetworkSite/NetworkSiteMaintenanceSuppression";
+import AggregateResultUtil from "../Types/Database/AggregateResultUtil";
+import { deviceRollupStateForGroup, } from "../Utils/NetworkDevice/DeviceHealthAggregation";
 /*
  * Both spellings of "this site's parent" in a write payload: the dashboard's
  * site form posts the `parentSite` relation while server-side callers write
@@ -47,6 +51,43 @@ export class Service extends DatabaseService {
     constructor() {
         super(Model);
     }
+    /**
+     * How many sites sit under each rolled-up MonitorStatus, counted in the
+     * database.
+     *
+     * One row per status the project actually uses (plus one for "no rollup
+     * yet"), rather than every site row shipped to a browser to be tallied
+     * there. Which of those statuses count as UNHEALTHY is deliberately not
+     * decided here: `isOperationalState` lives on MonitorStatus, the caller
+     * already holds those rows, and duplicating the flag into this query would
+     * be a second place for it to be read wrongly.
+     */
+    async getStatusCounts(data) {
+        const rows = await this.aggregateBy({
+            query: {
+                projectId: data.projectId,
+            },
+            groupBy: [
+                {
+                    expression: `"NetworkSite"."currentMonitorStatusId"`,
+                    alias: "monitorStatusId",
+                },
+            ],
+            select: [
+                {
+                    expression: `COUNT(*)`,
+                    alias: "siteCount",
+                },
+            ],
+            props: data.props,
+        });
+        return rows.map((row) => {
+            return {
+                monitorStatusId: AggregateResultUtil.toStringOrNull(row, "monitorStatusId"),
+                siteCount: AggregateResultUtil.toNumber(row, "siteCount"),
+            };
+        });
+    }
     /*
      * ------------------------------------------------------------------
      * Hierarchy maintenance (materializedPath + depth)
@@ -81,6 +122,29 @@ export class Service extends DatabaseService {
             [rid]: `${path}%`,
         });
     }
+    /*
+     * The same prefix predicate for SEVERAL paths at once, OR-ed together, so
+     * "every site under any of these" is one statement rather than one per
+     * root. Each prefix keeps its own bound parameter; an empty list would
+     * produce `()`, which is a syntax error, so callers must not reach here
+     * with one (getSubtreeSiteIds guards it).
+     */
+    pathStartsWithAny(paths) {
+        const parameters = {};
+        const names = [];
+        for (const path of paths) {
+            const rid = Text.generateRandomText(10);
+            parameters[rid] = `${path}%`;
+            names.push(rid);
+        }
+        return Raw((alias) => {
+            return `(${names
+                .map((name) => {
+                return `${alias} LIKE :${name}`;
+            })
+                .join(" OR ")})`;
+        }, parameters);
+    }
     /*
      * A stored path is trustworthy only when it agrees with parentSiteId: it
      * must end with the site's own id, and the segment before it must be the
@@ -235,6 +299,33 @@ export class Service extends DatabaseService {
     }
     async onUpdateSuccess(onUpdate, updatedItemIds) {
         var _a;
+        /*
+         * Editing the rollup POLICY (or its threshold) changes what the site's
+         * existing devices add up to, without touching a device or the tree. The
+         * five-minute stale sweep would eventually notice, but a settings page
+         * that leaves the status it just changed reading the old value for
+         * minutes looks broken, so re-roll immediately. Failures are swallowed:
+         * the sweep is still the backstop, and a rollup must not fail the save.
+         */
+        const policyKeys = [
+            "healthRollupPolicy",
+            "offlineThresholdPercent",
+        ];
+        const touchesPolicy = policyKeys.some((key) => {
+            return (onUpdate.updateBy.data[key] !== undefined);
+        });
+        if (touchesPolicy) {
+            for (const siteId of updatedItemIds) {
+                try {
+                    await this.recomputeRollupForSite(siteId);
+                }
+                catch (error) {
+                    logger.error(`NetworkSiteService.onUpdateSuccess: rollup after a policy change failed for site ${siteId.toString()}: ${error}`, {
+                        siteId: siteId.toString(),
+                    });
+                }
+            }
+        }
         const parentChange = onUpdate.carryForward || null;
         if (!parentChange) {
             return onUpdate;
@@ -579,16 +670,94 @@ export class Service extends DatabaseService {
             return descendant.id;
         });
     }
+    /*
+     * Every site id in the subtrees rooted at `siteIds`, INCLUDING the roots
+     * themselves.
+     *
+     * Two statements regardless of how many roots are passed: one to read the
+     * roots' materialized paths, one prefix scan OR-ing those paths together.
+     * The obvious loop over getDescendantSiteIds is two statements PER root,
+     * and the caller that needs this - expanding a maintenance window attached
+     * to a Region into the units it covers - can legitimately be handed
+     * hundreds of roots.
+     *
+     * Roots whose row is missing or whose path is unset contribute only
+     * themselves: a site with no path has no discoverable subtree, and
+     * silently dropping it would quietly un-cover a maintenance window.
+     */
+    async getSubtreeSiteIds(data) {
+        const result = new Set();
+        for (const siteId of data.siteIds) {
+            result.add(siteId.toString());
+        }
+        if (result.size === 0) {
+            return result;
+        }
+        const roots = await this.findBy({
+            query: {
+                projectId: data.projectId,
+                _id: QueryHelper.any(data.siteIds),
+            },
+            select: {
+                _id: true,
+                materializedPath: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        const paths = [];
+        for (const root of roots) {
+            if (root.materializedPath) {
+                paths.push(root.materializedPath);
+            }
+        }
+        if (paths.length === 0) {
+            return result;
+        }
+        const descendants = await this.findBy({
+            query: {
+                projectId: data.projectId,
+                materializedPath: this.pathStartsWithAny(paths),
+            },
+            select: {
+                _id: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: {
+                isRoot: true,
+            },
+        });
+        for (const descendant of descendants) {
+            if (descendant.id) {
+                result.add(descendant.id.toString());
+            }
+        }
+        return result;
+    }
     /*
      * ------------------------------------------------------------------
      * Persisted rollup engine
      * ------------------------------------------------------------------
      */
     /*
-     * Recomputes the worst-of rollup for one site from the devices in its
-     * subtree, persisting currentMonitorStatusId + lastRollupAt and keeping
-     * the NetworkSiteStatusTimeline in sync (close the open row, open a new
-     * one) whenever the status actually changes.
+     * Recomputes the rollup for one site from the devices in its subtree,
+     * persisting currentMonitorStatusId + lastRollupAt and keeping the
+     * NetworkSiteStatusTimeline in sync (close the open row, open a new one)
+     * whenever the status actually changes.
+     *
+     * WHICH rule turns those devices into one status is the site's own
+     * healthRollupPolicy - worst-of by default, or a share-of-devices-down
+     * threshold. See Types/NetworkSite/SiteHealthRollupPolicy.
+     *
+     * Descendants inside an ongoing scheduled maintenance window are dropped
+     * from the subtree first, so planned work on one unit does not turn its
+     * region red. The maintained site's OWN rollup keeps every device,
+     * including its own: someone looking at that unit must still see it is
+     * down. See NetworkSiteMaintenanceSuppression.
      */
     async recomputeRollupForSite(siteId) {
         var _a;
@@ -602,6 +771,8 @@ export class Service extends DatabaseService {
                 shouldAlertWhenUnhealthy: true,
                 alertSeverityId: true,
                 currentActiveAlertId: true,
+                healthRollupPolicy: true,
+                offlineThresholdPercent: true,
             },
             props: {
                 isRoot: true,
@@ -615,38 +786,70 @@ export class Service extends DatabaseService {
             ...(await this.getDescendantSiteIds(site.id, site.projectId)),
         ];
         /*
+         * Sites silenced by an ongoing maintenance window. A site that is itself
+         * inside one suppresses nothing - not even its own descendants - because
+         * its rollup is supposed to show the planned outage. Only an ancestor
+         * looking down past a maintained subtree drops it.
+         */
+        const maintainedSiteIds = await NetworkSiteMaintenanceSuppression.getSiteIdsUnderOngoingMaintenance(site.projectId);
+        const isSiteUnderMaintenance = maintainedSiteIds.has(site.id.toString());
+        const suppressesDescendants = !isSiteUnderMaintenance && maintainedSiteIds.size > 0;
+        const contributingSiteIds = suppressesDescendants
+            ? subtreeSiteIds.filter((id) => {
+                return !maintainedSiteIds.has(id.toString());
+            })
+            : subtreeSiteIds;
+        /*
+         * The suppressed part of THIS subtree. Their devices do not vote, but
+         * they are still counted, because a share needs a denominator that does
+         * not move when a window opens - see SiteStatusRollupUtil.
+         * deviceHealthShare for what goes wrong otherwise.
+         */
+        const suppressedSiteIds = suppressesDescendants
+            ? subtreeSiteIds.filter((id) => {
+                return maintainedSiteIds.has(id.toString());
+            })
+            : [];
+        const now = OneUptimeDate.getCurrentDate();
+        /*
+         * The subtree's devices, as health BUCKETS rather than rows.
+         *
+         * This used to read the devices themselves, capped at LIMIT_MAX. A
+         * franchise estate whose root site has more than ten thousand devices
+         * under it therefore rolled up from an arbitrary ten-thousand-row sample:
+         * the one dark switch in store 12,000 could not turn its region red, and
+         * nothing said so. Bucketing runs over the whole subtree however large it
+         * is, and returns a handful of rows to classify.
+         *
          * Archived devices are decommissioned: they keep their siteId but must
          * not vote in the rollup. An archived, never-monitored device otherwise
          * falls through to the freshness fallback (stale lastSeenAt -> Offline)
          * and pins its whole ancestor chain red forever, with the drill-down
          * showing zero devices because that query excludes archived rows.
          */
-        const devices = await NetworkDeviceService.findBy({
-            query: {
-                projectId: site.projectId,
-                siteId: QueryHelper.any(subtreeSiteIds),
-                isArchived: false,
-            },
-            select: {
-                _id: true,
-                currentMonitorStatusId: true,
-                /*
-                 * All four reachability inputs. isReachable is the one that decides
-                 * up/down; the other three only size the "polling has stopped
-                 * entirely" backstop. Selecting lastSeenAt alone silently drops the
-                 * rollup back to the old freshness rule.
-                 */
-                isReachable: true,
-                lastPolledAt: true,
-                lastSeenAt: true,
-                pollingIntervalInMinutes: true,
-            },
-            limit: LIMIT_MAX,
-            skip: 0,
+        const deviceGroups = await NetworkDeviceService.getHealthGroupsForSites({
+            projectId: site.projectId,
+            siteIds: contributingSiteIds,
+            now: now,
             props: {
                 isRoot: true,
             },
         });
+        /*
+         * One extra aggregate, and only while a window is actually running: the
+         * common path (no ongoing maintenance anywhere in the project) skips it
+         * entirely. Classified below, once the status ladder is known.
+         */
+        const suppressedGroups = suppressedSiteIds.length > 0
+            ? await NetworkDeviceService.getHealthGroupsForSites({
+                projectId: site.projectId,
+                siteIds: suppressedSiteIds,
+                now: now,
+                props: {
+                    isRoot: true,
+                },
+            })
+            : [];
         const statuses = await MonitorStatusService.findBy({
             query: {
                 projectId: site.projectId,
@@ -665,13 +868,22 @@ export class Service extends DatabaseService {
             },
         });
         const priorityByStatusId = new Map();
+        const isOperationalByStatusId = new Map();
         let operationalStatus = null;
         let offlineStatus = null;
+        /*
+         * The rung between the two, for the threshold policy: the WORST status
+         * that is neither operational nor offline (highest priority wins, the
+         * same direction worst-of reads the ladder). Projects that never created
+         * one leave this null and "some devices down" falls through to offline.
+         */
+        let degradedStatus = null;
         for (const status of statuses) {
             if (!status.id || typeof status.priority !== "number") {
                 continue;
             }
             priorityByStatusId.set(status.id.toString(), status.priority);
+            isOperationalByStatusId.set(status.id.toString(), Boolean(status.isOperationalState));
             if (status.isOperationalState && !operationalStatus) {
                 operationalStatus = {
                     monitorStatusId: status.id.toString(),
@@ -684,30 +896,96 @@ export class Service extends DatabaseService {
                     priority: status.priority,
                 };
             }
+            if (!status.isOperationalState &&
+                !status.isOfflineState &&
+                (!degradedStatus || status.priority > degradedStatus.priority)) {
+                degradedStatus = {
+                    monitorStatusId: status.id.toString(),
+                    priority: status.priority,
+                };
+            }
         }
-        const deviceStates = devices.map((device) => {
-            var _a;
-            const statusId = (_a = device.currentMonitorStatusId) === null || _a === void 0 ? void 0 : _a.toString();
-            return {
-                currentMonitorStatusId: statusId,
-                monitorStatusPriority: statusId
-                    ? priorityByStatusId.get(statusId)
+        /*
+         * The middle rung has to sit BELOW the offline rung, or the ladder is
+         * upside down. A project is free to define a status that is neither
+         * operational nor offline and give it a priority ABOVE its offline row
+         * ("Critical", say, at priority 4 next to Offline at 3) — and then a
+         * sub-threshold outage would stamp the WORSE status while crossing the
+         * threshold stamped the milder one. Nothing stops a project doing that,
+         * so the rollup has to.
+         */
+        if (degradedStatus &&
+            offlineStatus &&
+            degradedStatus.priority >= offlineStatus.priority) {
+            let milder = null;
+            for (const status of statuses) {
+                if (!status.id ||
+                    typeof status.priority !== "number" ||
+                    status.isOperationalState ||
+                    status.isOfflineState ||
+                    status.priority >= offlineStatus.priority) {
+                    continue;
+                }
+                if (!milder || status.priority > milder.priority) {
+                    milder = {
+                        monitorStatusId: status.id.toString(),
+                        priority: status.priority,
+                    };
+                }
+            }
+            degradedStatus = milder;
+        }
+        const deviceStates = deviceGroups.map((group) => {
+            return deviceRollupStateForGroup({
+                group: group,
+                monitorStatusPriority: group.monitorStatusId
+                    ? priorityByStatusId.get(group.monitorStatusId)
                     : undefined,
-                isReachable: device.isReachable,
-                lastPolledAt: device.lastPolledAt,
-                lastSeenAt: device.lastSeenAt,
-                pollingIntervalInMinutes: device.pollingIntervalInMinutes,
-            };
+                monitorStatusIsOperational: group.monitorStatusId
+                    ? isOperationalByStatusId.get(group.monitorStatusId)
+                    : undefined,
+                now: now,
+            });
         });
-        const worstStatusId = SiteStatusRollupUtil.worstStatus({
+        /*
+         * Sized with SiteStatusRollupUtil's own rule, not by summing
+         * `deviceCount`.
+         *
+         * A suppressed bucket of never-polled devices has a count like any
+         * other, but the share deliberately drops never-reported devices from
+         * BOTH sides of the fraction. Adding their raw count back as
+         * "suppressed" would pad the denominator with devices nothing was ever
+         * measuring — a region with 200 undiscovered devices under a window and
+         * 6 of 10 real ones dark would read 2.9% and call itself Degraded while
+         * more than half of everything that has ever answered is offline.
+         */
+        const suppressedDeviceCount = SiteStatusRollupUtil.reportingDeviceCount(suppressedGroups.map((group) => {
+            return deviceRollupStateForGroup({
+                group: group,
+                monitorStatusPriority: group.monitorStatusId
+                    ? priorityByStatusId.get(group.monitorStatusId)
+                    : undefined,
+                monitorStatusIsOperational: group.monitorStatusId
+                    ? isOperationalByStatusId.get(group.monitorStatusId)
+                    : undefined,
+                now: now,
+            });
+        }), now);
+        const rolledUpStatusId = SiteStatusRollupUtil.rollupStatus({
+            policy: parseSiteHealthRollupPolicy(site.healthRollupPolicy),
             deviceStates: deviceStates,
-            operationalStatus: operationalStatus,
-            offlineStatus: offlineStatus,
+            ladder: {
+                operationalStatus: operationalStatus,
+                degradedStatus: degradedStatus,
+                offlineStatus: offlineStatus,
+            },
+            offlineThresholdPercent: site.offlineThresholdPercent,
+            suppressedDeviceCount: suppressedDeviceCount,
+            now: now,
         });
-        const now = OneUptimeDate.getCurrentDate();
         const currentStatusId = ((_a = site.currentMonitorStatusId) === null || _a === void 0 ? void 0 : _a.toString()) || null;
         // No devices contribute -> leave the status alone, just stamp the run.
-        if (!worstStatusId || worstStatusId === currentStatusId) {
+        if (!rolledUpStatusId || rolledUpStatusId === currentStatusId) {
             await this.updateColumnsByIdWithoutHooks({
                 id: site.id,
                 data: {
@@ -719,7 +997,7 @@ export class Service extends DatabaseService {
         await this.updateColumnsByIdWithoutHooks({
             id: site.id,
             data: {
-                currentMonitorStatusId: new ObjectID(worstStatusId),
+                currentMonitorStatusId: new ObjectID(rolledUpStatusId),
                 lastRollupAt: now,
             },
         });
@@ -741,7 +1019,7 @@ export class Service extends DatabaseService {
         const timeline = new NetworkSiteStatusTimeline();
         timeline.projectId = site.projectId;
         timeline.siteId = site.id;
-        timeline.monitorStatusId = new ObjectID(worstStatusId);
+        timeline.monitorStatusId = new ObjectID(rolledUpStatusId);
         timeline.startsAt = now;
         await NetworkSiteStatusTimelineService.create({
             data: timeline,
@@ -758,7 +1036,7 @@ export class Service extends DatabaseService {
                 site: site,
                 newStatus: statuses.find((status) => {
                     var _a;
-                    return ((_a = status.id) === null || _a === void 0 ? void 0 : _a.toString()) === worstStatusId;
+                    return ((_a = status.id) === null || _a === void 0 ? void 0 : _a.toString()) === rolledUpStatusId;
                 }) || null,
             });
         }
@@ -833,7 +1111,7 @@ export class Service extends DatabaseService {
         const alert = new Alert();
         alert.projectId = site.projectId;
         alert.title = `Network site ${site.name || site.id.toString()} is ${statusName}`;
-        alert.description = `The health rollup of network site **${site.name || site.id.toString()}** changed to **${statusName}** — the worst status of the devices at this site and every site below it. This alert auto-resolves when the site rolls back up to an operational status.`;
+        alert.description = `The health rollup of network site **${site.name || site.id.toString()}** changed to **${statusName}**, rolled up from the devices at this site and every site below it. This alert auto-resolves when the site rolls back up to an operational status.`;
         alert.alertSeverityId = alertSeverityId;
         alert.rootCause = `Network site **${site.name || site.id.toString()}** rolled up to **${statusName}**.`;
         const createdAlert = await AlertService.create({
@@ -894,6 +1172,71 @@ export class Service extends DatabaseService {
             await this.recomputeRollupForSite(ancestorIds[i]);
         }
     }
+    /*
+     * Re-rolls the chains a maintenance window just started or stopped
+     * covering.
+     *
+     * Only the attached sites and their ANCESTORS can have changed verdict:
+     * the maintained subtree keeps every one of its own devices either way
+     * (see recomputeRollupForSite), so nothing below an attached site needs
+     * recomputing. Each chain is deduplicated, because attaching a Region and
+     * one of its Markets to the same window would otherwise walk the shared
+     * ancestors twice.
+     *
+     * Errors are swallowed per site: the five-minute stale-rollup sweep is the
+     * backstop, and a maintenance event must not fail to start because one
+     * site's rollup did.
+     */
+    async recomputeRollupsAfterMaintenanceChange(data) {
+        if (data.siteIds.length === 0) {
+            return;
+        }
+        /*
+         * The suppression set is cached for a few seconds; a window that has
+         * just flipped state must not be recomputed against the previous
+         * answer.
+         */
+        NetworkSiteMaintenanceSuppression.invalidateCache(data.projectId);
+        const recomputed = new Set();
+        /*
+         * Descendants of an attached site change verdict too, which is easy to
+         * miss because the attached site itself does not. While the window runs,
+         * a descendant D is maintained (coverage is inherited downward), so D
+         * suppresses nothing of its own. The moment the window ends D stops
+         * being maintained and starts suppressing ITS maintained descendants —
+         * a different answer, from the same devices. Nested and overlapping
+         * windows are ordinary on a franchise estate, so this is not a corner.
+         */
+        const subtreeIds = await this.getSubtreeSiteIds({
+            siteIds: data.siteIds,
+            projectId: data.projectId,
+        });
+        const roots = Array.from(subtreeIds).map((id) => {
+            return new ObjectID(id);
+        });
+        for (const siteId of roots) {
+            const chain = [
+                siteId,
+                ...(await this.getAncestorIds(siteId)).reverse(),
+            ];
+            for (const chainSiteId of chain) {
+                const key = chainSiteId.toString();
+                if (recomputed.has(key)) {
+                    continue;
+                }
+                recomputed.add(key);
+                try {
+                    await this.recomputeRollupForSite(chainSiteId);
+                }
+                catch (error) {
+                    logger.error(`NetworkSiteService.recomputeRollupsAfterMaintenanceChange: rollup failed for site ${key}: ${error}`, {
+                        projectId: data.projectId.toString(),
+                        siteId: key,
+                    });
+                }
+            }
+        }
+    }
     /*
      * ------------------------------------------------------------------
      * Monitor status bridge
@@ -1019,6 +1362,12 @@ export class Service extends DatabaseService {
         }
     }
 }
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getStatusCounts", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),
@@ -1073,6 +1422,12 @@ __decorate([
     __metadata("design:paramtypes", [ObjectID, Object]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "getDescendantSiteIds", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "getSubtreeSiteIds", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),
@@ -1091,6 +1446,12 @@ __decorate([
     __metadata("design:paramtypes", [ObjectID]),
     __metadata("design:returntype", Promise)
 ], Service.prototype, "recomputeRollupForSiteAndAncestors", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], Service.prototype, "recomputeRollupsAfterMaintenanceChange", null);
 __decorate([
     CaptureSpan(),
     __metadata("design:type", Function),