@oneuptime/common 12.0.25 → 12.0.27

package/Server/Services/NetworkSiteService.ts

@@ -40,6 +40,23 @@ import SiteStatusRollupUtil, {
   DeviceHealthState,
   RollupStatusOption,
 } from "../../Utils/NetworkSite/SiteStatusRollupUtil";
+import { parseSiteHealthRollupPolicy } from "../../Types/NetworkSite/SiteHealthRollupPolicy";
+import NetworkSiteMaintenanceSuppression from "../Utils/NetworkSite/NetworkSiteMaintenanceSuppression";
+import { AggregateRow } from "../Types/Database/AggregateBy";
+import AggregateResultUtil from "../Types/Database/AggregateResultUtil";
+import {
+  DeviceHealthGroup,
+  deviceRollupStateForGroup,
+} from "../Utils/NetworkDevice/DeviceHealthAggregation";
+
+/**
+ * How many sites are stamped with each MonitorStatus. `monitorStatusId` is
+ * null for the bucket of sites that have no rollup yet.
+ */
+export interface SiteStatusCount {
+  monitorStatusId: string | null;
+  siteCount: number;
+}
 
 /*
  * Both spellings of "this site's parent" in a write payload: the dashboard's
@@ -74,6 +91,52 @@ export class Service extends DatabaseService<Model> {
     super(Model);
   }
 
+  /**
+   * How many sites sit under each rolled-up MonitorStatus, counted in the
+   * database.
+   *
+   * One row per status the project actually uses (plus one for "no rollup
+   * yet"), rather than every site row shipped to a browser to be tallied
+   * there. Which of those statuses count as UNHEALTHY is deliberately not
+   * decided here: `isOperationalState` lives on MonitorStatus, the caller
+   * already holds those rows, and duplicating the flag into this query would
+   * be a second place for it to be read wrongly.
+   */
+  @CaptureSpan()
+  public async getStatusCounts(data: {
+    projectId: ObjectID;
+    props: DatabaseCommonInteractionProps;
+  }): Promise<Array<SiteStatusCount>> {
+    const rows: Array<AggregateRow> = await this.aggregateBy({
+      query: {
+        projectId: data.projectId,
+      },
+      groupBy: [
+        {
+          expression: `"NetworkSite"."currentMonitorStatusId"`,
+          alias: "monitorStatusId",
+        },
+      ],
+      select: [
+        {
+          expression: `COUNT(*)`,
+          alias: "siteCount",
+        },
+      ],
+      props: data.props,
+    });
+
+    return rows.map((row: AggregateRow): SiteStatusCount => {
+      return {
+        monitorStatusId: AggregateResultUtil.toStringOrNull(
+          row,
+          "monitorStatusId",
+        ),
+        siteCount: AggregateResultUtil.toNumber(row, "siteCount"),
+      };
+    });
+  }
+
   /*
    * ------------------------------------------------------------------
    * Hierarchy maintenance (materializedPath + depth)
@@ -122,6 +185,32 @@ export class Service extends DatabaseService<Model> {
     );
   }
 
+  /*
+   * The same prefix predicate for SEVERAL paths at once, OR-ed together, so
+   * "every site under any of these" is one statement rather than one per
+   * root. Each prefix keeps its own bound parameter; an empty list would
+   * produce `()`, which is a syntax error, so callers must not reach here
+   * with one (getSubtreeSiteIds guards it).
+   */
+  private pathStartsWithAny(paths: Array<string>): FindWhereProperty<any> {
+    const parameters: Record<string, string> = {};
+    const names: Array<string> = [];
+
+    for (const path of paths) {
+      const rid: string = Text.generateRandomText(10);
+      parameters[rid] = `${path}%`;
+      names.push(rid);
+    }
+
+    return Raw((alias: string) => {
+      return `(${names
+        .map((name: string) => {
+          return `${alias} LIKE :${name}`;
+        })
+        .join(" OR ")})`;
+    }, parameters);
+  }
+
   /*
    * A stored path is trustworthy only when it agrees with parentSiteId: it
    * must end with the site's own id, and the segment before it must be the
@@ -353,6 +442,39 @@ export class Service extends DatabaseService<Model> {
     onUpdate: OnUpdate<Model>,
     updatedItemIds: Array<ObjectID>,
   ): Promise<OnUpdate<Model>> {
+    /*
+     * Editing the rollup POLICY (or its threshold) changes what the site's
+     * existing devices add up to, without touching a device or the tree. The
+     * five-minute stale sweep would eventually notice, but a settings page
+     * that leaves the status it just changed reading the old value for
+     * minutes looks broken, so re-roll immediately. Failures are swallowed:
+     * the sweep is still the backstop, and a rollup must not fail the save.
+     */
+    const policyKeys: Array<string> = [
+      "healthRollupPolicy",
+      "offlineThresholdPercent",
+    ];
+    const touchesPolicy: boolean = policyKeys.some((key: string): boolean => {
+      return (
+        (onUpdate.updateBy.data as Record<string, unknown>)[key] !== undefined
+      );
+    });
+
+    if (touchesPolicy) {
+      for (const siteId of updatedItemIds) {
+        try {
+          await this.recomputeRollupForSite(siteId);
+        } catch (error) {
+          logger.error(
+            `NetworkSiteService.onUpdateSuccess: rollup after a policy change failed for site ${siteId.toString()}: ${error}`,
+            {
+              siteId: siteId.toString(),
+            } as LogAttributes,
+          );
+        }
+      }
+    }
+
     const parentChange: ParentChangeCarryForward | null =
       (onUpdate.carryForward as ParentChangeCarryForward | null) || null;
 
@@ -794,6 +916,88 @@ export class Service extends DatabaseService<Model> {
       });
   }
 
+  /*
+   * Every site id in the subtrees rooted at `siteIds`, INCLUDING the roots
+   * themselves.
+   *
+   * Two statements regardless of how many roots are passed: one to read the
+   * roots' materialized paths, one prefix scan OR-ing those paths together.
+   * The obvious loop over getDescendantSiteIds is two statements PER root,
+   * and the caller that needs this - expanding a maintenance window attached
+   * to a Region into the units it covers - can legitimately be handed
+   * hundreds of roots.
+   *
+   * Roots whose row is missing or whose path is unset contribute only
+   * themselves: a site with no path has no discoverable subtree, and
+   * silently dropping it would quietly un-cover a maintenance window.
+   */
+  @CaptureSpan()
+  public async getSubtreeSiteIds(data: {
+    siteIds: Array<ObjectID>;
+    projectId: ObjectID;
+  }): Promise<Set<string>> {
+    const result: Set<string> = new Set<string>();
+
+    for (const siteId of data.siteIds) {
+      result.add(siteId.toString());
+    }
+
+    if (result.size === 0) {
+      return result;
+    }
+
+    const roots: Array<Model> = await this.findBy({
+      query: {
+        projectId: data.projectId,
+        _id: QueryHelper.any(data.siteIds),
+      },
+      select: {
+        _id: true,
+        materializedPath: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    const paths: Array<string> = [];
+
+    for (const root of roots) {
+      if (root.materializedPath) {
+        paths.push(root.materializedPath);
+      }
+    }
+
+    if (paths.length === 0) {
+      return result;
+    }
+
+    const descendants: Array<Model> = await this.findBy({
+      query: {
+        projectId: data.projectId,
+        materializedPath: this.pathStartsWithAny(paths),
+      },
+      select: {
+        _id: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    for (const descendant of descendants) {
+      if (descendant.id) {
+        result.add(descendant.id.toString());
+      }
+    }
+
+    return result;
+  }
+
   /*
    * ------------------------------------------------------------------
    * Persisted rollup engine
@@ -801,10 +1005,20 @@ export class Service extends DatabaseService<Model> {
    */
 
   /*
-   * Recomputes the worst-of rollup for one site from the devices in its
-   * subtree, persisting currentMonitorStatusId + lastRollupAt and keeping
-   * the NetworkSiteStatusTimeline in sync (close the open row, open a new
-   * one) whenever the status actually changes.
+   * Recomputes the rollup for one site from the devices in its subtree,
+   * persisting currentMonitorStatusId + lastRollupAt and keeping the
+   * NetworkSiteStatusTimeline in sync (close the open row, open a new one)
+   * whenever the status actually changes.
+   *
+   * WHICH rule turns those devices into one status is the site's own
+   * healthRollupPolicy - worst-of by default, or a share-of-devices-down
+   * threshold. See Types/NetworkSite/SiteHealthRollupPolicy.
+   *
+   * Descendants inside an ongoing scheduled maintenance window are dropped
+   * from the subtree first, so planned work on one unit does not turn its
+   * region red. The maintained site's OWN rollup keeps every device,
+   * including its own: someone looking at that unit must still see it is
+   * down. See NetworkSiteMaintenanceSuppression.
    */
   @CaptureSpan()
   public async recomputeRollupForSite(siteId: ObjectID): Promise<void> {
@@ -818,6 +1032,8 @@ export class Service extends DatabaseService<Model> {
         shouldAlertWhenUnhealthy: true,
         alertSeverityId: true,
         currentActiveAlertId: true,
+        healthRollupPolicy: true,
+        offlineThresholdPercent: true,
       },
       props: {
         isRoot: true,
@@ -834,38 +1050,85 @@ export class Service extends DatabaseService<Model> {
     ];
 
     /*
+     * Sites silenced by an ongoing maintenance window. A site that is itself
+     * inside one suppresses nothing - not even its own descendants - because
+     * its rollup is supposed to show the planned outage. Only an ancestor
+     * looking down past a maintained subtree drops it.
+     */
+    const maintainedSiteIds: Set<string> =
+      await NetworkSiteMaintenanceSuppression.getSiteIdsUnderOngoingMaintenance(
+        site.projectId,
+      );
+
+    const isSiteUnderMaintenance: boolean = maintainedSiteIds.has(
+      site.id.toString(),
+    );
+
+    const suppressesDescendants: boolean =
+      !isSiteUnderMaintenance && maintainedSiteIds.size > 0;
+
+    const contributingSiteIds: Array<ObjectID> = suppressesDescendants
+      ? subtreeSiteIds.filter((id: ObjectID) => {
+          return !maintainedSiteIds.has(id.toString());
+        })
+      : subtreeSiteIds;
+
+    /*
+     * The suppressed part of THIS subtree. Their devices do not vote, but
+     * they are still counted, because a share needs a denominator that does
+     * not move when a window opens - see SiteStatusRollupUtil.
+     * deviceHealthShare for what goes wrong otherwise.
+     */
+    const suppressedSiteIds: Array<ObjectID> = suppressesDescendants
+      ? subtreeSiteIds.filter((id: ObjectID) => {
+          return maintainedSiteIds.has(id.toString());
+        })
+      : [];
+
+    const now: Date = OneUptimeDate.getCurrentDate();
+
+    /*
+     * The subtree's devices, as health BUCKETS rather than rows.
+     *
+     * This used to read the devices themselves, capped at LIMIT_MAX. A
+     * franchise estate whose root site has more than ten thousand devices
+     * under it therefore rolled up from an arbitrary ten-thousand-row sample:
+     * the one dark switch in store 12,000 could not turn its region red, and
+     * nothing said so. Bucketing runs over the whole subtree however large it
+     * is, and returns a handful of rows to classify.
+     *
      * Archived devices are decommissioned: they keep their siteId but must
      * not vote in the rollup. An archived, never-monitored device otherwise
      * falls through to the freshness fallback (stale lastSeenAt -> Offline)
      * and pins its whole ancestor chain red forever, with the drill-down
      * showing zero devices because that query excludes archived rows.
      */
-    const devices: Array<NetworkDevice> = await NetworkDeviceService.findBy({
-      query: {
+    const deviceGroups: Array<DeviceHealthGroup> =
+      await NetworkDeviceService.getHealthGroupsForSites({
         projectId: site.projectId,
-        siteId: QueryHelper.any(subtreeSiteIds),
-        isArchived: false,
-      },
-      select: {
-        _id: true,
-        currentMonitorStatusId: true,
-        /*
-         * All four reachability inputs. isReachable is the one that decides
-         * up/down; the other three only size the "polling has stopped
-         * entirely" backstop. Selecting lastSeenAt alone silently drops the
-         * rollup back to the old freshness rule.
-         */
-        isReachable: true,
-        lastPolledAt: true,
-        lastSeenAt: true,
-        pollingIntervalInMinutes: true,
-      },
-      limit: LIMIT_MAX,
-      skip: 0,
-      props: {
-        isRoot: true,
-      },
-    });
+        siteIds: contributingSiteIds,
+        now: now,
+        props: {
+          isRoot: true,
+        },
+      });
+
+    /*
+     * One extra aggregate, and only while a window is actually running: the
+     * common path (no ongoing maintenance anywhere in the project) skips it
+     * entirely. Classified below, once the status ladder is known.
+     */
+    const suppressedGroups: Array<DeviceHealthGroup> =
+      suppressedSiteIds.length > 0
+        ? await NetworkDeviceService.getHealthGroupsForSites({
+            projectId: site.projectId,
+            siteIds: suppressedSiteIds,
+            now: now,
+            props: {
+              isRoot: true,
+            },
+          })
+        : [];
 
     const statuses: Array<MonitorStatus> = await MonitorStatusService.findBy({
       query: {
@@ -886,14 +1149,26 @@ export class Service extends DatabaseService<Model> {
     });
 
     const priorityByStatusId: Map<string, number> = new Map();
+    const isOperationalByStatusId: Map<string, boolean> = new Map();
     let operationalStatus: RollupStatusOption | null = null;
     let offlineStatus: RollupStatusOption | null = null;
+    /*
+     * The rung between the two, for the threshold policy: the WORST status
+     * that is neither operational nor offline (highest priority wins, the
+     * same direction worst-of reads the ladder). Projects that never created
+     * one leave this null and "some devices down" falls through to offline.
+     */
+    let degradedStatus: RollupStatusOption | null = null;
 
     for (const status of statuses) {
       if (!status.id || typeof status.priority !== "number") {
         continue;
       }
       priorityByStatusId.set(status.id.toString(), status.priority);
+      isOperationalByStatusId.set(
+        status.id.toString(),
+        Boolean(status.isOperationalState),
+      );
       if (status.isOperationalState && !operationalStatus) {
         operationalStatus = {
           monitorStatusId: status.id.toString(),
@@ -906,37 +1181,114 @@ export class Service extends DatabaseService<Model> {
           priority: status.priority,
         };
       }
+      if (
+        !status.isOperationalState &&
+        !status.isOfflineState &&
+        (!degradedStatus || status.priority > degradedStatus.priority)
+      ) {
+        degradedStatus = {
+          monitorStatusId: status.id.toString(),
+          priority: status.priority,
+        };
+      }
+    }
+
+    /*
+     * The middle rung has to sit BELOW the offline rung, or the ladder is
+     * upside down. A project is free to define a status that is neither
+     * operational nor offline and give it a priority ABOVE its offline row
+     * ("Critical", say, at priority 4 next to Offline at 3) — and then a
+     * sub-threshold outage would stamp the WORSE status while crossing the
+     * threshold stamped the milder one. Nothing stops a project doing that,
+     * so the rollup has to.
+     */
+    if (
+      degradedStatus &&
+      offlineStatus &&
+      degradedStatus.priority >= offlineStatus.priority
+    ) {
+      let milder: RollupStatusOption | null = null;
+      for (const status of statuses) {
+        if (
+          !status.id ||
+          typeof status.priority !== "number" ||
+          status.isOperationalState ||
+          status.isOfflineState ||
+          status.priority >= offlineStatus.priority
+        ) {
+          continue;
+        }
+        if (!milder || status.priority > milder.priority) {
+          milder = {
+            monitorStatusId: status.id.toString(),
+            priority: status.priority,
+          };
+        }
+      }
+      degradedStatus = milder;
     }
 
-    const deviceStates: Array<DeviceHealthState> = devices.map(
-      (device: NetworkDevice) => {
-        const statusId: string | undefined =
-          device.currentMonitorStatusId?.toString();
-        return {
-          currentMonitorStatusId: statusId,
-          monitorStatusPriority: statusId
-            ? priorityByStatusId.get(statusId)
+    const deviceStates: Array<DeviceHealthState> = deviceGroups.map(
+      (group: DeviceHealthGroup) => {
+        return deviceRollupStateForGroup({
+          group: group,
+          monitorStatusPriority: group.monitorStatusId
+            ? priorityByStatusId.get(group.monitorStatusId)
             : undefined,
-          isReachable: device.isReachable,
-          lastPolledAt: device.lastPolledAt,
-          lastSeenAt: device.lastSeenAt,
-          pollingIntervalInMinutes: device.pollingIntervalInMinutes,
-        };
+          monitorStatusIsOperational: group.monitorStatusId
+            ? isOperationalByStatusId.get(group.monitorStatusId)
+            : undefined,
+          now: now,
+        });
       },
     );
 
-    const worstStatusId: string | null = SiteStatusRollupUtil.worstStatus({
+    /*
+     * Sized with SiteStatusRollupUtil's own rule, not by summing
+     * `deviceCount`.
+     *
+     * A suppressed bucket of never-polled devices has a count like any
+     * other, but the share deliberately drops never-reported devices from
+     * BOTH sides of the fraction. Adding their raw count back as
+     * "suppressed" would pad the denominator with devices nothing was ever
+     * measuring — a region with 200 undiscovered devices under a window and
+     * 6 of 10 real ones dark would read 2.9% and call itself Degraded while
+     * more than half of everything that has ever answered is offline.
+     */
+    const suppressedDeviceCount: number =
+      SiteStatusRollupUtil.reportingDeviceCount(
+        suppressedGroups.map((group: DeviceHealthGroup) => {
+          return deviceRollupStateForGroup({
+            group: group,
+            monitorStatusPriority: group.monitorStatusId
+              ? priorityByStatusId.get(group.monitorStatusId)
+              : undefined,
+            monitorStatusIsOperational: group.monitorStatusId
+              ? isOperationalByStatusId.get(group.monitorStatusId)
+              : undefined,
+            now: now,
+          });
+        }),
+        now,
+      );
+
+    const rolledUpStatusId: string | null = SiteStatusRollupUtil.rollupStatus({
+      policy: parseSiteHealthRollupPolicy(site.healthRollupPolicy),
       deviceStates: deviceStates,
-      operationalStatus: operationalStatus,
-      offlineStatus: offlineStatus,
+      ladder: {
+        operationalStatus: operationalStatus,
+        degradedStatus: degradedStatus,
+        offlineStatus: offlineStatus,
+      },
+      offlineThresholdPercent: site.offlineThresholdPercent,
+      suppressedDeviceCount: suppressedDeviceCount,
+      now: now,
     });
-
-    const now: Date = OneUptimeDate.getCurrentDate();
     const currentStatusId: string | null =
       site.currentMonitorStatusId?.toString() || null;
 
     // No devices contribute -> leave the status alone, just stamp the run.
-    if (!worstStatusId || worstStatusId === currentStatusId) {
+    if (!rolledUpStatusId || rolledUpStatusId === currentStatusId) {
       await this.updateColumnsByIdWithoutHooks({
         id: site.id,
         data: {
@@ -949,7 +1301,7 @@ export class Service extends DatabaseService<Model> {
     await this.updateColumnsByIdWithoutHooks({
       id: site.id,
       data: {
-        currentMonitorStatusId: new ObjectID(worstStatusId),
+        currentMonitorStatusId: new ObjectID(rolledUpStatusId),
         lastRollupAt: now,
       },
     });
@@ -973,7 +1325,7 @@ export class Service extends DatabaseService<Model> {
     const timeline: NetworkSiteStatusTimeline = new NetworkSiteStatusTimeline();
     timeline.projectId = site.projectId;
     timeline.siteId = site.id;
-    timeline.monitorStatusId = new ObjectID(worstStatusId);
+    timeline.monitorStatusId = new ObjectID(rolledUpStatusId);
     timeline.startsAt = now;
 
     await NetworkSiteStatusTimelineService.create({
@@ -992,7 +1344,7 @@ export class Service extends DatabaseService<Model> {
         site: site,
         newStatus:
           statuses.find((status: MonitorStatus) => {
-            return status.id?.toString() === worstStatusId;
+            return status.id?.toString() === rolledUpStatusId;
           }) || null,
       });
     } catch (err) {
@@ -1095,7 +1447,7 @@ export class Service extends DatabaseService<Model> {
     alert.title = `Network site ${site.name || site.id.toString()} is ${statusName}`;
     alert.description = `The health rollup of network site **${
       site.name || site.id.toString()
-    }** changed to **${statusName}** — the worst status of the devices at this site and every site below it. This alert auto-resolves when the site rolls back up to an operational status.`;
+    }** changed to **${statusName}**, rolled up from the devices at this site and every site below it. This alert auto-resolves when the site rolls back up to an operational status.`;
     alert.alertSeverityId = alertSeverityId;
     alert.rootCause = `Network site **${site.name || site.id.toString()}** rolled up to **${statusName}**.`;
 
@@ -1177,6 +1529,87 @@ export class Service extends DatabaseService<Model> {
     }
   }
 
+  /*
+   * Re-rolls the chains a maintenance window just started or stopped
+   * covering.
+   *
+   * Only the attached sites and their ANCESTORS can have changed verdict:
+   * the maintained subtree keeps every one of its own devices either way
+   * (see recomputeRollupForSite), so nothing below an attached site needs
+   * recomputing. Each chain is deduplicated, because attaching a Region and
+   * one of its Markets to the same window would otherwise walk the shared
+   * ancestors twice.
+   *
+   * Errors are swallowed per site: the five-minute stale-rollup sweep is the
+   * backstop, and a maintenance event must not fail to start because one
+   * site's rollup did.
+   */
+  @CaptureSpan()
+  public async recomputeRollupsAfterMaintenanceChange(data: {
+    projectId: ObjectID;
+    siteIds: Array<ObjectID>;
+  }): Promise<void> {
+    if (data.siteIds.length === 0) {
+      return;
+    }
+
+    /*
+     * The suppression set is cached for a few seconds; a window that has
+     * just flipped state must not be recomputed against the previous
+     * answer.
+     */
+    NetworkSiteMaintenanceSuppression.invalidateCache(data.projectId);
+
+    const recomputed: Set<string> = new Set<string>();
+
+    /*
+     * Descendants of an attached site change verdict too, which is easy to
+     * miss because the attached site itself does not. While the window runs,
+     * a descendant D is maintained (coverage is inherited downward), so D
+     * suppresses nothing of its own. The moment the window ends D stops
+     * being maintained and starts suppressing ITS maintained descendants —
+     * a different answer, from the same devices. Nested and overlapping
+     * windows are ordinary on a franchise estate, so this is not a corner.
+     */
+    const subtreeIds: Set<string> = await this.getSubtreeSiteIds({
+      siteIds: data.siteIds,
+      projectId: data.projectId,
+    });
+
+    const roots: Array<ObjectID> = Array.from(subtreeIds).map(
+      (id: string): ObjectID => {
+        return new ObjectID(id);
+      },
+    );
+
+    for (const siteId of roots) {
+      const chain: Array<ObjectID> = [
+        siteId,
+        ...(await this.getAncestorIds(siteId)).reverse(),
+      ];
+
+      for (const chainSiteId of chain) {
+        const key: string = chainSiteId.toString();
+        if (recomputed.has(key)) {
+          continue;
+        }
+        recomputed.add(key);
+
+        try {
+          await this.recomputeRollupForSite(chainSiteId);
+        } catch (error) {
+          logger.error(
+            `NetworkSiteService.recomputeRollupsAfterMaintenanceChange: rollup failed for site ${key}: ${error}`,
+            {
+              projectId: data.projectId.toString(),
+              siteId: key,
+            } as LogAttributes,
+          );
+        }
+      }
+    }
+  }
+
   /*
    * ------------------------------------------------------------------
    * Monitor status bridge