@oneuptime/common 12.0.32 → 12.0.33

package/Server/Services/MonitorStatusTimelineService.ts

@@ -7,6 +7,7 @@ import logger, { LogAttributes } from "../Utils/Logger";
 import ProjectScopedReferenceValidator from "../Utils/Database/ProjectScopedReferenceValidator";
 import DatabaseService from "./DatabaseService";
 import MonitorService from "./MonitorService";
+import NetworkSiteService from "./NetworkSiteService";
 import UserService from "./UserService";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
@@ -106,14 +107,26 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
     }
 
     /*
-     * The feed item and its workspace notification run AFTER the mutex is
-     * released: they can involve third-party HTTP (Slack/Teams) with unbounded
-     * latency, and holding the per-monitor lock across them would block every
-     * concurrent status write for this monitor until acquireTimeout - turning
-     * one slow webhook into refused status transitions. Only the predecessor
-     * read -> INSERT -> predecessor close needs the lock, and all of that has
-     * completed by this point. (The pre-fail-closed code released the lock at
-     * this same boundary, before the feed block.)
+     * Both side effects below run AFTER the mutex is released. Only the
+     * predecessor read -> INSERT -> predecessor close (plus the monitor's
+     * currentMonitorStatusId write in onCreateSuccess) needs the lock, and all
+     * of that has completed by this point.
+     *
+     * The network-site bridge goes first: it stamps the devices bound to this
+     * monitor and recomputes their site rollups (which can open alerts and post
+     * their own workspace notifications), and a device stamp must not wait on
+     * the feed item's Slack/Teams round trip below. It is deliberately NOT run
+     * from onCreateSuccess, which executes under the lock - see bridgeIfCurrent.
+     */
+    await this.bridgeIfCurrent(createdItem, createBy);
+
+    /*
+     * The feed item and its workspace notification can involve third-party
+     * HTTP (Slack/Teams) with unbounded latency, and holding the per-monitor
+     * lock across them would block every concurrent status write for this
+     * monitor until acquireTimeout - turning one slow webhook into refused
+     * status transitions. (The pre-fail-closed code released the lock at this
+     * same boundary, before the feed block.)
      */
     await this.createStatusChangeFeedItem(createdItem, createBy);
 
@@ -508,10 +521,121 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
         },
         props: onCreate.createBy.props,
       });
+
+      /*
+       * The network-site bridge for this new current status is deliberately
+       * NOT called from here: this hook runs with the per-monitor mutex held
+       * and the bridge can be slow. create() runs it via bridgeIfCurrent
+       * right after the mutex is released.
+       */
     }
     return createdItem;
   }
 
+  /*
+   * Bridges a NEW CURRENT row (no endsAt) to the network-site rollup engine.
+   *
+   * Why the timeline service bridges at all: this is the one place every
+   * status change passes through. The probe path (MonitorResource ->
+   * MonitorStatusTimelineService.create) writes the timeline row with root
+   * props - no tenantId - so the MonitorService.updateOneBy in onCreateSuccess
+   * never trips MonitorService.onUpdateSuccess's tenantId-gated
+   * changeMonitorStatus, and a bridge that lived only inside
+   * changeMonitorStatus fired for manual / incident / maintenance status
+   * changes but never for a probe result. Devices bound to a monitor therefore
+   * only ever moved on those, and a ping monitor going down left its device
+   * "Up".
+   *
+   * Why it is called from create() AFTER the per-monitor mutex is released and
+   * not from onCreateSuccess (which runs under the lock): the bridge stamps
+   * every bound device and recomputes the site chain of each (which can open
+   * alerts and post workspace notifications), and holding the lock across that
+   * would serialise every concurrent status write for this monitor behind it -
+   * past the semaphore's acquire timeout, refuse it. Only the predecessor
+   * read -> INSERT -> predecessor close needs the lock (see create()).
+   *
+   * Ordering note: two status writes for the same monitor bridge in release
+   * order, and a bridge carries the status of ITS row. Probe results for one
+   * monitor are already serialised by MonitorResource's outer lock, and a
+   * manual change racing a probe result is rare and self-corrects on the next
+   * status change, so this is the right side of the trade-off.
+   *
+   * The row's projectId is the tenant column and is populated by every writer
+   * (onBeforeCreate reads it or falls back to props.tenantId), so it is the
+   * authoritative source here; the createBy data and tenantId are only
+   * defensive fallbacks for a saved entity that somehow came back without it.
+   */
+  private async bridgeIfCurrent(
+    createdItem: MonitorStatusTimeline,
+    createBy: CreateBy<MonitorStatusTimeline>,
+  ): Promise<void> {
+    if (
+      createdItem.endsAt ||
+      !createdItem.monitorId ||
+      !createdItem.monitorStatusId
+    ) {
+      // A closed row is history, not the current status: nothing to stamp.
+      return;
+    }
+
+    const projectId: ObjectID | undefined =
+      createdItem.projectId ||
+      createBy.data.projectId ||
+      createBy.props.tenantId;
+
+    await this.bridgeCurrentStatusToNetworkSites({
+      projectId: projectId,
+      monitorId: createdItem.monitorId,
+      monitorStatusId: createdItem.monitorStatusId,
+      logAttributes: {
+        projectId: projectId?.toString(),
+        monitorId: createdItem.monitorId.toString(),
+      } as LogAttributes,
+    });
+  }
+
+  /*
+   * Stamps the NetworkDevices that this monitor reports on (a Network Device
+   * monitor's step devices, and any device bound through its monitorId) and
+   * refreshes their sites' rollups, via NetworkSiteService.
+   *
+   * Resilient by contract: onMonitorStatusChanged catches internally and never
+   * throws, and this wraps it once more, because it is called from create()
+   * (via bridgeIfCurrent, after the per-monitor mutex is released) and from
+   * onDeleteSuccess - in both cases after the timeline row and the monitor's
+   * currentMonitorStatusId have already been written. A rollup failure must
+   * never turn an already-persisted status change into an error for the
+   * caller (and, on the create path, must never surface as a failed probe
+   * ingest).
+   */
+  private async bridgeCurrentStatusToNetworkSites(data: {
+    projectId: ObjectID | undefined;
+    monitorId: ObjectID;
+    monitorStatusId: ObjectID;
+    logAttributes: LogAttributes;
+  }): Promise<void> {
+    if (!data.projectId) {
+      logger.warn(
+        `MonitorStatusTimeline: cannot bridge the status change of monitor ${data.monitorId.toString()} to network sites because no projectId was resolved; any network device bound to this monitor keeps its previous stamped status.`,
+        data.logAttributes,
+      );
+      return;
+    }
+
+    try {
+      await NetworkSiteService.onMonitorStatusChanged({
+        projectId: data.projectId,
+        monitorIds: [data.monitorId],
+        monitorStatusId: data.monitorStatusId,
+      });
+    } catch (err) {
+      logger.error(
+        `MonitorStatusTimeline: failed to update network site rollups for monitor ${data.monitorId.toString()}: ${err}`,
+        data.logAttributes,
+      );
+    }
+  }
+
   /*
    * Writes the monitor feed item (and its workspace notification, which can be
    * third-party HTTP to Slack/Teams) for a status change. Called from create()
@@ -757,6 +881,8 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
           select: {
             _id: true,
             monitorStatusId: true,
+            // For the network-site bridge below.
+            projectId: true,
           },
         });
 
@@ -772,6 +898,22 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
             isRoot: true,
           },
         });
+
+        /*
+         * The monitor just fell back to the surviving latest status, and the
+         * update above is root (no tenantId) - the same gap as on the create
+         * path - so the devices bound to this monitor must be re-stamped
+         * from here or they keep reporting the deleted row's status.
+         */
+        await this.bridgeCurrentStatusToNetworkSites({
+          projectId: monitorStatusTimeline.projectId,
+          monitorId: monitorId,
+          monitorStatusId: monitorStatusTimeline.monitorStatusId,
+          logAttributes: {
+            projectId: monitorStatusTimeline.projectId?.toString(),
+            monitorId: monitorId.toString(),
+          } as LogAttributes,
+        });
       }
     }
 

package/Server/Services/MonitorTemplateService.ts

@@ -1,8 +1,14 @@
 import DatabaseService from "./DatabaseService";
 import MonitorService from "./MonitorService";
+import NetworkAlertPolicyEngineService from "./NetworkAlertPolicyEngineService";
+import NetworkAlertPolicyService from "./NetworkAlertPolicyService";
+import NetworkDeviceAutoImportRuleService from "./NetworkDeviceAutoImportRuleService";
+import NetworkAlertPolicy from "../../Models/DatabaseModels/NetworkAlertPolicy";
+import NetworkDeviceAutoImportRule from "../../Models/DatabaseModels/NetworkDeviceAutoImportRule";
 import CreateBy from "../Types/Database/CreateBy";
+import DeleteBy from "../Types/Database/DeleteBy";
 import UpdateBy from "../Types/Database/UpdateBy";
-import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
+import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
 import MonitorStepsProjectValidator from "../Utils/Monitor/MonitorStepsProjectValidator";
 import DatabaseCommonInteractionProps from "../../Types/BaseDatabase/DatabaseCommonInteractionProps";
 import BadDataException from "../../Types/Exception/BadDataException";
@@ -108,6 +114,170 @@ export class Service extends DatabaseService<Model> {
     return { updateBy, carryForward: null };
   }
 
+  /*
+   * A template that something PROVISIONS FROM cannot be deleted.
+   *
+   * Both foreign keys pointing here are ON DELETE SET NULL, so without this
+   * guard the delete succeeds and the damage is silent and deferred:
+   *
+   *   - A NETWORK ALERT POLICY loses its template and stops provisioning.
+   *     Its existing monitors stay (they are the fleet's incident history)
+   *     but nothing new is ever covered, and the settings table shows a
+   *     policy that looks live and is not. The operator finds out when a
+   *     switch they added last month turns out never to have been alerted
+   *     on. There is no undo either: pointing the policy at a replacement
+   *     template makes the engine tear the old fleet down and re-clone it.
+   *   - An AUTO-IMPORT RULE loses its template and quietly imports devices
+   *     with no monitor at all, which is the same failure one scan later.
+   *
+   * So the delete is refused, by name and by count, and the operator is
+   * told which thing to detach first. This is the same contract
+   * NetworkDeviceOidTemplateService.onBeforeDelete enforces for OID
+   * templates, and it is the reason the policy's SET NULL branch is a
+   * backstop rather than a path anything takes on purpose.
+   */
+  @CaptureSpan()
+  protected override async onBeforeDelete(
+    deleteBy: DeleteBy<Model>,
+  ): Promise<OnDelete<Model>> {
+    const templatesToDelete: Array<Model> = await this.findBy({
+      /*
+       * This hook runs BEFORE DatabaseService permission-checks the query, so
+       * a raw isRoot read of deleteBy.query would hand back other tenants'
+       * templates — and their policy names in the refusal message with them.
+       */
+      query: this.scopeQueryToCallerTenant(deleteBy.query, deleteBy.props),
+      select: {
+        _id: true,
+        templateName: true,
+        projectId: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+      },
+    });
+
+    for (const template of templatesToDelete) {
+      if (!template.id || !template.projectId) {
+        continue;
+      }
+
+      const templateName: string =
+        template.templateName || "This monitor template";
+
+      const policies: Array<NetworkAlertPolicy> =
+        await NetworkAlertPolicyService.findBy({
+          query: {
+            projectId: template.projectId,
+            monitorTemplateId: template.id,
+          },
+          select: {
+            _id: true,
+            name: true,
+          },
+          limit: LIMIT_MAX,
+          skip: 0,
+          props: {
+            isRoot: true,
+          },
+        });
+
+      if (policies.length > 0) {
+        throw new BadDataException(
+          `${templateName} is used by ${policies.length} network alert ${
+            policies.length === 1 ? "policy" : "policies"
+          } (${this.describeNames(
+            policies.map((policy: NetworkAlertPolicy): string => {
+              return policy.name || policy.id?.toString() || "unnamed";
+            }),
+          )}). Point ${
+            policies.length === 1 ? "it" : "them"
+          } at another template, or delete ${
+            policies.length === 1 ? "it" : "them"
+          }, before deleting this template.`,
+        );
+      }
+
+      const rules: Array<NetworkDeviceAutoImportRule> =
+        await NetworkDeviceAutoImportRuleService.findBy({
+          query: {
+            projectId: template.projectId,
+            monitorTemplateId: template.id,
+          },
+          select: {
+            _id: true,
+            name: true,
+          },
+          limit: LIMIT_MAX,
+          skip: 0,
+          props: {
+            isRoot: true,
+          },
+        });
+
+      if (rules.length > 0) {
+        throw new BadDataException(
+          `${templateName} is used by ${rules.length} network device auto-import ${
+            rules.length === 1 ? "rule" : "rules"
+          } (${this.describeNames(
+            rules.map((rule: NetworkDeviceAutoImportRule): string => {
+              return rule.name || rule.id?.toString() || "unnamed";
+            }),
+          )}). Clear it from ${
+            rules.length === 1 ? "that rule" : "those rules"
+          } before deleting this template, or devices they import will have no monitor.`,
+        );
+      }
+    }
+
+    return { deleteBy, carryForward: null };
+  }
+
+  /*
+   * See NetworkSiteService for the full explanation: hooks run before
+   * ModelPermission scopes the caller's query, so anything a hook reads with
+   * isRoot has to be re-scoped by hand or it spans projects.
+   */
+  private scopeQueryToCallerTenant(
+    query: Query<Model>,
+    props: DatabaseCommonInteractionProps,
+  ): Query<Model> {
+    if (props.isRoot || !props.tenantId) {
+      return query;
+    }
+
+    return {
+      ...query,
+      projectId: props.tenantId,
+    };
+  }
+
+  /*
+   * "Warehouse switches", "Warehouse switches and Core routers",
+   * "Warehouse switches, Core routers and 3 more" — enough for the operator
+   * to recognise what they have to detach, without pasting a hundred names
+   * into a toast.
+   */
+  private describeNames(names: Array<string>): string {
+    const MAX_NAMED: number = 3;
+
+    if (names.length <= MAX_NAMED) {
+      if (names.length === 1) {
+        return names[0] as string;
+      }
+
+      return `${names.slice(0, names.length - 1).join(", ")} and ${
+        names[names.length - 1]
+      }`;
+    }
+
+    return `${names.slice(0, MAX_NAMED).join(", ")} and ${
+      names.length - MAX_NAMED
+    } more`;
+  }
+
   /**
    * Count monitors created from this template.
    * Caller must already have read access on the template via the API layer.
@@ -319,6 +489,13 @@ export class Service extends DatabaseService<Model> {
         });
       }
 
+      await this.stampPolicyTemplateSync({
+        monitorTemplateId: template.id!,
+        projectId: template.projectId,
+        totalLinkedMonitors,
+        syncedMonitors,
+      });
+
       return { totalLinkedMonitors, syncedMonitors };
     }
 
@@ -391,12 +568,51 @@ export class Service extends DatabaseService<Model> {
       });
     }
 
+    await this.stampPolicyTemplateSync({
+      monitorTemplateId: template.id!,
+      projectId: template.projectId,
+      totalLinkedMonitors,
+      syncedMonitors,
+    });
+
     return {
       totalLinkedMonitors,
       syncedMonitors,
     };
   }
 
+  /*
+   * "Template Synced" on the alert-policies table means one thing: every
+   * monitor the policy owns is running the template's current
+   * configuration. This push is what makes that true, so this is where the
+   * column is stamped — not in the policy engine, which reconciles the
+   * device SET and has no opinion about a criteria edit that left the set
+   * unchanged.
+   *
+   * Only stamped when the push reached EVERY linked monitor. A caller whose
+   * label scopes hide half the fleet has synced half the fleet, and a date
+   * against that would read as "your criteria edit has landed" when it has
+   * landed on some of the devices.
+   */
+  private async stampPolicyTemplateSync(data: {
+    monitorTemplateId: ObjectID;
+    projectId: ObjectID;
+    totalLinkedMonitors: number;
+    syncedMonitors: number;
+  }): Promise<void> {
+    if (
+      data.syncedMonitors <= 0 ||
+      data.syncedMonitors < data.totalLinkedMonitors
+    ) {
+      return;
+    }
+
+    await NetworkAlertPolicyEngineService.onMonitorTemplateSynced({
+      monitorTemplateId: data.monitorTemplateId,
+      projectId: data.projectId,
+    });
+  }
+
   /**
    * Sync the template's current configuration onto a single monitor that was
    * created from it. The monitor must be linked to this template — passing an