@oneuptime/common 12.0.8 → 12.0.9

package/Server/Services/NetworkSiteService.ts

@@ -1173,10 +1173,20 @@ export class Service extends DatabaseService<Model> {
 
   /*
    * Called by MonitorService.changeMonitorStatus after a status persists.
-   * Resolves which NetworkDevices the monitors reference in their steps,
-   * stamps those devices' currentMonitorStatusId, then recomputes the
-   * rollup for every affected site chain. Never throws - a rollup failure
-   * must never break a monitor status change.
+   * Resolves which NetworkDevices these monitors report on, stamps those
+   * devices' currentMonitorStatusId, then recomputes the rollup for every
+   * affected site chain. Never throws - a rollup failure must never break a
+   * monitor status change.
+   *
+   * A device is reached two ways, and both are checked:
+   *
+   *   - a Network Device monitor names it inside its step data. This is the
+   *     SNMP path: the device is polled by its probe and the monitor exists
+   *     to alert on what the walk reports.
+   *   - the device points AT a monitor through monitorId. This is the
+   *     monitor-backed path (monitoringMethod "Monitor") for gear that does
+   *     not speak SNMP, so ANY monitor type qualifies — a Ping monitor on an
+   *     access point is the whole point of it.
    */
   @CaptureSpan()
   public async onMonitorStatusChanged(data: {
@@ -1207,33 +1217,75 @@ export class Service extends DatabaseService<Model> {
         },
       });
 
-      if (monitors.length === 0) {
-        return;
+      const referencedDeviceIds: Array<string> =
+        monitors.length > 0
+          ? NetworkDeviceHydrationUtil.getReferencedNetworkDeviceIds(monitors)
+          : [];
+
+      /*
+       * Deduplicated by id, because a device can legitimately be reached
+       * both ways — an SNMP-polled device may also carry a monitorId — and
+       * stamping it twice would double every rollup it triggers.
+       */
+      const devicesById: Map<string, NetworkDevice> = new Map<
+        string,
+        NetworkDevice
+      >();
+
+      const collect: (rows: Array<NetworkDevice>) => void = (
+        rows: Array<NetworkDevice>,
+      ): void => {
+        for (const row of rows) {
+          if (row.id) {
+            devicesById.set(row.id.toString(), row);
+          }
+        }
+      };
+
+      if (referencedDeviceIds.length > 0) {
+        collect(
+          await NetworkDeviceService.findBy({
+            query: {
+              _id: QueryHelper.any(referencedDeviceIds),
+              projectId: data.projectId,
+            },
+            select: {
+              _id: true,
+              siteId: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: {
+              isRoot: true,
+            },
+          }),
+        );
       }
 
-      const deviceIds: Array<string> =
-        NetworkDeviceHydrationUtil.getReferencedNetworkDeviceIds(monitors);
+      collect(
+        await NetworkDeviceService.findBy({
+          query: {
+            monitorId: QueryHelper.any(data.monitorIds),
+            projectId: data.projectId,
+          },
+          select: {
+            _id: true,
+            siteId: true,
+          },
+          limit: LIMIT_MAX,
+          skip: 0,
+          props: {
+            isRoot: true,
+          },
+        }),
+      );
+
+      const devices: Array<NetworkDevice> = Array.from(devicesById.values());
 
-      if (deviceIds.length === 0) {
+      if (devices.length === 0) {
         return;
       }
 
-      const devices: Array<NetworkDevice> = await NetworkDeviceService.findBy({
-        query: {
-          _id: QueryHelper.any(deviceIds),
-          projectId: data.projectId,
-        },
-        select: {
-          _id: true,
-          siteId: true,
-        },
-        limit: LIMIT_MAX,
-        skip: 0,
-        props: {
-          isRoot: true,
-        },
-      });
-
       for (const device of devices) {
         if (!device.id) {
           continue;

package/Server/Services/NetworkTopologySuppressionService.ts

@@ -0,0 +1,84 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/NetworkTopologySuppression";
+import { OnCreate } from "../Types/Database/Hooks";
+import CreateBy from "../Types/Database/CreateBy";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import BadDataException from "../../Types/Exception/BadDataException";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  /*
+   * Hiding a node twice is a no-op the UI should not have to guard against —
+   * a second click, two people on the same map — so a duplicate is rejected
+   * rather than quietly creating a second row that would then need deleting
+   * twice to restore the node.
+   */
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    const nodeKey: string | undefined = createBy.data.nodeKey?.trim();
+
+    if (!nodeKey) {
+      throw new BadDataException("Node key is required.");
+    }
+
+    createBy.data.nodeKey = nodeKey;
+
+    if (!createBy.data.projectId) {
+      throw new BadDataException("Project ID is required.");
+    }
+
+    const existing: Model | null = await this.findOneBy({
+      query: {
+        projectId: createBy.data.projectId,
+        nodeKey: nodeKey,
+      },
+      select: { _id: true },
+      props: { isRoot: true },
+    });
+
+    if (existing) {
+      throw new BadDataException("This node is already hidden from the map.");
+    }
+
+    return { createBy, carryForward: null };
+  }
+
+  /*
+   * The node keys hidden in a project, for the topology builder. Returns a
+   * Set so the builder's per-node lookup stays O(1).
+   */
+  @CaptureSpan()
+  public async getSuppressedNodeKeys(data: {
+    projectId: Model["projectId"];
+  }): Promise<Set<string>> {
+    if (!data.projectId) {
+      return new Set<string>();
+    }
+
+    const rows: Array<Model> = await this.findBy({
+      query: { projectId: data.projectId },
+      select: { nodeKey: true },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    return new Set<string>(
+      rows
+        .map((row: Model) => {
+          return row.nodeKey || "";
+        })
+        .filter((key: string) => {
+          return key.length > 0;
+        }),
+    );
+  }
+}
+
+export default new Service();

package/Server/Services/OnCallDutyPolicyEscalationRuleScheduleService.ts

@@ -85,6 +85,43 @@ export class Service extends DatabaseService<Model> {
         },
       );
 
+    /*
+     * Record the attachment in the policy feed BEFORE the early return below.
+     *
+     * This used to sit at the bottom of the method, after `if (!userOnSchedule)
+     * return`, so attaching a schedule that happened to have nobody on call at
+     * that instant produced no feed item and no workspace message at all — the
+     * audit trail silently depended on the roster state at the moment of the
+     * click. Note the asymmetry it created: the delete path emits its feed item
+     * from onBeforeDelete, which runs unconditionally, so removals were always
+     * recorded while additions could vanish.
+     *
+     * When nobody is on call, the entry says so — attaching a schedule that
+     * currently pages no one is exactly the thing an operator wants to see.
+     */
+    const feedOnCallDutyPolicyId: ObjectID | undefined | null =
+      createdModel.onCallDutyPolicy?.id;
+
+    if (feedOnCallDutyPolicyId) {
+      const noCoverageSuffix: string = userOnSchedule
+        ? ""
+        : " ⚠️ **No one is currently on call in this schedule**, so alerts escalating to this rule will not notify anyone until coverage resumes.";
+
+      await OnCallDutyPolicyFeedService.createOnCallDutyPolicyFeedItem({
+        onCallDutyPolicyId: feedOnCallDutyPolicyId,
+        projectId: createdModel.projectId!,
+        onCallDutyPolicyFeedEventType:
+          OnCallDutyPolicyFeedEventType.OnCallDutyScheduleAdded,
+        displayColor: userOnSchedule ? Gray500 : Red500,
+        feedInfoInMarkdown: `📅 Added on-call schedule **${createdModel.onCallDutyPolicySchedule?.name || ""}** from the [On-Call Policy ${createdModel.onCallDutyPolicy?.name || ""}](${(await OnCallDutyPolicyService.getOnCallDutyPolicyLinkInDashboard(createdModel.projectId!, feedOnCallDutyPolicyId)).toString()}) escalation rule **${createdModel.onCallDutyPolicyEscalationRule?.name}** with order **${createdModel.onCallDutyPolicyEscalationRule?.order}**.${noCoverageSuffix}`,
+        userId: createdModel.createdByUserId || undefined,
+        workspaceNotification: {
+          sendWorkspaceNotification: true,
+          notifyUserId: createdModel.createdByUserId || undefined,
+        },
+      });
+    }
+
     if (!userOnSchedule) {
       return createdItem;
     }
@@ -202,35 +239,10 @@ export class Service extends DatabaseService<Model> {
       eventType,
     });
 
-    // add workspace message.
-
-    const onCallDutyPolicyId: ObjectID | undefined | null =
-      createdModel.onCallDutyPolicy!.id;
-    const projectId: ObjectID | undefined = createdModel.projectId;
-
-    const createdByUserId: ObjectID | undefined | null =
-      createdModel.createdByUserId;
-
-    const onCallSchedule: OnCallDutyPolicySchedule | undefined =
-      createdModel.onCallDutyPolicySchedule;
-    const onCallDutyPolicyName: string | null =
-      createdModel.onCallDutyPolicy?.name || "";
-
-    if (onCallDutyPolicyId) {
-      await OnCallDutyPolicyFeedService.createOnCallDutyPolicyFeedItem({
-        onCallDutyPolicyId: onCallDutyPolicyId,
-        projectId: projectId!,
-        onCallDutyPolicyFeedEventType:
-          OnCallDutyPolicyFeedEventType.OnCallDutyScheduleAdded,
-        displayColor: Gray500,
-        feedInfoInMarkdown: `📅 Added on-call schedule **${onCallSchedule?.name || ""}** from the [On-Call Policy ${onCallDutyPolicyName}](${(await OnCallDutyPolicyService.getOnCallDutyPolicyLinkInDashboard(projectId!, onCallDutyPolicyId!)).toString()}) escalation rule **${createdModel.onCallDutyPolicyEscalationRule?.name}** with order **${createdModel.onCallDutyPolicyEscalationRule?.order}**.`,
-        userId: createdByUserId || undefined,
-        workspaceNotification: {
-          sendWorkspaceNotification: true,
-          notifyUserId: createdByUserId || undefined,
-        },
-      });
-    }
+    /*
+     * The "schedule added" feed item is emitted above, before the
+     * no-one-on-call early return, so it is recorded unconditionally.
+     */
 
     return createdItem;
   }

package/Server/Services/OnCallDutyPolicyExecutionLogService.ts

@@ -332,6 +332,12 @@ export class Service extends DatabaseService<Model> {
         return Yellow500;
       case OnCallDutyPolicyStatus.Completed:
         return Green500;
+      /*
+       * Amber, not green: the policy finished, but finishing without paging
+       * anyone is a failure of the thing the policy exists to do.
+       */
+      case OnCallDutyPolicyStatus.CompletedWithNoNotifications:
+        return Yellow500;
       case OnCallDutyPolicyStatus.Error:
         return Red500;
       default:
@@ -349,6 +355,8 @@ export class Service extends DatabaseService<Model> {
         return "▶️";
       case OnCallDutyPolicyStatus.Completed:
         return "🏁";
+      case OnCallDutyPolicyStatus.CompletedWithNoNotifications:
+        return "⚠️";
       case OnCallDutyPolicyStatus.Error:
         return "❌";
       default:

package/Server/Services/OnCallDutyPolicyExecutionLogTimelineService.ts

@@ -91,6 +91,13 @@ export class Service extends DatabaseService<Model> {
           triggeredByIncidentId: true,
           triggeredByAlertId: true,
           triggeredByAlertEpisodeId: true,
+          /*
+           * Was missing from this select while the guard below (and the feed
+           * branch further down) read it — so it was always undefined and every
+           * incident-episode-triggered execution returned early, producing no
+           * on-call feed entries at all for that trigger type.
+           */
+          triggeredByIncidentEpisodeId: true,
           projectId: true,
           status: true,
           statusMessage: true,
@@ -221,21 +228,63 @@ export class Service extends DatabaseService<Model> {
           incidentOrAlertLink = `[Incident Episode ${incidentEpisodeNumberResult.numberWithPrefix || "#" + incidentEpisodeNumberResult.number}](${(await IncidentEpisodeService.getEpisodeLinkInDashboard(onCallDutyPolicyExecutionLogTimeline.projectId!, onCallDutyPolicyExecutionLogTimeline.triggeredByIncidentEpisodeId)).toString()})`;
         }
 
-        let feedInfoInMarkdown: string = `**${this.getEmojiBasedOnStatus(status)} ${incidentOrAlertLink} On-Call Alert ${status} to ${await UserService.getUserMarkdownString(
-          {
-            userId: onCallDutyPolicyExecutionLogTimeline.alertSentToUserId!,
-            projectId: onCallDutyPolicyExecutionLogTimeline.projectId!,
-          },
-        )}**
+        const policyLink: string = `**[${onCallDutyPolicyExecutionLogTimeline.onCallDutyPolicy.name}](${(await OnCallDutyPolicyService.getOnCallDutyPolicyLinkInDashboard(onCallDutyPolicyExecutionLogTimeline.projectId!, onCallDutyPolicyExecutionLogTimeline.onCallDutyPolicy.id!)).toString()})**`;
 
-The on-call policy **[${onCallDutyPolicyExecutionLogTimeline.onCallDutyPolicy.name}](${(await OnCallDutyPolicyService.getOnCallDutyPolicyLinkInDashboard(onCallDutyPolicyExecutionLogTimeline.projectId!, onCallDutyPolicyExecutionLogTimeline.onCallDutyPolicy.id!)).toString()})** has been triggered. The escalation rule **${onCallDutyPolicyExecutionLogTimeline.onCallDutyPolicyEscalationRule?.name}** ${onCallDutyPolicyExecutionLogTimeline.onCallDutySchedule?.name ? String(" and schedule **" + onCallDutyPolicyExecutionLogTimeline.onCallDutySchedule?.name + "**") : ""} were applied. ${await UserService.getUserMarkdownString(
-          {
-            userId: onCallDutyPolicyExecutionLogTimeline.alertSentToUserId!,
-            projectId: onCallDutyPolicyExecutionLogTimeline.projectId!,
-          },
-        )} was alerted. The status of this alert is **${status}** with the message: \`${onCallDutyPolicyExecutionLogTimeline.statusMessage}\`. ${onCallDutyPolicyExecutionLogTimeline.userBelongsToTeam?.name ? "The alert was sent because the user belogs to the team **" + onCallDutyPolicyExecutionLogTimeline.userBelongsToTeam?.name + "** " : ""} ${onCallDutyPolicyExecutionLogTimeline.isAcknowledged ? "The alert was acknowledged at **" + onCallDutyPolicyExecutionLogTimeline.acknowledgedAt + "** " : ""}`;
+        const scheduleClause: string = onCallDutyPolicyExecutionLogTimeline
+          .onCallDutySchedule?.name
+          ? String(
+              " and schedule **" +
+                onCallDutyPolicyExecutionLogTimeline.onCallDutySchedule?.name +
+                "**",
+            )
+          : "";
+
+        /*
+         * Some timeline rows have NO recipient by construction: a schedule that
+         * currently has nobody on call (a coverage gap), or an escalation rule
+         * with no responders at all. Those rows are exactly the ones an operator
+         * most needs to understand, so they get their own wording.
+         *
+         * Building the "was alerted" sentence for them used to interpolate
+         * `alertSentToUserId!` — a non-null assertion on a field that is never
+         * set here — which rendered a feed entry claiming somebody was paged
+         * when in fact nobody was.
+         */
+        const hasRecipient: boolean = Boolean(
+          onCallDutyPolicyExecutionLogTimeline.alertSentToUserId,
+        );
+
+        let feedInfoInMarkdown: string = "";
+
+        if (!hasRecipient) {
+          const noRecipientReason: string = onCallDutyPolicyExecutionLogTimeline
+            .onCallDutySchedule?.name
+            ? `no one was on call in schedule **${onCallDutyPolicyExecutionLogTimeline.onCallDutySchedule.name}**`
+            : "this escalation rule had no responders";
+
+          feedInfoInMarkdown = `**${this.getEmojiBasedOnStatus(status)} ${incidentOrAlertLink} On-Call Alert ${status} — nobody was notified**
+
+The on-call policy ${policyLink} has been triggered. The escalation rule **${onCallDutyPolicyExecutionLogTimeline.onCallDutyPolicyEscalationRule?.name}**${scheduleClause} were applied, but ${noRecipientReason}, so **no one was notified at this step**. The status of this step is **${status}** with the message: \`${onCallDutyPolicyExecutionLogTimeline.statusMessage}\`.`;
+        } else {
+          feedInfoInMarkdown = `**${this.getEmojiBasedOnStatus(status)} ${incidentOrAlertLink} On-Call Alert ${status} to ${await UserService.getUserMarkdownString(
+            {
+              userId: onCallDutyPolicyExecutionLogTimeline.alertSentToUserId!,
+              projectId: onCallDutyPolicyExecutionLogTimeline.projectId!,
+            },
+          )}**
+
+The on-call policy ${policyLink} has been triggered. The escalation rule **${onCallDutyPolicyExecutionLogTimeline.onCallDutyPolicyEscalationRule?.name}** ${scheduleClause} were applied. ${await UserService.getUserMarkdownString(
+            {
+              userId: onCallDutyPolicyExecutionLogTimeline.alertSentToUserId!,
+              projectId: onCallDutyPolicyExecutionLogTimeline.projectId!,
+            },
+          )} was alerted. The status of this alert is **${status}** with the message: \`${onCallDutyPolicyExecutionLogTimeline.statusMessage}\`. ${onCallDutyPolicyExecutionLogTimeline.userBelongsToTeam?.name ? "The alert was sent because the user belogs to the team **" + onCallDutyPolicyExecutionLogTimeline.userBelongsToTeam?.name + "** " : ""} ${onCallDutyPolicyExecutionLogTimeline.isAcknowledged ? "The alert was acknowledged at **" + onCallDutyPolicyExecutionLogTimeline.acknowledgedAt + "** " : ""}`;
+        }
 
-        if (onCallDutyPolicyExecutionLogTimeline.overridedByUser) {
+        if (
+          hasRecipient &&
+          onCallDutyPolicyExecutionLogTimeline.overridedByUser
+        ) {
           feedInfoInMarkdown += `The alert was supposed to be sent to **${await UserService.getUserMarkdownString(
             {
               userId: onCallDutyPolicyExecutionLogTimeline.overridedByUser.id!,

package/Server/Services/OnCallDutyPolicyScheduleService.ts

@@ -39,7 +39,7 @@ import logger, { LogAttributes } from "../Utils/Logger";
 import Semaphore, { SemaphoreMutex } from "../Infrastructure/Semaphore";
 import OnCallDutyPolicyFeedService from "./OnCallDutyPolicyFeedService";
 import { OnCallDutyPolicyFeedEventType } from "../../Models/DatabaseModels/OnCallDutyPolicyFeed";
-import { Green500 } from "../../Types/BrandColors";
+import { Green500, Red500 } from "../../Types/BrandColors";
 import OnCallDutyPolicyTimeLogService from "./OnCallDutyPolicyTimeLogService";
 import OnCallDutyPolicyScheduleLabelRuleEngineService from "./OnCallDutyPolicyScheduleLabelRuleEngineService";
 import OnCallDutyPolicyScheduleOwnerRuleEngineService from "./OnCallDutyPolicyScheduleOwnerRuleEngineService";
@@ -548,6 +548,66 @@ export class Service extends DatabaseService<OnCallDutyPolicySchedule> {
             },
           });
         }
+
+        /*
+         * Somebody was on call and now nobody is. This branch is the mirror of
+         * the two above and used to be missing entirely: the "now on-call"
+         * branch requires a NEW user to exist, and the "no longer on-call"
+         * branch notifies only the departing person — who is precisely the one
+         * individual that can no longer do anything about it. So a schedule
+         * running off the end of its rotation, or into restricted hours with no
+         * fallback layer, opened a coverage gap in complete silence.
+         *
+         * Deliberately NOT tied to a user notification bundle (email/SMS/call):
+         * there is no on-call person to send it to. It goes to the policy feed
+         * and out to the workspace channel, where whoever owns the policy sees it.
+         */
+        if (
+          previousInformation.currentUserIdOnRoster?.toString() &&
+          !newInformation.currentUserIdOnRoster?.toString()
+        ) {
+          const onCallDutyPolicyId: ObjectID =
+            escalationRule.onCallDutyPolicy!.id!;
+
+          const policyLink: string = (
+            await OnCallDutyPolicyService.getOnCallDutyPolicyLinkInDashboard(
+              projectId!,
+              onCallDutyPolicyId!,
+            )
+          ).toString();
+
+          /*
+           * When a next user is known, say when coverage resumes — that turns
+           * "nobody is on call" from an alarm into an actionable window.
+           */
+          const resumesClause: string = newInformation.nextUserIdOnRoster
+            ? ` Coverage resumes at **${OneUptimeDate.getDateAsFormattedStringInMultipleTimezones(
+                {
+                  date:
+                    newInformation.nextRosterStartAt ||
+                    newInformation.nextHandOffTimeAt ||
+                    OneUptimeDate.getCurrentDate(),
+                  timezones: [Timezone.GMT],
+                },
+              )}** with **${await UserService.getUserMarkdownString({
+                userId: newInformation.nextUserIdOnRoster,
+                projectId: projectId!,
+              })}**.`
+            : " No further shifts are scheduled, so this schedule will keep paging no one until it is fixed.";
+
+          await OnCallDutyPolicyFeedService.createOnCallDutyPolicyFeedItem({
+            onCallDutyPolicyId: onCallDutyPolicyId,
+            projectId: projectId!,
+            onCallDutyPolicyFeedEventType:
+              OnCallDutyPolicyFeedEventType.CoverageGapStarted,
+            displayColor: Red500,
+            feedInfoInMarkdown: `⚠️ **Coverage gap: no one is on call in schedule ${onCallSchedule.name}.** [On-Call Policy ${escalationRule.onCallDutyPolicy?.name}](${policyLink}) escalation rule **${escalationRule.onCallDutyPolicyEscalationRule?.name}** with order **${escalationRule.onCallDutyPolicyEscalationRule?.order}** targets this schedule, so any alert that escalates to it right now will notify nobody.${resumesClause}`,
+            workspaceNotification: {
+              sendWorkspaceNotification: true,
+              notifyUserId: undefined,
+            },
+          });
+        }
       }
 
       // send an email to the next user.