@oneuptime/common 12.0.31 → 12.0.33

package/Server/Services/UserNotificationEmailRollupSettingService.ts

@@ -0,0 +1,113 @@
+import DatabaseService from "./DatabaseService";
+import CreateBy from "../Types/Database/CreateBy";
+import { OnCreate } from "../Types/Database/Hooks";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import Model from "../../Models/DatabaseModels/UserNotificationEmailRollupSetting";
+import BadDataException from "../../Types/Exception/BadDataException";
+import ObjectID from "../../Types/ObjectID";
+
+/*
+ * The escape hatch from owner-email burst rollup, one row per (user, project).
+ *
+ * The read below is the ONLY place the preference is consulted, and it is
+ * deliberately not cached. A cache would mean a person who has just clicked
+ * "send me every email immediately" keeps getting batched mail for the length
+ * of the TTL - which is precisely the complaint the toggle exists to answer,
+ * reproduced by the mechanism meant to serve it. The query is a single indexed
+ * row on a table with at most one row per project member, on a path that
+ * already runs a dozen queries, and the write path only reaches it when it is
+ * about to DEFER - never on the common below-the-threshold send.
+ */
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  /*
+   * True when this person's owner notification emails may be coalesced.
+   *
+   * Absent row means enabled. Rollup ships on by default with no backfill, so
+   * almost nobody has a row here; treating "no row" as anything other than the
+   * default would make the feature depend on data that does not exist.
+   *
+   * This does NOT swallow errors. The one caller reads it inside the write
+   * path's fail-open catch, which turns any failure into an immediate send -
+   * the same direction this method's `false` means, and the same direction
+   * every other failure in that path takes.
+   */
+  @CaptureSpan()
+  public async isRollupEnabledForUser(data: {
+    userId: ObjectID;
+    projectId: ObjectID;
+  }): Promise<boolean> {
+    const setting: Model | null = await this.findOneBy({
+      query: {
+        userId: data.userId,
+        projectId: data.projectId,
+      },
+      select: {
+        isEnabled: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    if (!setting) {
+      return true;
+    }
+
+    /*
+     * Only an explicit false opts out. A row whose isEnabled somehow arrived
+     * NULL - a hand-written insert, a column added ahead of its default - is
+     * not an opt-out, because opting somebody out of individual emails is the
+     * change that needs to have been asked for.
+     */
+    return setting.isEnabled !== false;
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    if (!createBy.data.projectId) {
+      throw new BadDataException("projectId is required");
+    }
+
+    if (!createBy.data.userId) {
+      throw new BadDataException("userId is required");
+    }
+
+    /*
+     * One row per person per project. The page creates lazily on the first
+     * toggle, so a double click or two open tabs would otherwise leave two
+     * rows and make findOneBy's answer depend on which one it happened to
+     * pick. There is no unique index behind this check - the same
+     * check-then-create shape UserNotificationSettingService.onBeforeCreate
+     * already uses - so it narrows the window rather than closing it; the read
+     * above tolerates a duplicate because both rows say the same thing in
+     * every flow the UI can produce.
+     */
+    const existingCount: number = (
+      await this.countBy({
+        query: {
+          projectId: createBy.data.projectId,
+          userId: createBy.data.userId,
+        },
+        props: {
+          isRoot: true,
+        },
+      })
+    ).toNumber();
+
+    if (existingCount > 0) {
+      throw new BadDataException(
+        "An email rollup setting already exists for this user in this project.",
+      );
+    }
+
+    return { createBy, carryForward: null };
+  }
+}
+
+export default new Service();

package/Server/Services/UserNotificationSettingService.ts

@@ -3,7 +3,6 @@ import { OnCreate } from "../Types/Database/Hooks";
 import logger from "../Utils/Logger";
 import CallService from "./CallService";
 import DatabaseService from "./DatabaseService";
-import MailService from "./MailService";
 import ProjectCallSMSConfigService from "./ProjectCallSMSConfigService";
 import SmsService from "./SmsService";
 import TeamMemberService from "./TeamMemberService";
@@ -51,6 +50,7 @@ import {
   WorkspaceMessageBlock,
   WorkspacePayloadMarkdown,
 } from "../../Types/Workspace/WorkspaceMessagePayload";
+import EmailRollupWriter from "../Utils/EmailRollup/EmailRollupWriter";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { appendRecipientToWhatsAppMessage } from "../Utils/WhatsAppTemplateUtil";
 
@@ -76,6 +76,16 @@ export class Service extends DatabaseService<UserNotificationSetting> {
      */
     whatsAppMessage?: WhatsAppMessagePayload | undefined;
     telegramMessage?: TelegramMessagePayload | undefined;
+    /*
+     * Bypasses burst coalescing for this send, before any rollup bookkeeping
+     * runs. Some producers reuse another family's event type: the SLA-breach
+     * job at App/FeatureSet/Workers/Jobs/IncidentSla/CheckSlaBreaches.ts sends
+     * under EmailTemplateType.IncidentOwnerResourceCreated at :265 and reuses
+     * SEND_INCIDENT_CREATED_OWNER_NOTIFICATION at :286, so the event type
+     * alone cannot express that this particular message is urgent. Only a
+     * caller knows that, so only a caller can say so.
+     */
+    forceImmediate?: boolean | undefined;
     incidentId?: ObjectID | undefined;
     alertId?: ObjectID | undefined;
     alertEpisodeId?: ObjectID | undefined;
@@ -140,33 +150,60 @@ export class Service extends DatabaseService<UserNotificationSetting> {
         });
 
         for (const userEmail of userEmails) {
-          MailService.sendMail(
-            {
-              ...data.emailEnvelope,
-              toEmail: userEmail.email!,
-            },
-            {
+          /*
+           * The one seam where an owner email can be held back. Below the
+           * burst threshold this is byte-for-byte the send that was here
+           * before - same envelope, same correlation ids, still
+           * fire-and-forget inside the writer - and above it the message is
+           * queued for a rollup instead of dropped. Awaited, unlike the raw
+           * send it replaces, because the queue row has to exist before this
+           * method returns; the writer itself never awaits MailService.
+           *
+           * data.emailEnvelope is passed by reference and is never mutated by
+           * the writer, which the Telegram fallback below and the Slack /
+           * Microsoft Teams bodies further down depend on: all three
+           * synthesise their message from data.emailEnvelope.subject.
+           *
+           * Guarded even though sendOrRollup is written not to throw. The send
+           * it replaces was fire-and-forget, so one bad address could never
+           * cost another address its email - let alone cost this notification
+           * its SMS, call, push, Telegram, workspace and webhook deliveries
+           * further down. Awaiting reintroduces that possibility; this catch
+           * takes it back out.
+           */
+          try {
+            await EmailRollupWriter.sendOrRollup({
               projectId: data.projectId,
-              incidentId: data.incidentId,
-              alertId: data.alertId,
-              alertEpisodeId: data.alertEpisodeId,
-              incidentEpisodeId: data.incidentEpisodeId,
-              monitorId: data.monitorId,
-              scheduledMaintenanceId: data.scheduledMaintenanceId,
-              statusPageId: data.statusPageId,
-              statusPageAnnouncementId: data.statusPageAnnouncementId,
               userId: data.userId,
-              teamId: data.teamId,
-              // OnCall-related fields
-              onCallPolicyId: data.onCallPolicyId,
-              onCallPolicyEscalationRuleId: data.onCallPolicyEscalationRuleId,
-              onCallDutyPolicyExecutionLogTimelineId:
-                data.onCallDutyPolicyExecutionLogTimelineId,
-              onCallScheduleId: data.onCallScheduleId,
-            },
-          ).catch((err: Error) => {
+              toEmail: userEmail.email!,
+              eventType: data.eventType,
+              emailEnvelope: data.emailEnvelope,
+              mailOptions: {
+                projectId: data.projectId,
+                incidentId: data.incidentId,
+                alertId: data.alertId,
+                alertEpisodeId: data.alertEpisodeId,
+                incidentEpisodeId: data.incidentEpisodeId,
+                monitorId: data.monitorId,
+                scheduledMaintenanceId: data.scheduledMaintenanceId,
+                statusPageId: data.statusPageId,
+                statusPageAnnouncementId: data.statusPageAnnouncementId,
+                userId: data.userId,
+                teamId: data.teamId,
+                // OnCall-related fields
+                onCallPolicyId: data.onCallPolicyId,
+                onCallPolicyEscalationRuleId: data.onCallPolicyEscalationRuleId,
+                onCallDutyPolicyExecutionLogTimelineId:
+                  data.onCallDutyPolicyExecutionLogTimelineId,
+                onCallScheduleId: data.onCallScheduleId,
+              },
+              ...(data.forceImmediate !== undefined && {
+                forceImmediate: data.forceImmediate,
+              }),
+            });
+          } catch (err) {
             logger.error(err);
-          });
+          }
         }
       }
 
@@ -800,12 +837,6 @@ export class Service extends DatabaseService<UserNotificationSetting> {
     userId: ObjectID,
     projectId: ObjectID,
   ): Promise<void> {
-    await this.addNotificationSettingIfNotExists(
-      userId,
-      projectId,
-      NotificationSettingEventType.SEND_MONITOR_STATUS_CHANGED_OWNER_NOTIFICATION,
-    );
-
     await this.addNotificationSettingIfNotExists(
       userId,
       projectId,

package/Server/Services/UserService.ts

@@ -353,6 +353,13 @@ export class Service extends DatabaseService<Model> {
       carryForward = users;
     }
 
+    if (updateBy.data.email) {
+      await this.expirePasswordResetTokensForEmailChange({
+        newEmail: updateBy.data.email as Email,
+        existingUsers: carryForward,
+      });
+    }
+
     /*
      * There used to be a guard here refusing to set `enableTwoFactorAuth` on
      * an account with no verified authenticator, on the grounds that doing so
@@ -412,6 +419,94 @@ export class Service extends DatabaseService<Model> {
     return { updateBy, carryForward: carryForward };
   }
 
+  /**
+   * Kill any outstanding password-reset link for every account whose email
+   * address this update is about to change.
+   *
+   * A reset link is a bearer credential addressed to ONE mailbox: holding it
+   * proves only that somebody could read mail sent to the address the account
+   * had when the link was minted. Move the account to a different address and
+   * that proof is about a mailbox the account no longer uses, so the link has
+   * to die with the address it was sent to.
+   *
+   * Without this, changing your email is not the account-recovery step that
+   * users -- and our own "You have changed your email" mail -- take it to be:
+   *
+   *   1. An attacker who can read the victim's mailbox requests a password
+   *      reset and does NOT spend the link. Nothing about the account looks
+   *      wrong, because nothing has changed yet.
+   *   2. The victim notices the mailbox is compromised and moves the account
+   *      to an address the attacker cannot read. This is exactly the advice
+   *      they would be given, and they now believe they are safe.
+   *   3. The attacker spends the link from step 1. `/reset-password` finds the
+   *      row BY TOKEN HASH ALONE -- it never looks at the email address -- so
+   *      it happily sets a new password on the account at its NEW address, and
+   *      the victim is locked out of an account they had just rescued.
+   *
+   * Cleared BEFORE the email is written rather than afterwards in
+   * `onUpdateSuccess`, so there is no instant in which the new address is
+   * committed while a link mailed to the old one is still redeemable. The
+   * price of that ordering is that an email update which subsequently fails
+   * has still burned a pending link, and the user has to request another one.
+   * For a credential that is the right way round.
+   *
+   * Only rows whose address actually CHANGES are touched. Directory syncs
+   * rewrite `email` with the value it already has on every push (see
+   * `App/FeatureSet/Identity/API/SCIM.ts`, which updates whenever either the
+   * email or the name is present), and clearing on those would let an
+   * unrelated SCIM run invalidate a reset link the user is part-way through
+   * using.
+   *
+   * A row that came back with no `email` at all is treated as changed. That
+   * happens when the caller could not read the column, and on a credential the
+   * safe assumption is the one that expires the token.
+   *
+   * NOTE: this runs from `onBeforeUpdate`, so an email write made with
+   * `ignoreHooks: true` would slip past it. Nothing does that today -- the
+   * dashboard, the SCIM endpoints and the admin API all go through the hooks
+   * -- and any new caller that writes `email` hook-free has to clear these two
+   * columns itself.
+   */
+  @CaptureSpan()
+  private async expirePasswordResetTokensForEmailChange(data: {
+    newEmail: Email;
+    existingUsers: Array<Model>;
+  }): Promise<void> {
+    const newEmail: string = data.newEmail.toString().toLowerCase();
+
+    for (const user of data.existingUsers) {
+      if (!user.id) {
+        continue;
+      }
+
+      const currentEmail: string | undefined = user.email
+        ?.toString()
+        .toLowerCase();
+
+      if (currentEmail === newEmail) {
+        continue;
+      }
+
+      await this.updateOneById({
+        id: user.id,
+        data: {
+          resetPasswordToken: null!,
+          resetPasswordExpires: null!,
+        },
+        /*
+         * Root because `resetPasswordToken` is declared with `update: []` --
+         * no permission can write it -- and hook-free because this IS the
+         * update hook; re-entering it would have this write go looking for an
+         * email change of its own.
+         */
+        props: {
+          isRoot: true,
+          ignoreHooks: true,
+        },
+      });
+    }
+  }
+
   @CaptureSpan()
   protected override async onBeforeDelete(
     deleteBy: DeleteBy<Model>,

package/Server/Types/Database/QueryHelper.ts

@@ -656,7 +656,7 @@ export default class QueryHelper {
   }
 
   @CaptureSpan()
-  public static greaterThan<T extends number | Date>(
+  public static greaterThan<T extends number | Date | ObjectID>(
     value: T,
   ): FindWhereProperty<any> {
     const rid: string = Text.generateRandomText(10);
@@ -665,7 +665,7 @@ export default class QueryHelper {
         return `(${alias} > :${rid})`;
       },
       {
-        [rid]: value,
+        [rid]: value instanceof ObjectID ? value.toString() : value,
       },
     ) as FindWhereProperty<any>;
   }

package/Server/Utils/AI/Chat/ObservabilityChatPrompt.ts

@@ -61,14 +61,25 @@ function buildEntityContextGuidance(context: AIChatPageContext): string {
   }
 }
 
+/*
+ * Per-area guidance for list and explorer pages. This text is the only
+ * capability claim in the request that is scoped to the page the user is on,
+ * so the model weighs it against the tool schemas — which means it must never
+ * describe a tool as NARROWER than it is. It previously summarised
+ * query_incidents as "recent incidents, or one by incidentId", and the model
+ * paraphrased that back as "I have no tool to list active incidents, give me
+ * an incidentId" on the very page whose own suggested prompt card asks
+ * "which incidents are currently active or unresolved?". Every affordance an
+ * area page's suggested prompts depend on has to be named here.
+ */
 function buildAreaContextGuidance(type: AIChatPageContextType): string {
   switch (type) {
     case AIChatPageContextType.IncidentsList:
-      return `the incidents list. Questions about "these incidents" or incident activity are answered with query_incidents (recent incidents, or one by incidentId) and search_incidents (free-text search over past incidents).`;
+      return `the incidents list. Questions about "these incidents" or incident activity are answered with query_incidents: pass state="active" to list every incident that is unresolved RIGHT NOW (this drops the time window, so incidents opened weeks ago are still returned), leave state unset for recent activity in a window, or pass incidentId for one incident's full details. search_incidents does free-text search over past incidents.`;
     case AIChatPageContextType.AlertsList:
-      return `the alerts list. Questions about alert activity are answered with query_alerts.`;
+      return `the alerts list. Questions about alert activity are answered with query_alerts: pass state="active" to list every alert that is unresolved RIGHT NOW regardless of age, leave state unset for recent activity in a window, or pass alertId for one alert's full details.`;
     case AIChatPageContextType.MonitorsList:
-      return `the monitors list. Questions about monitors and their status are answered with query_monitors.`;
+      return `the monitors list. Questions about monitors and their status are answered with query_monitors: pass problemsOnly=true to list the monitors that are NOT operational right now, or monitorId for one monitor's details and its recent status timeline.`;
     case AIChatPageContextType.ScheduledMaintenanceList:
       return `the scheduled maintenance list. Questions about maintenance windows are answered with query_scheduled_maintenance (past, ongoing and upcoming events; one event by scheduledMaintenanceId). recent_changes also merges maintenance into a chronological feed alongside other changes.`;
     case AIChatPageContextType.LogsExplorer:

package/Server/Utils/AI/SRE/Insights/Detectors/ExceptionSpikeDetector.ts

@@ -4,6 +4,8 @@ import SortOrder from "../../../../../../Types/BaseDatabase/SortOrder";
 import InBetween from "../../../../../../Types/BaseDatabase/InBetween";
 import PositiveNumber from "../../../../../../Types/PositiveNumber";
 import { JSONObject } from "../../../../../../Types/JSON";
+import IncludesNone from "../../../../../../Types/BaseDatabase/IncludesNone";
+import { NON_ACTIONABLE_ERROR_CLASSES } from "../../../../../../Types/Telemetry/ErrorClass";
 import { LIMIT_PER_PROJECT } from "../../../../../../Types/Database/LimitMax";
 import AIInsightType from "../../../../../../Types/AI/AIInsightType";
 import AIInsightSeverity from "../../../../../../Types/AI/AIInsightSeverity";
@@ -176,6 +178,14 @@ export default class ExceptionSpikeDetector implements InsightDetector {
       firstSeenAt: QueryHelper.lessThan(establishedBefore),
       isResolved: false,
       isArchived: false,
+      /*
+       * Muted fault domains never file an insight. A spike in rejected
+       * logins or malformed input is a traffic story, not a code defect —
+       * and each candidate costs two ClickHouse counts before it can even
+       * be dropped, so excluding them here saves the scan as well as the
+       * LLM budget. Mirrors the Issues list scope.
+       */
+      errorClass: new IncludesNone([...NON_ACTIONABLE_ERROR_CLASSES]),
     };
 
     const exceptions: Array<TelemetryException> =

package/Server/Utils/AI/SRE/Insights/Detectors/NewExceptionDetector.ts

@@ -2,6 +2,8 @@ import ObjectID from "../../../../../../Types/ObjectID";
 import OneUptimeDate from "../../../../../../Types/Date";
 import SortOrder from "../../../../../../Types/BaseDatabase/SortOrder";
 import { JSONObject } from "../../../../../../Types/JSON";
+import IncludesNone from "../../../../../../Types/BaseDatabase/IncludesNone";
+import { NON_ACTIONABLE_ERROR_CLASSES } from "../../../../../../Types/Telemetry/ErrorClass";
 import AIInsightType from "../../../../../../Types/AI/AIInsightType";
 import AIInsightSeverity from "../../../../../../Types/AI/AIInsightSeverity";
 import TelemetryException from "../../../../../../Models/DatabaseModels/TelemetryException";
@@ -171,6 +173,14 @@ export default class NewExceptionDetector implements InsightDetector {
       firstSeenAt: QueryHelper.greaterThanEqualTo(firstSeenSince),
       isResolved: false,
       isArchived: false,
+      /*
+       * Muted fault domains never file an insight. A user-error or an
+       * expected-denial group is not a defect in this code, so triaging it
+       * would spend an LLM budget grant to be told exactly what the
+       * emitting code already declared. Mirrors the Issues list scope, so
+       * the inbox and the list agree on what counts as a real failure.
+       */
+      errorClass: new IncludesNone([...NON_ACTIONABLE_ERROR_CLASSES]),
       occuranceCount: QueryHelper.greaterThanEqualTo(
         NEW_EXCEPTION_MIN_OCCURRENCE_COUNT,
       ),

package/Server/Utils/AI/SRE/Insights/InsightTriageRunner.ts

@@ -215,7 +215,9 @@ export default class InsightTriageRunner {
    *
    * 1. Stamp the verdict onto the exception group row (aiClassification)
    *    so the exceptions list can filter on it and the fix lane can skip
-   *    known-non-defects without re-triaging.
+   *    known-non-defects without re-triaging, and PROMOTE it into the
+   *    platform-wide errorClass when nothing better has claimed that field
+   *    yet (see below).
    * 2. code-fault → route the automatic fix (InsightFixRouting owns every
    *    gate: enableAi, enableInsightFixTasks, budget, readiness, dedupe)
    *    and flip the insight to FixOpened when a run was queued.
@@ -244,6 +246,12 @@ export default class InsightTriageRunner {
      * meantime gets no PR and keeps its terminal status.
      */
     if (insight.telemetryExceptionId) {
+      /*
+       * aiClassification is the AUDIT RECORD of what the LLM said and is
+       * written unconditionally — it must stay readable even when a
+       * stronger source owns errorClass, so a human comparing the two can
+       * see where the model disagreed.
+       */
       await TelemetryExceptionService.updateOneById({
         id: insight.telemetryExceptionId,
         data: {
@@ -251,6 +259,59 @@ export default class InsightTriageRunner {
         },
         props: { isRoot: true },
       });
+
+      /*
+       * errorClass is the ACTED-ON field (Issues scope, emit-path
+       * suppression, fix routing), and its precedence is
+       * declared > manual > ai > default. A class the throwing code
+       * declared for itself is a site-level statement by a developer, and
+       * 'manual' is a human's verdict on this exact group; a probabilistic
+       * LLM answer outranks neither. So the promotion is guarded on the
+       * source still being 'default' — nobody has classified this group.
+       *
+       * The guard is a WHERE clause rather than a read-then-write: the
+       * triage run has been in flight for minutes, so a 'declared' event
+       * ingesting or a human reclassifying between a read and this write
+       * is a real race, and a conditional UPDATE cannot clobber either.
+       * (Same conditional-flip idiom as the ActionRequired -> FixOpened
+       * write below.)
+       */
+      try {
+        const promotedRowCount: number =
+          await TelemetryExceptionService.updateOneBy({
+            query: {
+              _id: insight.telemetryExceptionId.toString(),
+              errorClassSource: "default",
+            },
+            data: {
+              errorClass: classification,
+              errorClassSource: "ai",
+            },
+            props: { isRoot: true },
+          });
+
+        if (promotedRowCount === 0) {
+          logger.debug(
+            `AI insights: kept the existing error class on exception ${insight.telemetryExceptionId.toString()} — a declared or manual classification outranks the AI verdict ${classification}.`,
+          );
+        }
+      } catch (err) {
+        /*
+         * Non-fatal on purpose. This is a secondary, self-healing write: the
+         * audit record above already landed, and live traffic re-stamps
+         * errorClass from the declared class on the group's next occurrence.
+         *
+         * The auto-archive decision below IS user-visible and irreversible by
+         * comparison, so it must not be lost to a failure here. Without this
+         * guard the two are coupled only by statement order, which is exactly
+         * the kind of coupling nobody notices until an incident.
+         */
+        logger.warn(
+          `AI insights: could not promote the error class on exception ${insight.telemetryExceptionId.toString()}: ${
+            err instanceof Error ? err.message : String(err)
+          }`,
+        );
+      }
     }
 
     const currentInsight: AIInsight | null = await AIInsightService.findOneById(

package/Server/Utils/AI/Toolbox/AlertTools.ts

@@ -42,7 +42,7 @@ type AlertStateFilter = "active" | "resolved" | "any";
 export const QueryAlertsTool: ObservabilityTool = {
   name: "query_alerts",
   description:
-    "Query alerts in this project with their current state and severity. To answer 'which alerts are active/firing right now?' pass state='active' — it returns every unresolved alert regardless of age (no time window unless createdWithinHours is passed explicitly). state='resolved' returns only resolved alerts; the default 'any' returns both, limited to the createdWithinHours window. Results are newest-first; the result reports the total match count — pass skip to page through more. Pass alertId to get full details of one alert, including its description and its owners (teams and users).",
+    "Query alerts in this project with their current state and severity. Every row also carries the monitor the alert was raised for, so counting rows by monitor answers 'which monitor or source is the noisiest?'. To answer 'which alerts are active/firing right now?' pass state='active' — it returns every unresolved alert regardless of age (no time window unless createdWithinHours is passed explicitly). state='resolved' returns only resolved alerts; the default 'any' returns both, limited to the createdWithinHours window. Results are newest-first; the result reports the total match count — pass skip to page through more. Pass alertId to get full details of one alert, including its description and its owners (teams and users).",
   inputSchema: {
     type: "object",
     properties: {
@@ -97,6 +97,9 @@ export const QueryAlertsTool: ObservabilityTool = {
           alertSeverity: {
             name: true,
           },
+          monitor: {
+            name: true,
+          },
         },
         props: ctx.props,
       });
@@ -177,6 +180,7 @@ export const QueryAlertsTool: ObservabilityTool = {
               description: alert.description,
               state: alert.currentAlertState?.name,
               severity: alert.alertSeverity?.name,
+              monitor: alert.monitor?.name,
               createdAt: alert.createdAt,
               ownerTeams: ownerTeamNames || undefined,
               ownerUsers: ownerUserNames || undefined,
@@ -286,6 +290,15 @@ export const QueryAlertsTool: ObservabilityTool = {
         alertSeverity: {
           name: true,
         },
+        /*
+         * The monitor an alert was raised for is what turns a list of alerts
+         * into "which source is noisiest" — the question the alerts page
+         * offers as a first-class suggested prompt. Without it the model can
+         * only guess a source from alert titles.
+         */
+        monitor: {
+          name: true,
+        },
       },
       sort: {
         createdAt: SortOrder.Descending,
@@ -309,6 +322,7 @@ export const QueryAlertsTool: ObservabilityTool = {
         title: alert.title,
         state: alert.currentAlertState?.name,
         severity: alert.alertSeverity?.name,
+        monitor: alert.monitor?.name,
         createdAt: alert.createdAt,
       };
     });

package/Server/Utils/Billing/BillingFailureNoticeThrottle.ts

@@ -0,0 +1,80 @@
+import GlobalCache from "../../Infrastructure/GlobalCache";
+import logger from "../Logger";
+import OneUptimeDate from "../../../Types/Date";
+import ObjectID from "../../../Types/ObjectID";
+
+/*
+ * Why this file exists.
+ *
+ * The auto-recharge paths in NotificationService and AIBillingService run
+ * inline on every notification attempt, so when a card is declined they used
+ * to mail every project owner once per paging attempt - a genuine mail bomb
+ * arriving during the exact outage the pages were about.
+ *
+ * The obvious fix - gate the mail on the project's existing
+ * failed...NotificationSentToOwners boolean, which is only cleared by a
+ * SUCCESSFUL recharge - trades a storm for something worse: permanent silence.
+ * A project with no payment method mails once and latches; the owner adds a
+ * card; the card is declined; the recharge can therefore never succeed, so the
+ * latch can never clear, and the owner is never told the new card failed. SMS
+ * and voice paging simply stops working and nobody hears about it.
+ *
+ * So this is a WINDOW, not a latch. At most one failure notice per project per
+ * day, per kind of failure, and the window expires on its own whether or not
+ * anything is ever fixed. A storm collapses to one email a day; a new and
+ * newly-actionable failure is still reported within a day.
+ *
+ * It FAILS OPEN. If Redis is unreachable the notice is sent, which is the same
+ * behaviour the product had before any of this existed. Failing closed would
+ * mean a Redis blip could silently suppress "we cannot page anyone for you",
+ * and there is no worse thing for a monitoring product to lose.
+ */
+
+const BILLING_FAILURE_NOTICE_NAMESPACE: string = "billing-failure-notice";
+
+export enum BillingFailureNoticeKind {
+  SmsAndCallRechargeFailed = "sms-and-call-recharge-failed",
+  AiCreditRechargeFailed = "ai-credit-recharge-failed",
+}
+
+export type ShouldSendBillingFailureNoticeFunction = (data: {
+  projectId: ObjectID;
+  kind: BillingFailureNoticeKind;
+}) => Promise<boolean>;
+
+/*
+ * Claims the day's window for this (project, kind). True means "you own it,
+ * send the email"; false means somebody already told them today.
+ */
+export const shouldSendBillingFailureNotice: ShouldSendBillingFailureNoticeFunction =
+  async (data: {
+    projectId: ObjectID;
+    kind: BillingFailureNoticeKind;
+  }): Promise<boolean> => {
+    const key: string = `${data.projectId.toString()}-${data.kind}`;
+
+    try {
+      /*
+       * SET NX rather than read-then-write: several notification attempts
+       * failing at once is the normal case here, not the edge one, and a
+       * check-then-act would let every one of them decide it was first.
+       */
+      return await GlobalCache.setStringIfNotExists(
+        BILLING_FAILURE_NOTICE_NAMESPACE,
+        key,
+        ObjectID.generate().toString(),
+        {
+          expiresInSeconds: OneUptimeDate.getSecondsInDays(1),
+        },
+      );
+    } catch (err) {
+      logger.warn(
+        `Billing failure notice throttle: the shared cache is unavailable, so the 24h window for project ${data.projectId.toString()} (${
+          data.kind
+        }) is not enforced. Sending, because a suppressed billing failure is worse than a duplicated one.`,
+      );
+      logger.warn(err);
+
+      return true;
+    }
+  };