@oneuptime/common 12.0.32 → 12.0.33

package/Server/Utils/AI/SRE/Insights/Detectors/ExceptionSpikeDetector.ts

@@ -4,6 +4,8 @@ import SortOrder from "../../../../../../Types/BaseDatabase/SortOrder";
 import InBetween from "../../../../../../Types/BaseDatabase/InBetween";
 import PositiveNumber from "../../../../../../Types/PositiveNumber";
 import { JSONObject } from "../../../../../../Types/JSON";
+import IncludesNone from "../../../../../../Types/BaseDatabase/IncludesNone";
+import { NON_ACTIONABLE_ERROR_CLASSES } from "../../../../../../Types/Telemetry/ErrorClass";
 import { LIMIT_PER_PROJECT } from "../../../../../../Types/Database/LimitMax";
 import AIInsightType from "../../../../../../Types/AI/AIInsightType";
 import AIInsightSeverity from "../../../../../../Types/AI/AIInsightSeverity";
@@ -176,6 +178,14 @@ export default class ExceptionSpikeDetector implements InsightDetector {
       firstSeenAt: QueryHelper.lessThan(establishedBefore),
       isResolved: false,
       isArchived: false,
+      /*
+       * Muted fault domains never file an insight. A spike in rejected
+       * logins or malformed input is a traffic story, not a code defect —
+       * and each candidate costs two ClickHouse counts before it can even
+       * be dropped, so excluding them here saves the scan as well as the
+       * LLM budget. Mirrors the Issues list scope.
+       */
+      errorClass: new IncludesNone([...NON_ACTIONABLE_ERROR_CLASSES]),
     };
 
     const exceptions: Array<TelemetryException> =

package/Server/Utils/AI/SRE/Insights/Detectors/NewExceptionDetector.ts

@@ -2,6 +2,8 @@ import ObjectID from "../../../../../../Types/ObjectID";
 import OneUptimeDate from "../../../../../../Types/Date";
 import SortOrder from "../../../../../../Types/BaseDatabase/SortOrder";
 import { JSONObject } from "../../../../../../Types/JSON";
+import IncludesNone from "../../../../../../Types/BaseDatabase/IncludesNone";
+import { NON_ACTIONABLE_ERROR_CLASSES } from "../../../../../../Types/Telemetry/ErrorClass";
 import AIInsightType from "../../../../../../Types/AI/AIInsightType";
 import AIInsightSeverity from "../../../../../../Types/AI/AIInsightSeverity";
 import TelemetryException from "../../../../../../Models/DatabaseModels/TelemetryException";
@@ -171,6 +173,14 @@ export default class NewExceptionDetector implements InsightDetector {
       firstSeenAt: QueryHelper.greaterThanEqualTo(firstSeenSince),
       isResolved: false,
       isArchived: false,
+      /*
+       * Muted fault domains never file an insight. A user-error or an
+       * expected-denial group is not a defect in this code, so triaging it
+       * would spend an LLM budget grant to be told exactly what the
+       * emitting code already declared. Mirrors the Issues list scope, so
+       * the inbox and the list agree on what counts as a real failure.
+       */
+      errorClass: new IncludesNone([...NON_ACTIONABLE_ERROR_CLASSES]),
       occuranceCount: QueryHelper.greaterThanEqualTo(
         NEW_EXCEPTION_MIN_OCCURRENCE_COUNT,
       ),

package/Server/Utils/AI/SRE/Insights/InsightTriageRunner.ts

@@ -215,7 +215,9 @@ export default class InsightTriageRunner {
    *
    * 1. Stamp the verdict onto the exception group row (aiClassification)
    *    so the exceptions list can filter on it and the fix lane can skip
-   *    known-non-defects without re-triaging.
+   *    known-non-defects without re-triaging, and PROMOTE it into the
+   *    platform-wide errorClass when nothing better has claimed that field
+   *    yet (see below).
    * 2. code-fault → route the automatic fix (InsightFixRouting owns every
    *    gate: enableAi, enableInsightFixTasks, budget, readiness, dedupe)
    *    and flip the insight to FixOpened when a run was queued.
@@ -244,6 +246,12 @@ export default class InsightTriageRunner {
      * meantime gets no PR and keeps its terminal status.
      */
     if (insight.telemetryExceptionId) {
+      /*
+       * aiClassification is the AUDIT RECORD of what the LLM said and is
+       * written unconditionally — it must stay readable even when a
+       * stronger source owns errorClass, so a human comparing the two can
+       * see where the model disagreed.
+       */
       await TelemetryExceptionService.updateOneById({
         id: insight.telemetryExceptionId,
         data: {
@@ -251,6 +259,59 @@ export default class InsightTriageRunner {
         },
         props: { isRoot: true },
       });
+
+      /*
+       * errorClass is the ACTED-ON field (Issues scope, emit-path
+       * suppression, fix routing), and its precedence is
+       * declared > manual > ai > default. A class the throwing code
+       * declared for itself is a site-level statement by a developer, and
+       * 'manual' is a human's verdict on this exact group; a probabilistic
+       * LLM answer outranks neither. So the promotion is guarded on the
+       * source still being 'default' — nobody has classified this group.
+       *
+       * The guard is a WHERE clause rather than a read-then-write: the
+       * triage run has been in flight for minutes, so a 'declared' event
+       * ingesting or a human reclassifying between a read and this write
+       * is a real race, and a conditional UPDATE cannot clobber either.
+       * (Same conditional-flip idiom as the ActionRequired -> FixOpened
+       * write below.)
+       */
+      try {
+        const promotedRowCount: number =
+          await TelemetryExceptionService.updateOneBy({
+            query: {
+              _id: insight.telemetryExceptionId.toString(),
+              errorClassSource: "default",
+            },
+            data: {
+              errorClass: classification,
+              errorClassSource: "ai",
+            },
+            props: { isRoot: true },
+          });
+
+        if (promotedRowCount === 0) {
+          logger.debug(
+            `AI insights: kept the existing error class on exception ${insight.telemetryExceptionId.toString()} — a declared or manual classification outranks the AI verdict ${classification}.`,
+          );
+        }
+      } catch (err) {
+        /*
+         * Non-fatal on purpose. This is a secondary, self-healing write: the
+         * audit record above already landed, and live traffic re-stamps
+         * errorClass from the declared class on the group's next occurrence.
+         *
+         * The auto-archive decision below IS user-visible and irreversible by
+         * comparison, so it must not be lost to a failure here. Without this
+         * guard the two are coupled only by statement order, which is exactly
+         * the kind of coupling nobody notices until an incident.
+         */
+        logger.warn(
+          `AI insights: could not promote the error class on exception ${insight.telemetryExceptionId.toString()}: ${
+            err instanceof Error ? err.message : String(err)
+          }`,
+        );
+      }
     }
 
     const currentInsight: AIInsight | null = await AIInsightService.findOneById(

package/Server/Utils/AI/Toolbox/AlertTools.ts

@@ -42,7 +42,7 @@ type AlertStateFilter = "active" | "resolved" | "any";
 export const QueryAlertsTool: ObservabilityTool = {
   name: "query_alerts",
   description:
-    "Query alerts in this project with their current state and severity. To answer 'which alerts are active/firing right now?' pass state='active' — it returns every unresolved alert regardless of age (no time window unless createdWithinHours is passed explicitly). state='resolved' returns only resolved alerts; the default 'any' returns both, limited to the createdWithinHours window. Results are newest-first; the result reports the total match count — pass skip to page through more. Pass alertId to get full details of one alert, including its description and its owners (teams and users).",
+    "Query alerts in this project with their current state and severity. Every row also carries the monitor the alert was raised for, so counting rows by monitor answers 'which monitor or source is the noisiest?'. To answer 'which alerts are active/firing right now?' pass state='active' — it returns every unresolved alert regardless of age (no time window unless createdWithinHours is passed explicitly). state='resolved' returns only resolved alerts; the default 'any' returns both, limited to the createdWithinHours window. Results are newest-first; the result reports the total match count — pass skip to page through more. Pass alertId to get full details of one alert, including its description and its owners (teams and users).",
   inputSchema: {
     type: "object",
     properties: {
@@ -97,6 +97,9 @@ export const QueryAlertsTool: ObservabilityTool = {
           alertSeverity: {
             name: true,
           },
+          monitor: {
+            name: true,
+          },
         },
         props: ctx.props,
       });
@@ -177,6 +180,7 @@ export const QueryAlertsTool: ObservabilityTool = {
               description: alert.description,
               state: alert.currentAlertState?.name,
               severity: alert.alertSeverity?.name,
+              monitor: alert.monitor?.name,
               createdAt: alert.createdAt,
               ownerTeams: ownerTeamNames || undefined,
               ownerUsers: ownerUserNames || undefined,
@@ -286,6 +290,15 @@ export const QueryAlertsTool: ObservabilityTool = {
         alertSeverity: {
           name: true,
         },
+        /*
+         * The monitor an alert was raised for is what turns a list of alerts
+         * into "which source is noisiest" — the question the alerts page
+         * offers as a first-class suggested prompt. Without it the model can
+         * only guess a source from alert titles.
+         */
+        monitor: {
+          name: true,
+        },
       },
       sort: {
         createdAt: SortOrder.Descending,
@@ -309,6 +322,7 @@ export const QueryAlertsTool: ObservabilityTool = {
         title: alert.title,
         state: alert.currentAlertState?.name,
         severity: alert.alertSeverity?.name,
+        monitor: alert.monitor?.name,
         createdAt: alert.createdAt,
       };
     });

package/Server/Utils/Billing/BillingFailureNoticeThrottle.ts

@@ -0,0 +1,80 @@
+import GlobalCache from "../../Infrastructure/GlobalCache";
+import logger from "../Logger";
+import OneUptimeDate from "../../../Types/Date";
+import ObjectID from "../../../Types/ObjectID";
+
+/*
+ * Why this file exists.
+ *
+ * The auto-recharge paths in NotificationService and AIBillingService run
+ * inline on every notification attempt, so when a card is declined they used
+ * to mail every project owner once per paging attempt - a genuine mail bomb
+ * arriving during the exact outage the pages were about.
+ *
+ * The obvious fix - gate the mail on the project's existing
+ * failed...NotificationSentToOwners boolean, which is only cleared by a
+ * SUCCESSFUL recharge - trades a storm for something worse: permanent silence.
+ * A project with no payment method mails once and latches; the owner adds a
+ * card; the card is declined; the recharge can therefore never succeed, so the
+ * latch can never clear, and the owner is never told the new card failed. SMS
+ * and voice paging simply stops working and nobody hears about it.
+ *
+ * So this is a WINDOW, not a latch. At most one failure notice per project per
+ * day, per kind of failure, and the window expires on its own whether or not
+ * anything is ever fixed. A storm collapses to one email a day; a new and
+ * newly-actionable failure is still reported within a day.
+ *
+ * It FAILS OPEN. If Redis is unreachable the notice is sent, which is the same
+ * behaviour the product had before any of this existed. Failing closed would
+ * mean a Redis blip could silently suppress "we cannot page anyone for you",
+ * and there is no worse thing for a monitoring product to lose.
+ */
+
+const BILLING_FAILURE_NOTICE_NAMESPACE: string = "billing-failure-notice";
+
+export enum BillingFailureNoticeKind {
+  SmsAndCallRechargeFailed = "sms-and-call-recharge-failed",
+  AiCreditRechargeFailed = "ai-credit-recharge-failed",
+}
+
+export type ShouldSendBillingFailureNoticeFunction = (data: {
+  projectId: ObjectID;
+  kind: BillingFailureNoticeKind;
+}) => Promise<boolean>;
+
+/*
+ * Claims the day's window for this (project, kind). True means "you own it,
+ * send the email"; false means somebody already told them today.
+ */
+export const shouldSendBillingFailureNotice: ShouldSendBillingFailureNoticeFunction =
+  async (data: {
+    projectId: ObjectID;
+    kind: BillingFailureNoticeKind;
+  }): Promise<boolean> => {
+    const key: string = `${data.projectId.toString()}-${data.kind}`;
+
+    try {
+      /*
+       * SET NX rather than read-then-write: several notification attempts
+       * failing at once is the normal case here, not the edge one, and a
+       * check-then-act would let every one of them decide it was first.
+       */
+      return await GlobalCache.setStringIfNotExists(
+        BILLING_FAILURE_NOTICE_NAMESPACE,
+        key,
+        ObjectID.generate().toString(),
+        {
+          expiresInSeconds: OneUptimeDate.getSecondsInDays(1),
+        },
+      );
+    } catch (err) {
+      logger.warn(
+        `Billing failure notice throttle: the shared cache is unavailable, so the 24h window for project ${data.projectId.toString()} (${
+          data.kind
+        }) is not enforced. Sending, because a suppressed billing failure is worse than a duplicated one.`,
+      );
+      logger.warn(err);
+
+      return true;
+    }
+  };

package/Server/Utils/DataSource/HttpFetch.ts

@@ -6,6 +6,7 @@ import {
   DATA_SOURCE_MAX_RESPONSE_SIZE_IN_BYTES,
   DATA_SOURCE_QUERY_TIMEOUT_IN_MS,
 } from "../../../Types/DataSource/DataSourceLimits";
+import OutboundUserAgent from "../OutboundUserAgent";
 import DataSourceEgressGuard, { EgressGuardOptions } from "./EgressGuard";
 import { DataSourceConnectionSettings } from "./Types";
 
@@ -20,7 +21,11 @@ import { DataSourceConnectionSettings } from "./Types";
  *    window while TLS still verifies against the original hostname;
  *  - refuses redirects (maxRedirects: 0) — a 3xx from the target could
  *    otherwise bounce the request to an unvalidated host;
- *  - caps response size and wall-clock time.
+ *  - caps response size and wall-clock time;
+ *  - identifies itself with OneUptime's User-Agent unless the caller set
+ *    one. Left to axios, every request would go out as `axios/<version>`,
+ *    which WAFs and bot-management products block by default — a target
+ *    that answers 403 to a bare library UA never gets to see the query.
  *
  * Uses axios directly (not Common/Utils/API.fetch) because the shared
  * wrapper does not expose maxContentLength, and the response-size cap is a
@@ -99,7 +104,9 @@ export default class DataSourceHttpFetch {
         request.egressOptions,
       );
 
-    const headers: Dictionary<string> = { ...(request.headers || {}) };
+    const headers: Dictionary<string> = OutboundUserAgent.withDefault(
+      request.headers,
+    );
 
     let data: string | URLSearchParams | undefined = undefined;
     if (request.method === "POST" && request.body !== undefined) {

package/Server/Utils/EmailRollup/EmailRollupConstants.ts

@@ -0,0 +1,98 @@
+/*
+ * Every tuning knob the owner-email burst rollup has, in one file.
+ *
+ * The feature is a no-op below its threshold: a project that produces three
+ * owner emails a day keeps producing three owner emails a day, from the same
+ * code path and the same template. Only an address that is already being
+ * flooded sees a change. That property is entirely a function of the numbers
+ * below, which is why they live together rather than being scattered across
+ * the writer, the flush runner and the cron.
+ *
+ * A self-hoster who disagrees with any of these judgements changes behaviour
+ * with a one-line patch here, and cloud and self-hosted stay identical
+ * because there is no environment variable for the two to diverge on.
+ */
+
+export const ROLLUP_JOB_NAME: string = "EmailRollup:FlushDueRollups";
+export const ROLLUP_SWEEP_LOCK_NAMESPACE: string = "Workers.Cron";
+
+/*
+ * The FIRST BURST_THRESHOLD rollup-eligible owner emails for one
+ * (project, user, address, category) inside BURST_WINDOW_MINUTES are sent
+ * immediately; the next one and everything after it is coalesced. Four is
+ * chosen so one incident's whole normal lifecycle (created -> acknowledged ->
+ * resolved) plus one more event stays immediate; the fifth event in ten
+ * minutes is where a human starts noticing.
+ */
+export const BURST_THRESHOLD: number = 4;
+export const BURST_WINDOW_MINUTES: number = 10;
+
+/*
+ * A deferred item is due once it is this old. Also the claim-epoch length,
+ * which is what makes "at most 12 rollups per hour per address" true rather
+ * than aspirational.
+ *
+ * CLAIM_EPOCH_MINUTES === FLUSH_AFTER_MINUTES is load-bearing. A bucket
+ * becomes due only when its oldest pending item is FLUSH_AFTER_MINUTES old,
+ * and a flush stamps every pending row for the bucket, so a legitimate
+ * consecutive flush of the same bucket is always at least FLUSH_AFTER_MINUTES
+ * after the previous one and therefore always lands in a LATER epoch. The
+ * unique index on the batch table never blocks a legitimate flush, and it
+ * hard-caps a bucket at 60 / CLAIM_EPOCH_MINUTES flushes per hour. Pull the
+ * two numbers apart and one of those two guarantees breaks.
+ */
+export const FLUSH_AFTER_MINUTES: number = 5;
+export const CLAIM_EPOCH_MINUTES: number = 5;
+
+export const MAX_ITEMS_PER_ROLLUP: number = 500; // claimed + stamped per flush
+export const MAX_ROWS_IN_ROLLUP: number = 100; // folded rows rendered
+export const MAX_ITEMS_SCANNED_PER_TICK: number = 5000;
+export const MAX_BUCKETS_PER_TICK: number = 50;
+
+/*
+ * How many pages of MAX_ITEMS_SCANNED_PER_TICK the sweep will read looking for
+ * distinct recipients before giving up for this tick.
+ *
+ * One page is not enough. The scan is ordered oldest-first, so a single
+ * recipient with more pending items than one page can fill it entirely, and
+ * every other tenant's due rollup would then be invisible for as long as that
+ * saturation lasted. Paging past a hog costs a handful of indexed reads in a
+ * case that should never happen, and removes the possibility that one project
+ * stalls the whole fleet.
+ */
+export const MAX_DISCOVERY_PAGES_PER_TICK: number = 4;
+
+/*
+ * A hard ceiling on one rollup send.
+ *
+ * MailService.sendMail is an HTTP POST to the notification service with no
+ * timeout of its own, so a hung connection would hang this await forever. That
+ * matters more here than anywhere else the product sends mail, because the
+ * sweep holds a Redis mutex whose lock is auto-refreshed while it is held: a
+ * single wedged send would therefore stop EVERY replica from flushing ANY
+ * recipient's rollup, indefinitely, while the queue behind it grew and its
+ * oldest rows aged out of retention unsent. One minute is far longer than a
+ * healthy send and far shorter than the sweep's budget.
+ */
+export const ROLLUP_SEND_TIMEOUT_MS: number = 60 * 1000;
+
+export const ROLLUP_SWEEP_BUDGET_MS: number = 3 * 60 * 1000;
+export const ROLLUP_JOB_TIMEOUT_MS: number = 4 * 60 * 1000; // below JobDictionary's 5-min default
+export const ROLLUP_SWEEP_LOCK_TIMEOUT_MS: number = 5 * 60 * 1000; // outlives the job timeout
+export const ROLLUP_ITEM_RETENTION_DAYS: number = 7;
+export const ROLLUP_BATCH_RETENTION_DAYS: number = 30;
+export const ROLLUP_SUBJECT_MAX_LENGTH: number = 500; // ColumnLength.LongText
+export const ROLLUP_SUBJECT_LEAD_TITLE_MAX: number = 80;
+export const ROLLUP_PROJECT_NAME_MAX: number = 60;
+
+/*
+ * How many category counts the SUBJECT line spells out before it gives up and
+ * says "+N more".
+ *
+ * A subject is read in a list, at a glance, and most clients show somewhere
+ * around 60-90 characters of it. "12 notifications" alone says nothing about
+ * whether this is a probe flapping or production burning, and all eleven
+ * categories would be truncated by the client at a point this code does not
+ * control. Three is what fits after the project name and the count.
+ */
+export const ROLLUP_SUBJECT_MAX_CATEGORIES: number = 3;