@oneuptime/common 12.0.5 → 12.0.6

package/Server/Utils/AI/SRE/Insights/Detectors/NewExceptionDetector.ts

@@ -9,7 +9,13 @@ import Service from "../../../../../../Models/DatabaseModels/Service";
 import TelemetryExceptionService from "../../../../../Services/TelemetryExceptionService";
 import ServiceService from "../../../../../Services/ServiceService";
 import QueryHelper from "../../../../../Types/Database/QueryHelper";
+import logger from "../../../../Logger";
 import { sanitizeExceptionMessage } from "../../../../Telemetry/ExceptionSanitizer";
+import {
+  EXCEPTION_LABEL_MESSAGE_MAX_LENGTH,
+  buildExceptionFailureModeKey,
+  buildExceptionLabel,
+} from "./ExceptionIdentity";
 import {
   InsightCandidate,
   InsightDetector,
@@ -23,6 +29,18 @@ import {
  * the exception groups table carries firstSeenAt and the lifetime
  * occurrence count, so no ClickHouse scan is needed at all. Deterministic —
  * no LLM anywhere in this detector.
+ *
+ * ONE INSIGHT PER FAILURE MODE, NOT PER EXCEPTION GROUP. Group identity at
+ * ingest hashes the normalized STACK TRACE alongside the type and message, so
+ * one logical failure raised from several call sites is several groups. This
+ * detector used to file one insight per group, which filed the same finding
+ * over and over — and because the title was the exceptionType alone, the rows
+ * were not merely duplicated but indistinguishable (a service whose
+ * instrumentation reports `exception.type = "401"` produced a wall of
+ * identical "New exception: 401 in api" rows). Candidates are therefore
+ * bucketed on the (entity, type, normalized message) failure mode, counts are
+ * summed across the member groups, and the insight is fingerprinted on that
+ * bucket — see ExceptionIdentity.
  */
 
 /*
@@ -36,38 +54,72 @@ export const NEW_EXCEPTION_LOOKBACK_HOURS: number = 24;
 /*
  * A single occurrence is noise (a deploy hiccup, a one-off bad request).
  * Three occurrences means the failure path is being re-hit — worth a
- * human's attention.
+ * human's attention. Applied per exception GROUP, before bucketing: a
+ * failure mode must have at least one group that is genuinely recurring,
+ * rather than qualifying by adding up single-occurrence stack variants.
  */
 export const NEW_EXCEPTION_MIN_OCCURRENCE_COUNT: number = 3;
 
 /*
  * At 50+ occurrences inside its first day the exception is hammering a hot
- * path — escalate to High so it sorts above the long tail.
+ * path — escalate to High so it sorts above the long tail. Measured on the
+ * failure mode's summed count, which is what the insight reports.
  */
 export const NEW_EXCEPTION_HIGH_SEVERITY_OCCURRENCE_COUNT: number = 50;
 
 /*
- * Upper bound on candidates per scan tick. Mirrors the TopExceptionsTool
- * cap — a project melting down does not need 500 insights, and the scanner
+ * How many exception GROUPS one tick reads. Higher than the candidate cap
+ * because groups collapse into failure modes: a single noisy throw site can
+ * easily own dozens of stack-trace variants, and reading only 25 of them
+ * would let one loud failure mode crowd every other one out of the tick. It
+ * is one indexed Postgres query either way.
+ */
+export const NEW_EXCEPTION_GROUP_SCAN_LIMIT: number = 100;
+
+/*
+ * Upper bound on candidates per scan tick, applied AFTER bucketing — a
+ * project melting down does not need 500 insights, and the scanner
  * additionally caps new insights per tick.
  */
 export const NEW_EXCEPTION_CANDIDATE_LIMIT: number = 25;
 
 // Titles must stay scannable in a list — truncate long exception messages.
-export const NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH: number = 80;
+export const NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH: number =
+  EXCEPTION_LABEL_MESSAGE_MAX_LENGTH;
 
 export interface NewExceptionDecision {
   qualifies: boolean;
   severity: AIInsightSeverity;
 }
 
+/*
+ * One failure mode: the exception groups that share an (entity, type,
+ * normalized message) identity, rolled up into the numbers the insight
+ * reports.
+ */
+export interface NewExceptionFailureMode {
+  failureModeKey: string;
+  // The loudest member group — what the insight links to and labels itself by.
+  representative: TelemetryException;
+  // The representative's id, narrowed once at bucket time.
+  representativeExceptionId: ObjectID;
+  // Summed lifetime occurrences across every member group.
+  totalOccurrenceCount: number;
+  // How many distinct exception groups (stack variants) rolled up here.
+  groupCount: number;
+  // Earliest firstSeenAt across the member groups.
+  firstSeenAt: Date | undefined;
+  // The representative's message, sanitized once.
+  safeMessage: string;
+}
+
 export default class NewExceptionDetector implements InsightDetector {
   public insightType: AIInsightType = AIInsightType.NewException;
 
   /*
-   * Pure decision: does a lifetime occurrence count qualify, and how
-   * urgent is it. The query already filters on the minimum count — this
-   * re-check keeps the rule in one unit-testable place.
+   * Pure decision: does an occurrence count qualify, and how urgent is it.
+   * The query already filters on the minimum count — this re-check keeps the
+   * rule in one unit-testable place.
    */
   public static evaluateNewException(
     occurrenceCount: number,
@@ -82,33 +134,28 @@ export default class NewExceptionDetector implements InsightDetector {
   }
 
   /*
-   * Human label for the exception: the type when present, else the first
-   * NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH characters of the message.
+   * Human label for the exception: the type AND the message, because either
+   * one alone collides across findings (see ExceptionIdentity).
    */
   public static buildExceptionLabel(
     exceptionType: string | undefined,
     message: string | undefined,
   ): string {
-    if (exceptionType && exceptionType.trim().length > 0) {
-      return exceptionType.trim();
-    }
-    const trimmedMessage: string = (message || "").trim();
-    if (trimmedMessage.length === 0) {
-      return "Unknown exception";
-    }
-    if (trimmedMessage.length <= NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH) {
-      return trimmedMessage;
-    }
-    return `${trimmedMessage.slice(0, NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH)}…`;
+    return buildExceptionLabel(
+      exceptionType,
+      message,
+      NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH,
+    );
   }
 
   /*
-   * Stable dedupe key. Keyed on the exception GROUP id (not the message):
-   * the group id is the ingest-side dedupe unit, so the insight refreshes
-   * instead of duplicating across scan ticks.
+   * Stable dedupe key. Keyed on the FAILURE MODE (entity + exception type +
+   * normalized message), never on the exception group id: group identity
+   * includes the stack trace, so one failure raised from several call sites
+   * would otherwise file one insight per call site.
    */
-  public static buildFingerprint(telemetryExceptionId: string): string {
-    return `new-exception:${telemetryExceptionId}`;
+  public static buildFingerprint(failureModeKey: string): string {
+    return `new-exception:${failureModeKey}`;
   }
 
   public async detect(
@@ -132,6 +179,13 @@ export default class NewExceptionDetector implements InsightDetector {
     /*
      * Root read with an explicit projectId in the query — the scanner is a
      * server-side worker (monitor-worker precedent), never a user ACL.
+     *
+     * firstSeenAt is a deterministic tie-break on the LIMIT boundary:
+     * occuranceCount is incremented on every ingested event, so the long
+     * tail of a noisy endpoint sits densely tied right at the cut and an
+     * untie-broken ORDER BY hands back an arbitrary, shifting subset each
+     * tick. Bucketing already keeps that churn from filing duplicates; the
+     * tie-break keeps the tick itself reproducible.
      */
     const exceptions: Array<TelemetryException> =
       await TelemetryExceptionService.findBy({
@@ -146,44 +200,43 @@ export default class NewExceptionDetector implements InsightDetector {
         },
         sort: {
           occuranceCount: SortOrder.Descending,
+          firstSeenAt: SortOrder.Ascending,
         },
-        limit: NEW_EXCEPTION_CANDIDATE_LIMIT,
+        limit: NEW_EXCEPTION_GROUP_SCAN_LIMIT,
         skip: 0,
         props: { isRoot: true },
       });
 
+    const failureModes: Array<NewExceptionFailureMode> =
+      NewExceptionDetector.bucketIntoFailureModes(exceptions);
+
     const candidates: Array<InsightCandidate> = [];
 
-    for (const exception of exceptions) {
-      const occurrenceCount: number = exception.occuranceCount || 0;
+    // Service names are resolved once per entity, not once per group.
+    const serviceNameCache: Map<string, string | null> = new Map();
+
+    for (const failureMode of failureModes) {
       const decision: NewExceptionDecision =
-        NewExceptionDetector.evaluateNewException(occurrenceCount);
+        NewExceptionDetector.evaluateNewException(
+          failureMode.totalOccurrenceCount,
+        );
 
-      if (!decision.qualifies || !exception.id) {
+      if (!decision.qualifies) {
         continue;
       }
 
       const serviceName: string | null = await this.resolveServiceName(
-        exception.primaryEntityId,
-      );
-
-      /*
-       * The message flows into the insight title/detail/evidence, and from
-       * there into the triage LLM prompt — sanitize it once here so no
-       * interpolated user data (IDs, emails, domains) rides along. The raw
-       * message stays on the exception row for the dashboard.
-       */
-      const safeMessage: string = sanitizeExceptionMessage(
-        exception.message || "",
+        failureMode.representative.primaryEntityId,
+        serviceNameCache,
       );
 
       const label: string = NewExceptionDetector.buildExceptionLabel(
-        exception.exceptionType,
-        safeMessage,
+        failureMode.representative.exceptionType,
+        failureMode.safeMessage,
       );
 
-      const firstSeenAtIso: string | undefined = exception.firstSeenAt
-        ? exception.firstSeenAt.toISOString()
+      const firstSeenAtIso: string | undefined = failureMode.firstSeenAt
+        ? failureMode.firstSeenAt.toISOString()
         : undefined;
 
       const detailLines: Array<string> = [
@@ -191,59 +244,205 @@ export default class NewExceptionDetector implements InsightDetector {
         "",
         `- Exception: \`${label}\``,
       ];
-      if (safeMessage) {
-        detailLines.push(`- Message: ${safeMessage}`);
+      if (failureMode.safeMessage) {
+        detailLines.push(`- Message: ${failureMode.safeMessage}`);
       }
       if (serviceName) {
         detailLines.push(`- Service: ${serviceName}`);
       }
       detailLines.push(
         `- First seen: ${firstSeenAtIso || "unknown"} (within the last ${NEW_EXCEPTION_LOOKBACK_HOURS}h)`,
-        `- Occurrences so far: ${occurrenceCount}`,
+        `- Occurrences so far: ${failureMode.totalOccurrenceCount}`,
       );
+      if (failureMode.groupCount > 1) {
+        detailLines.push(
+          `- Raised from ${failureMode.groupCount} distinct stack traces (same exception type and message — counted together as one failure mode)`,
+        );
+      }
 
       candidates.push({
         insightType: AIInsightType.NewException,
         fingerprint: NewExceptionDetector.buildFingerprint(
-          exception.id.toString(),
+          failureMode.failureModeKey,
         ),
         title: `New exception: ${label}${serviceName ? ` in ${serviceName}` : ""}`,
         detailMarkdown: detailLines.join("\n"),
         severity: decision.severity,
         serviceName: serviceName || undefined,
-        telemetryServiceId: serviceName ? exception.primaryEntityId : undefined,
-        telemetryExceptionId: exception.id,
+        telemetryServiceId: serviceName
+          ? failureMode.representative.primaryEntityId
+          : undefined,
+        telemetryExceptionId: failureMode.representativeExceptionId,
         evidence: {
           exception: {
-            exceptionMessage: safeMessage,
-            exceptionType: exception.exceptionType,
-            totalOccurrenceCount: occurrenceCount,
+            exceptionMessage: failureMode.safeMessage,
+            exceptionType: failureMode.representative.exceptionType,
+            totalOccurrenceCount: failureMode.totalOccurrenceCount,
+            distinctExceptionGroupCount: failureMode.groupCount,
             firstSeenAt: firstSeenAtIso,
           },
         },
       });
+
+      if (candidates.length >= NEW_EXCEPTION_CANDIDATE_LIMIT) {
+        // No silent caps — say what this tick left on the floor.
+        if (failureModes.length > candidates.length) {
+          logger.debug(
+            `AI Insights: NewException capped at ${NEW_EXCEPTION_CANDIDATE_LIMIT} candidates for project ${context.projectId.toString()} — ${
+              failureModes.length - candidates.length
+            } further failure mode(s) this tick were not emitted. They re-surface on later ticks while their exceptions stay inside the ${NEW_EXCEPTION_LOOKBACK_HOURS}h window.`,
+          );
+        }
+        break;
+      }
     }
 
     return candidates;
   }
 
+  /*
+   * Roll the tick's exception groups up into failure modes, loudest first.
+   *
+   * The representative is the group with the highest lifetime occurrence
+   * count (ties broken by the earliest firstSeenAt, then by id, so the choice
+   * is deterministic rather than dependent on row order). It supplies the
+   * label and the exception the insight links to — but NOT the fingerprint,
+   * which is the bucket key, so a representative that changes as counts move
+   * can never file a second insight for the same failure mode.
+   */
+  public static bucketIntoFailureModes(
+    exceptions: Array<TelemetryException>,
+  ): Array<NewExceptionFailureMode> {
+    const buckets: Map<string, NewExceptionFailureMode> = new Map();
+
+    for (const exception of exceptions) {
+      const exceptionId: ObjectID | undefined = exception.id || undefined;
+
+      if (!exceptionId) {
+        continue;
+      }
+
+      const failureModeKey: string = buildExceptionFailureModeKey({
+        primaryEntityId: exception.primaryEntityId,
+        exceptionType: exception.exceptionType,
+        message: exception.message,
+      });
+
+      const occurrenceCount: number = exception.occuranceCount || 0;
+      const existing: NewExceptionFailureMode | undefined =
+        buckets.get(failureModeKey);
+
+      if (!existing) {
+        buckets.set(failureModeKey, {
+          failureModeKey: failureModeKey,
+          representative: exception,
+          representativeExceptionId: exceptionId,
+          totalOccurrenceCount: occurrenceCount,
+          groupCount: 1,
+          firstSeenAt: exception.firstSeenAt,
+          /*
+           * The message flows into the insight title/detail/evidence, and
+           * from there into the triage LLM prompt — sanitize it once here so
+           * no interpolated user data (IDs, emails, domains) rides along. The
+           * raw message stays on the exception row for the dashboard.
+           */
+          safeMessage: sanitizeExceptionMessage(exception.message || ""),
+        });
+        continue;
+      }
+
+      existing.totalOccurrenceCount += occurrenceCount;
+      existing.groupCount++;
+
+      if (
+        exception.firstSeenAt &&
+        (!existing.firstSeenAt || exception.firstSeenAt < existing.firstSeenAt)
+      ) {
+        existing.firstSeenAt = exception.firstSeenAt;
+      }
+
+      if (
+        NewExceptionDetector.isBetterRepresentative(
+          exception,
+          existing.representative,
+        )
+      ) {
+        existing.representative = exception;
+        existing.representativeExceptionId = exceptionId;
+        existing.safeMessage = sanitizeExceptionMessage(
+          exception.message || "",
+        );
+      }
+    }
+
+    /*
+     * Loudest first, ties broken on the key — the candidate cap cuts this
+     * list, so the order decides what a tick emits and it must not depend on
+     * the arbitrary order rows came back in.
+     */
+    return Array.from(buckets.values()).sort(
+      (a: NewExceptionFailureMode, b: NewExceptionFailureMode): number => {
+        if (a.totalOccurrenceCount !== b.totalOccurrenceCount) {
+          return b.totalOccurrenceCount - a.totalOccurrenceCount;
+        }
+        return a.failureModeKey < b.failureModeKey ? -1 : 1;
+      },
+    );
+  }
+
+  // Loudest wins; ties go to the older group, then to the lower id.
+  private static isBetterRepresentative(
+    candidate: TelemetryException,
+    current: TelemetryException,
+  ): boolean {
+    const candidateCount: number = candidate.occuranceCount || 0;
+    const currentCount: number = current.occuranceCount || 0;
+
+    if (candidateCount !== currentCount) {
+      return candidateCount > currentCount;
+    }
+
+    if (candidate.firstSeenAt && current.firstSeenAt) {
+      const candidateTime: number = candidate.firstSeenAt.getTime();
+      const currentTime: number = current.firstSeenAt.getTime();
+      if (candidateTime !== currentTime) {
+        return candidateTime < currentTime;
+      }
+    }
+
+    return (candidate.id?.toString() || "") < (current.id?.toString() || "");
+  }
+
   /*
    * Best-effort service attribution: primaryEntityId is polymorphic
    * (Service/Host/DockerHost/...), and findOneById simply returns null for
    * the non-Service kinds (AIInvestigationAPI precedent) — the insight then
-   * ships without a service name.
+   * ships without a service name. Cached per tick: a project's exceptions
+   * cluster on a handful of services, and the lookup is the only per-
+   * candidate IO left in this detector.
    */
   private async resolveServiceName(
     primaryEntityId: ObjectID | undefined,
+    cache: Map<string, string | null>,
   ): Promise<string | null> {
     if (!primaryEntityId) {
       return null;
     }
+
+    const cacheKey: string = primaryEntityId.toString();
+    const cached: string | null | undefined = cache.get(cacheKey);
+    if (cached !== undefined) {
+      return cached;
+    }
+
     const service: Service | null = await ServiceService.findOneById({
       id: primaryEntityId,
       select: { name: true },
       props: { isRoot: true },
     });
-    return service?.name || null;
+
+    const serviceName: string | null = service?.name || null;
+    cache.set(cacheKey, serviceName);
+    return serviceName;
   }
 }

package/Server/Utils/AI/SRE/Insights/InsightStore.ts

@@ -33,11 +33,30 @@ export const MAX_NEW_INSIGHTS_PER_PROJECT_PER_SCAN: number = 10;
  * G11 noise posture: a human dismissal is the strongest precision signal we
  * have, so a finding with the same fingerprint stays suppressed for this
  * many days after the dismissal instead of reappearing on the next tick.
- * After the cooldown (or after a Resolved insight's signal returns) a fresh
- * insight is warranted — a resolved issue that reappears is a regression.
  */
 export const DISMISSED_COOLDOWN_DAYS: number = 7;
 
+/*
+ * The same idea for the other human terminal state, with a much shorter
+ * clock: a resolved issue that comes back IS a regression worth re-filing,
+ * but it has to actually come back first.
+ *
+ * WITHOUT THIS, RESOLVE DID NOT STICK. Detectors emit on a standing
+ * condition, not on an event — NewException re-emits an unchanged candidate
+ * on all ~96 ticks of the 24h window after the exception's firstSeenAt, and
+ * resolving an insight changes nothing the detector looks at. So a Resolved
+ * row (terminal, therefore never refreshed) fell straight through to CREATE
+ * and a byte-identical insight reappeared within 15 minutes, complete with
+ * its own triage LLM run. Resolving again just minted another one.
+ *
+ * 24 hours is chosen to cover NEW_EXCEPTION_LOOKBACK_HOURS: it outlasts the
+ * window in which a NewException candidate is re-emitted from the same
+ * evidence, so that finding stays closed, while the standing-condition
+ * detectors (spikes, drift) can still re-file a day later if their signal
+ * genuinely returns.
+ */
+export const RESOLVED_COOLDOWN_HOURS: number = 24;
+
 /*
  * Column-safety clamps: DatabaseService validates string columns against the
  * model's declared lengths and THROWS on overflow, and detectors embed raw
@@ -74,8 +93,8 @@ export default class InsightStore {
    *   - none                          → CREATE (subject to the per-scan cap)
    *   - non-terminal                  → REFRESH (never touches status)
    *   - Dismissed within the cooldown → suppress
-   *   - Dismissed past the cooldown,
-   *     or Resolved                   → CREATE (regression / recurrence)
+   *   - Resolved within the cooldown  → suppress
+   *   - either, past its cooldown     → CREATE (regression / recurrence)
    * All access is root-props: the scanner is a system actor with explicit
    * projectId scoping, not a per-user ACL consumer.
    */
@@ -129,6 +148,8 @@ export default class InsightStore {
             status: true,
             occurrenceCount: true,
             humanVerdictAt: true,
+            // The cooldown fallback when no verdict was ever stamped.
+            lastSeenAt: true,
           },
           sort: { createdAt: SortOrder.Descending },
           props: { isRoot: true },
@@ -152,6 +173,17 @@ export default class InsightStore {
               lastSeenAt: data.now,
               occurrenceCount: (existing.occurrenceCount || 1) + 1,
               severity: candidate.severity,
+              /*
+               * The title is refreshed with the rest of the picture, not
+               * frozen at creation. Detectors put live numbers in it (a spike
+               * multiplier, a latency factor) and the label itself sharpens as
+               * more of the failure is observed — a row left showing "at 5.2x"
+               * beside a High severity and a 12x detail body reads as a bug.
+               */
+              title: this.clampToColumn(
+                candidate.title,
+                INSIGHT_TITLE_MAX_LENGTH,
+              ),
               detailMarkdown: candidate.detailMarkdown,
               evidence: candidate.evidence,
             },
@@ -162,27 +194,53 @@ export default class InsightStore {
           continue;
         }
 
-        if (existing && existing.status === AIInsightStatus.Dismissed) {
+        if (
+          existing &&
+          existing.status &&
+          AIInsightStatusHelper.isTerminalStatus(existing.status)
+        ) {
+          /*
+           * Both human terminal states get a cooldown, measured from the
+           * human's action and inclusive at the boundary (a cooldown exactly
+           * elapsed still suppresses). They differ only in length: a
+           * dismissal says "this is not worth my time" (7 days), a resolve
+           * says "handled" (24 hours — long enough that the detector stops
+           * re-emitting the very evidence the human just closed).
+           */
+          const isDismissed: boolean =
+            existing.status === AIInsightStatus.Dismissed;
+
+          const cooldownMs: number = isDismissed
+            ? OneUptimeDate.getMillisecondsInDays(DISMISSED_COOLDOWN_DAYS)
+            : OneUptimeDate.getMillisecondsInHours(RESOLVED_COOLDOWN_HOURS);
+
           /*
-           * Cooldown is measured from the human's dismissal, inclusive at the
-           * boundary (exactly DISMISSED_COOLDOWN_DAYS old still suppresses).
-           * A Dismissed row without a verdict timestamp cannot prove the
-           * cooldown elapsed, so it suppresses too — when in doubt, stay
-           * quiet (G11 noise posture).
+           * humanVerdictAt is the moment the human acted. It can be missing
+           * (defensive) and, on a Confirm-then-Resolve, it holds the CONFIRM
+           * time — resolveInsight leaves an existing verdict alone. lastSeenAt
+           * is the fallback: refreshes stop the moment a row goes terminal, so
+           * it is pinned to the last tick before the human closed it, which is
+           * never later than the close. Neither available → suppress; a
+           * terminal row we cannot date must not be re-filed every 15 minutes
+           * (G11 noise posture: when in doubt, stay quiet).
            */
-          const cooldownMs: number = OneUptimeDate.getMillisecondsInDays(
-            DISMISSED_COOLDOWN_DAYS,
-          );
+          const closedAt: Date | undefined = existing.humanVerdictAt
+            ? OneUptimeDate.fromString(existing.humanVerdictAt)
+            : existing.lastSeenAt
+              ? OneUptimeDate.fromString(existing.lastSeenAt)
+              : undefined;
 
-          const withinCooldown: boolean = existing.humanVerdictAt
-            ? data.now.getTime() -
-                OneUptimeDate.fromString(existing.humanVerdictAt).getTime() <=
-              cooldownMs
+          const withinCooldown: boolean = closedAt
+            ? data.now.getTime() - closedAt.getTime() <= cooldownMs
             : true;
 
           if (withinCooldown) {
             logger.debug(
-              `AI Insights: suppressing candidate ${fingerprint} for project ${data.projectId.toString()} — dismissed by a human within the last ${DISMISSED_COOLDOWN_DAYS} days.`,
+              `AI Insights: suppressing candidate ${fingerprint} for project ${data.projectId.toString()} — ${
+                isDismissed
+                  ? `dismissed by a human within the last ${DISMISSED_COOLDOWN_DAYS} days`
+                  : `resolved by a human within the last ${RESOLVED_COOLDOWN_HOURS} hours`
+              }.`,
             );
 
             result.suppressed++;