@oneuptime/common 12.0.5 → 12.0.7

package/Server/Utils/AI/SRE/Insights/Detectors/ExceptionIdentity.ts

@@ -0,0 +1,125 @@
+import ObjectID from "../../../../../../Types/ObjectID";
+import Crypto from "../../../../../../Utils/Crypto";
+import { normalizeExceptionText } from "../../../../Telemetry/ExceptionSanitizer";
+
+/*
+ * Shared exception identity for the two exception detectors (NewException and
+ * ExceptionSpike).
+ *
+ * THE PROBLEM THIS SOLVES. A TelemetryException GROUP is keyed at ingest on
+ * sha256(projectId + primaryEntityId + normalizedMessage + normalizedStackTrace
+ * + exceptionType) — the STACK TRACE is part of that key. One logical failure
+ * raised from several call sites is therefore several groups, and a detector
+ * that emits one insight per group files the same finding several times over.
+ * Worse, the insight TITLE used to be the exceptionType alone, so those rows
+ * were not merely duplicated, they were indistinguishable: a service whose
+ * instrumentation reports `exception.type = "401"` produced a wall of
+ * identical "New exception: 401 in api" rows.
+ *
+ * THE FAILURE MODE is the unit a human actually triages: one service, one
+ * exception type, one normalized message. Everything below that (which call
+ * stack it surfaced on) is detail that belongs inside the insight, not a
+ * separate row in the inbox.
+ */
+
+/*
+ * How much of the message rides in the insight title. Titles must stay
+ * scannable in a list, but they must also carry enough of the message to tell
+ * two findings apart — the type alone frequently cannot (HTTP status codes,
+ * bare `Error`, `Exception`).
+ */
+export const EXCEPTION_LABEL_MESSAGE_MAX_LENGTH: number = 80;
+
+/*
+ * Human label for an exception: the type AND the message, because either one
+ * alone collides. `TypeError` is not a finding; `TypeError: Cannot read
+ * properties of undefined (reading 'user')` is.
+ *
+ * Only the message is truncated (the type is short by construction), so a
+ * label is never longer than the type plus the cap plus the separator.
+ */
+export function buildExceptionLabel(
+  exceptionType: string | undefined,
+  message: string | undefined,
+  messageMaxLength: number = EXCEPTION_LABEL_MESSAGE_MAX_LENGTH,
+): string {
+  const type: string = (exceptionType || "").trim();
+  const fullMessage: string = (message || "").trim();
+
+  if (!type && !fullMessage) {
+    return "Unknown exception";
+  }
+
+  if (!fullMessage) {
+    return type;
+  }
+
+  const shortMessage: string =
+    fullMessage.length <= messageMaxLength
+      ? fullMessage
+      : `${fullMessage.slice(0, messageMaxLength)}…`;
+
+  if (!type) {
+    return shortMessage;
+  }
+
+  /*
+   * Runtimes routinely prefix the message with the type ("Error: connect
+   * ECONNREFUSED", "ValueError: invalid literal"), and some SDKs report the
+   * message as the type verbatim. Saying it twice reads as a bug.
+   */
+  const lowerType: string = type.toLowerCase();
+  const lowerMessage: string = fullMessage.toLowerCase();
+
+  if (
+    lowerMessage === lowerType ||
+    lowerMessage.startsWith(`${lowerType}:`) ||
+    lowerMessage.startsWith(`${lowerType} `)
+  ) {
+    return shortMessage;
+  }
+
+  return `${type}: ${shortMessage}`;
+}
+
+/*
+ * Stable identity of a FAILURE MODE — the dedupe key insights are filed under.
+ *
+ * (entity, type, normalized message) and deliberately NOT the stack trace:
+ * collapsing stack-trace-only variants of one failure into one insight is the
+ * entire point. The entity id stays in the clear so a fingerprint is greppable
+ * back to a service, and it guarantees the key can never collide across
+ * services; the type+message half is hashed so an unbounded telemetry string
+ * cannot blow past the fingerprint column.
+ *
+ * `${exceptionType}|${normalizeExceptionText(message)}` is the SAME signature
+ * TelemetryExceptionService.validateNoOpenFixForSimilarException already uses
+ * to stop the fix lane opening near-identical pull requests for one throw site
+ * ("observed: 15 near-identical PRs for one invalid-uuid throw site"). The
+ * insight lane had no equivalent — this is that same rule, applied one stage
+ * earlier so the duplicates never reach the inbox in the first place.
+ *
+ * The RAW message is normalized here rather than the sanitized one: the
+ * fingerprint is an identity that MUST stay byte-for-byte stable across ticks
+ * and releases (a key that drifts files a duplicate on every drift), and
+ * normalizeExceptionText carries exactly that stability contract, while the
+ * secret redactor layered on top of it for display does not.
+ */
+export function buildExceptionFailureModeKey(data: {
+  primaryEntityId: ObjectID | undefined;
+  exceptionType: string | undefined;
+  message: string | undefined;
+}): string {
+  const entityId: string =
+    data.primaryEntityId?.toString() || "unattributed-entity";
+  const type: string = (data.exceptionType || "").trim();
+  const normalizedMessage: string = normalizeExceptionText(
+    (data.message || "").trim(),
+  );
+
+  const contentHash: string = Crypto.getSha256Hash(
+    `${type}|${normalizedMessage}`,
+  );
+
+  return `${entityId}:${contentHash}`;
+}

package/Server/Utils/AI/SRE/Insights/Detectors/ExceptionSpikeDetector.ts

@@ -14,6 +14,10 @@ import ExceptionInstanceService from "../../../../../Services/ExceptionInstanceS
 import ServiceService from "../../../../../Services/ServiceService";
 import QueryHelper from "../../../../../Types/Database/QueryHelper";
 import { sanitizeExceptionMessage } from "../../../../Telemetry/ExceptionSanitizer";
+import {
+  EXCEPTION_LABEL_MESSAGE_MAX_LENGTH,
+  buildExceptionLabel,
+} from "./ExceptionIdentity";
 import {
   InsightCandidate,
   InsightDetector,
@@ -65,7 +69,8 @@ export const EXCEPTION_SPIKE_HIGH_SEVERITY_MULTIPLIER: number = 10;
 export const EXCEPTION_SPIKE_CANDIDATE_LIMIT: number = 25;
 
 // Titles must stay scannable in a list — truncate long exception messages.
-export const EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH: number = 80;
+export const EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH: number =
+  EXCEPTION_LABEL_MESSAGE_MAX_LENGTH;
 
 export interface ExceptionSpikeDecision {
   isSpike: boolean;
@@ -114,27 +119,32 @@ export default class ExceptionSpikeDetector implements InsightDetector {
   }
 
   /*
-   * Human label for the exception: the type when present, else the first
-   * EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH characters of the message.
+   * Human label for the exception: the type AND the message, because either
+   * one alone collides across findings (see ExceptionIdentity).
    */
   public static buildExceptionLabel(
     exceptionType: string | undefined,
     message: string | undefined,
   ): string {
-    if (exceptionType && exceptionType.trim().length > 0) {
-      return exceptionType.trim();
-    }
-    const trimmedMessage: string = (message || "").trim();
-    if (trimmedMessage.length === 0) {
-      return "Unknown exception";
-    }
-    if (trimmedMessage.length <= EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH) {
-      return trimmedMessage;
-    }
-    return `${trimmedMessage.slice(0, EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH)}…`;
+    return buildExceptionLabel(
+      exceptionType,
+      message,
+      EXCEPTION_SPIKE_TITLE_MESSAGE_MAX_LENGTH,
+    );
   }
 
-  // Stable dedupe key, keyed on the exception GROUP id (ingest dedupe unit).
+  /*
+   * Stable dedupe key, keyed on the exception GROUP id (ingest dedupe unit).
+   *
+   * DELIBERATELY NOT the failure-mode key NewExceptionDetector files under.
+   * That detector's evidence is a Postgres column it can simply add up across
+   * the groups of one failure mode; a spike is a RATE, measured per ingest
+   * fingerprint in ClickHouse. Rolling several fingerprints into one insight
+   * would mean summing their windows and re-deciding the spike on the total —
+   * which changes what counts as a spike, not just how many rows it files.
+   * The title carries the type, the message and the multiplier, so sibling
+   * groups read as distinct rows rather than as copies.
+   */
   public static buildFingerprint(telemetryExceptionId: string): string {
     return `exception-spike:${telemetryExceptionId}`;
   }

package/Server/Utils/AI/SRE/Insights/Detectors/NewExceptionDetector.ts

@@ -9,7 +9,13 @@ import Service from "../../../../../../Models/DatabaseModels/Service";
 import TelemetryExceptionService from "../../../../../Services/TelemetryExceptionService";
 import ServiceService from "../../../../../Services/ServiceService";
 import QueryHelper from "../../../../../Types/Database/QueryHelper";
+import logger from "../../../../Logger";
 import { sanitizeExceptionMessage } from "../../../../Telemetry/ExceptionSanitizer";
+import {
+  EXCEPTION_LABEL_MESSAGE_MAX_LENGTH,
+  buildExceptionFailureModeKey,
+  buildExceptionLabel,
+} from "./ExceptionIdentity";
 import {
   InsightCandidate,
   InsightDetector,
@@ -23,6 +29,18 @@ import {
  * the exception groups table carries firstSeenAt and the lifetime
  * occurrence count, so no ClickHouse scan is needed at all. Deterministic —
  * no LLM anywhere in this detector.
+ *
+ * ONE INSIGHT PER FAILURE MODE, NOT PER EXCEPTION GROUP. Group identity at
+ * ingest hashes the normalized STACK TRACE alongside the type and message, so
+ * one logical failure raised from several call sites is several groups. This
+ * detector used to file one insight per group, which filed the same finding
+ * over and over — and because the title was the exceptionType alone, the rows
+ * were not merely duplicated but indistinguishable (a service whose
+ * instrumentation reports `exception.type = "401"` produced a wall of
+ * identical "New exception: 401 in api" rows). Candidates are therefore
+ * bucketed on the (entity, type, normalized message) failure mode, counts are
+ * summed across the member groups, and the insight is fingerprinted on that
+ * bucket — see ExceptionIdentity.
  */
 
 /*
@@ -36,38 +54,72 @@ export const NEW_EXCEPTION_LOOKBACK_HOURS: number = 24;
 /*
  * A single occurrence is noise (a deploy hiccup, a one-off bad request).
  * Three occurrences means the failure path is being re-hit — worth a
- * human's attention.
+ * human's attention. Applied per exception GROUP, before bucketing: a
+ * failure mode must have at least one group that is genuinely recurring,
+ * rather than qualifying by adding up single-occurrence stack variants.
  */
 export const NEW_EXCEPTION_MIN_OCCURRENCE_COUNT: number = 3;
 
 /*
  * At 50+ occurrences inside its first day the exception is hammering a hot
- * path — escalate to High so it sorts above the long tail.
+ * path — escalate to High so it sorts above the long tail. Measured on the
+ * failure mode's summed count, which is what the insight reports.
  */
 export const NEW_EXCEPTION_HIGH_SEVERITY_OCCURRENCE_COUNT: number = 50;
 
 /*
- * Upper bound on candidates per scan tick. Mirrors the TopExceptionsTool
- * cap — a project melting down does not need 500 insights, and the scanner
+ * How many exception GROUPS one tick reads. Higher than the candidate cap
+ * because groups collapse into failure modes: a single noisy throw site can
+ * easily own dozens of stack-trace variants, and reading only 25 of them
+ * would let one loud failure mode crowd every other one out of the tick. It
+ * is one indexed Postgres query either way.
+ */
+export const NEW_EXCEPTION_GROUP_SCAN_LIMIT: number = 100;
+
+/*
+ * Upper bound on candidates per scan tick, applied AFTER bucketing — a
+ * project melting down does not need 500 insights, and the scanner
  * additionally caps new insights per tick.
  */
 export const NEW_EXCEPTION_CANDIDATE_LIMIT: number = 25;
 
 // Titles must stay scannable in a list — truncate long exception messages.
-export const NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH: number = 80;
+export const NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH: number =
+  EXCEPTION_LABEL_MESSAGE_MAX_LENGTH;
 
 export interface NewExceptionDecision {
   qualifies: boolean;
   severity: AIInsightSeverity;
 }
 
+/*
+ * One failure mode: the exception groups that share an (entity, type,
+ * normalized message) identity, rolled up into the numbers the insight
+ * reports.
+ */
+export interface NewExceptionFailureMode {
+  failureModeKey: string;
+  // The loudest member group — what the insight links to and labels itself by.
+  representative: TelemetryException;
+  // The representative's id, narrowed once at bucket time.
+  representativeExceptionId: ObjectID;
+  // Summed lifetime occurrences across every member group.
+  totalOccurrenceCount: number;
+  // How many distinct exception groups (stack variants) rolled up here.
+  groupCount: number;
+  // Earliest firstSeenAt across the member groups.
+  firstSeenAt: Date | undefined;
+  // The representative's message, sanitized once.
+  safeMessage: string;
+}
+
 export default class NewExceptionDetector implements InsightDetector {
   public insightType: AIInsightType = AIInsightType.NewException;
 
   /*
-   * Pure decision: does a lifetime occurrence count qualify, and how
-   * urgent is it. The query already filters on the minimum count — this
-   * re-check keeps the rule in one unit-testable place.
+   * Pure decision: does an occurrence count qualify, and how urgent is it.
+   * The query already filters on the minimum count — this re-check keeps the
+   * rule in one unit-testable place.
    */
   public static evaluateNewException(
     occurrenceCount: number,
@@ -82,33 +134,28 @@ export default class NewExceptionDetector implements InsightDetector {
   }
 
   /*
-   * Human label for the exception: the type when present, else the first
-   * NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH characters of the message.
+   * Human label for the exception: the type AND the message, because either
+   * one alone collides across findings (see ExceptionIdentity).
    */
   public static buildExceptionLabel(
     exceptionType: string | undefined,
     message: string | undefined,
   ): string {
-    if (exceptionType && exceptionType.trim().length > 0) {
-      return exceptionType.trim();
-    }
-    const trimmedMessage: string = (message || "").trim();
-    if (trimmedMessage.length === 0) {
-      return "Unknown exception";
-    }
-    if (trimmedMessage.length <= NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH) {
-      return trimmedMessage;
-    }
-    return `${trimmedMessage.slice(0, NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH)}…`;
+    return buildExceptionLabel(
+      exceptionType,
+      message,
+      NEW_EXCEPTION_TITLE_MESSAGE_MAX_LENGTH,
+    );
   }
 
   /*
-   * Stable dedupe key. Keyed on the exception GROUP id (not the message):
-   * the group id is the ingest-side dedupe unit, so the insight refreshes
-   * instead of duplicating across scan ticks.
+   * Stable dedupe key. Keyed on the FAILURE MODE (entity + exception type +
+   * normalized message), never on the exception group id: group identity
+   * includes the stack trace, so one failure raised from several call sites
+   * would otherwise file one insight per call site.
    */
-  public static buildFingerprint(telemetryExceptionId: string): string {
-    return `new-exception:${telemetryExceptionId}`;
+  public static buildFingerprint(failureModeKey: string): string {
+    return `new-exception:${failureModeKey}`;
   }
 
   public async detect(
@@ -132,6 +179,13 @@ export default class NewExceptionDetector implements InsightDetector {
     /*
      * Root read with an explicit projectId in the query — the scanner is a
      * server-side worker (monitor-worker precedent), never a user ACL.
+     *
+     * firstSeenAt is a deterministic tie-break on the LIMIT boundary:
+     * occuranceCount is incremented on every ingested event, so the long
+     * tail of a noisy endpoint sits densely tied right at the cut and an
+     * untie-broken ORDER BY hands back an arbitrary, shifting subset each
+     * tick. Bucketing already keeps that churn from filing duplicates; the
+     * tie-break keeps the tick itself reproducible.
      */
     const exceptions: Array<TelemetryException> =
       await TelemetryExceptionService.findBy({
@@ -146,44 +200,43 @@ export default class NewExceptionDetector implements InsightDetector {
         },
         sort: {
           occuranceCount: SortOrder.Descending,
+          firstSeenAt: SortOrder.Ascending,
         },
-        limit: NEW_EXCEPTION_CANDIDATE_LIMIT,
+        limit: NEW_EXCEPTION_GROUP_SCAN_LIMIT,
         skip: 0,
         props: { isRoot: true },
       });
 
+    const failureModes: Array<NewExceptionFailureMode> =
+      NewExceptionDetector.bucketIntoFailureModes(exceptions);
+
     const candidates: Array<InsightCandidate> = [];
 
-    for (const exception of exceptions) {
-      const occurrenceCount: number = exception.occuranceCount || 0;
+    // Service names are resolved once per entity, not once per group.
+    const serviceNameCache: Map<string, string | null> = new Map();
+
+    for (const failureMode of failureModes) {
       const decision: NewExceptionDecision =
-        NewExceptionDetector.evaluateNewException(occurrenceCount);
+        NewExceptionDetector.evaluateNewException(
+          failureMode.totalOccurrenceCount,
+        );
 
-      if (!decision.qualifies || !exception.id) {
+      if (!decision.qualifies) {
         continue;
       }
 
       const serviceName: string | null = await this.resolveServiceName(
-        exception.primaryEntityId,
-      );
-
-      /*
-       * The message flows into the insight title/detail/evidence, and from
-       * there into the triage LLM prompt — sanitize it once here so no
-       * interpolated user data (IDs, emails, domains) rides along. The raw
-       * message stays on the exception row for the dashboard.
-       */
-      const safeMessage: string = sanitizeExceptionMessage(
-        exception.message || "",
+        failureMode.representative.primaryEntityId,
+        serviceNameCache,
       );
 
       const label: string = NewExceptionDetector.buildExceptionLabel(
-        exception.exceptionType,
-        safeMessage,
+        failureMode.representative.exceptionType,
+        failureMode.safeMessage,
       );
 
-      const firstSeenAtIso: string | undefined = exception.firstSeenAt
-        ? exception.firstSeenAt.toISOString()
+      const firstSeenAtIso: string | undefined = failureMode.firstSeenAt
+        ? failureMode.firstSeenAt.toISOString()
         : undefined;
 
       const detailLines: Array<string> = [
@@ -191,59 +244,205 @@ export default class NewExceptionDetector implements InsightDetector {
         "",
         `- Exception: \`${label}\``,
       ];
-      if (safeMessage) {
-        detailLines.push(`- Message: ${safeMessage}`);
+      if (failureMode.safeMessage) {
+        detailLines.push(`- Message: ${failureMode.safeMessage}`);
       }
       if (serviceName) {
         detailLines.push(`- Service: ${serviceName}`);
       }
       detailLines.push(
         `- First seen: ${firstSeenAtIso || "unknown"} (within the last ${NEW_EXCEPTION_LOOKBACK_HOURS}h)`,
-        `- Occurrences so far: ${occurrenceCount}`,
+        `- Occurrences so far: ${failureMode.totalOccurrenceCount}`,
       );
+      if (failureMode.groupCount > 1) {
+        detailLines.push(
+          `- Raised from ${failureMode.groupCount} distinct stack traces (same exception type and message — counted together as one failure mode)`,
+        );
+      }
 
       candidates.push({
         insightType: AIInsightType.NewException,
         fingerprint: NewExceptionDetector.buildFingerprint(
-          exception.id.toString(),
+          failureMode.failureModeKey,
         ),
         title: `New exception: ${label}${serviceName ? ` in ${serviceName}` : ""}`,
         detailMarkdown: detailLines.join("\n"),
         severity: decision.severity,
         serviceName: serviceName || undefined,
-        telemetryServiceId: serviceName ? exception.primaryEntityId : undefined,
-        telemetryExceptionId: exception.id,
+        telemetryServiceId: serviceName
+          ? failureMode.representative.primaryEntityId
+          : undefined,
+        telemetryExceptionId: failureMode.representativeExceptionId,
         evidence: {
           exception: {
-            exceptionMessage: safeMessage,
-            exceptionType: exception.exceptionType,
-            totalOccurrenceCount: occurrenceCount,
+            exceptionMessage: failureMode.safeMessage,
+            exceptionType: failureMode.representative.exceptionType,
+            totalOccurrenceCount: failureMode.totalOccurrenceCount,
+            distinctExceptionGroupCount: failureMode.groupCount,
             firstSeenAt: firstSeenAtIso,
           },
         },
       });
+
+      if (candidates.length >= NEW_EXCEPTION_CANDIDATE_LIMIT) {
+        // No silent caps — say what this tick left on the floor.
+        if (failureModes.length > candidates.length) {
+          logger.debug(
+            `AI Insights: NewException capped at ${NEW_EXCEPTION_CANDIDATE_LIMIT} candidates for project ${context.projectId.toString()} — ${
+              failureModes.length - candidates.length
+            } further failure mode(s) this tick were not emitted. They re-surface on later ticks while their exceptions stay inside the ${NEW_EXCEPTION_LOOKBACK_HOURS}h window.`,
+          );
+        }
+        break;
+      }
     }
 
     return candidates;
   }
 
+  /*
+   * Roll the tick's exception groups up into failure modes, loudest first.
+   *
+   * The representative is the group with the highest lifetime occurrence
+   * count (ties broken by the earliest firstSeenAt, then by id, so the choice
+   * is deterministic rather than dependent on row order). It supplies the
+   * label and the exception the insight links to — but NOT the fingerprint,
+   * which is the bucket key, so a representative that changes as counts move
+   * can never file a second insight for the same failure mode.
+   */
+  public static bucketIntoFailureModes(
+    exceptions: Array<TelemetryException>,
+  ): Array<NewExceptionFailureMode> {
+    const buckets: Map<string, NewExceptionFailureMode> = new Map();
+
+    for (const exception of exceptions) {
+      const exceptionId: ObjectID | undefined = exception.id || undefined;
+
+      if (!exceptionId) {
+        continue;
+      }
+
+      const failureModeKey: string = buildExceptionFailureModeKey({
+        primaryEntityId: exception.primaryEntityId,
+        exceptionType: exception.exceptionType,
+        message: exception.message,
+      });
+
+      const occurrenceCount: number = exception.occuranceCount || 0;
+      const existing: NewExceptionFailureMode | undefined =
+        buckets.get(failureModeKey);
+
+      if (!existing) {
+        buckets.set(failureModeKey, {
+          failureModeKey: failureModeKey,
+          representative: exception,
+          representativeExceptionId: exceptionId,
+          totalOccurrenceCount: occurrenceCount,
+          groupCount: 1,
+          firstSeenAt: exception.firstSeenAt,
+          /*
+           * The message flows into the insight title/detail/evidence, and
+           * from there into the triage LLM prompt — sanitize it once here so
+           * no interpolated user data (IDs, emails, domains) rides along. The
+           * raw message stays on the exception row for the dashboard.
+           */
+          safeMessage: sanitizeExceptionMessage(exception.message || ""),
+        });
+        continue;
+      }
+
+      existing.totalOccurrenceCount += occurrenceCount;
+      existing.groupCount++;
+
+      if (
+        exception.firstSeenAt &&
+        (!existing.firstSeenAt || exception.firstSeenAt < existing.firstSeenAt)
+      ) {
+        existing.firstSeenAt = exception.firstSeenAt;
+      }
+
+      if (
+        NewExceptionDetector.isBetterRepresentative(
+          exception,
+          existing.representative,
+        )
+      ) {
+        existing.representative = exception;
+        existing.representativeExceptionId = exceptionId;
+        existing.safeMessage = sanitizeExceptionMessage(
+          exception.message || "",
+        );
+      }
+    }
+
+    /*
+     * Loudest first, ties broken on the key — the candidate cap cuts this
+     * list, so the order decides what a tick emits and it must not depend on
+     * the arbitrary order rows came back in.
+     */
+    return Array.from(buckets.values()).sort(
+      (a: NewExceptionFailureMode, b: NewExceptionFailureMode): number => {
+        if (a.totalOccurrenceCount !== b.totalOccurrenceCount) {
+          return b.totalOccurrenceCount - a.totalOccurrenceCount;
+        }
+        return a.failureModeKey < b.failureModeKey ? -1 : 1;
+      },
+    );
+  }
+
+  // Loudest wins; ties go to the older group, then to the lower id.
+  private static isBetterRepresentative(
+    candidate: TelemetryException,
+    current: TelemetryException,
+  ): boolean {
+    const candidateCount: number = candidate.occuranceCount || 0;
+    const currentCount: number = current.occuranceCount || 0;
+
+    if (candidateCount !== currentCount) {
+      return candidateCount > currentCount;
+    }
+
+    if (candidate.firstSeenAt && current.firstSeenAt) {
+      const candidateTime: number = candidate.firstSeenAt.getTime();
+      const currentTime: number = current.firstSeenAt.getTime();
+      if (candidateTime !== currentTime) {
+        return candidateTime < currentTime;
+      }
+    }
+
+    return (candidate.id?.toString() || "") < (current.id?.toString() || "");
+  }
+
   /*
    * Best-effort service attribution: primaryEntityId is polymorphic
    * (Service/Host/DockerHost/...), and findOneById simply returns null for
    * the non-Service kinds (AIInvestigationAPI precedent) — the insight then
-   * ships without a service name.
+   * ships without a service name. Cached per tick: a project's exceptions
+   * cluster on a handful of services, and the lookup is the only per-
+   * candidate IO left in this detector.
    */
   private async resolveServiceName(
     primaryEntityId: ObjectID | undefined,
+    cache: Map<string, string | null>,
   ): Promise<string | null> {
     if (!primaryEntityId) {
       return null;
     }
+
+    const cacheKey: string = primaryEntityId.toString();
+    const cached: string | null | undefined = cache.get(cacheKey);
+    if (cached !== undefined) {
+      return cached;
+    }
+
     const service: Service | null = await ServiceService.findOneById({
       id: primaryEntityId,
       select: { name: true },
       props: { isRoot: true },
     });
-    return service?.name || null;
+
+    const serviceName: string | null = service?.name || null;
+    cache.set(cacheKey, serviceName);
+    return serviceName;
   }
 }

package/Server/Utils/AI/SRE/Insights/FixRouting.ts

@@ -21,8 +21,9 @@ import CaptureSpan from "../../../Telemetry/CaptureSpan";
  * Project.enableAi master switch AND on Project.enableInsightFixTasks
  * (default FALSE): insights whose evidence points at code (new/spiking
  * exceptions with a resolvable repository, latency regressions with
- * span-tree findings) queue a CodeFix AIRun that opens a draft pull
- * request. The decision is readiness/evidence-based, never LLM-decided.
+ * span-tree findings) queue a CodeFix AIRun that opens a pull request,
+ * ready for review. The decision is readiness/evidence-based, never
+ * LLM-decided.
  *
  * Everything downstream reuses the existing budgeted creation paths, so
  * every guardrail comes free: the daily fix-run budget (G11, pre-checked