@oneuptime/common 12.0.4 → 12.0.6

package/Server/Utils/AI/SRE/FixFromIncidentTaskTrigger.ts

@@ -1,7 +1,13 @@
 import ObjectID from "../../../../Types/ObjectID";
 import AIRunType from "../../../../Types/AI/AIRunType";
 import AIRunStatus from "../../../../Types/AI/AIRunStatus";
+import AIRunCodeFixRecommendation from "../../../../Types/AI/AIRunCodeFixRecommendation";
+import CodeFixTaskContext, {
+  getInvestigationCodeFixTaskSnapshot,
+  InvestigationCodeFixTaskSnapshot,
+} from "../../../../Types/AI/CodeFixTaskContext";
 import CodeFixTaskType from "../../../../Types/AI/CodeFixTaskType";
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
 import BadDataException from "../../../../Types/Exception/BadDataException";
 import AIRun from "../../../../Models/DatabaseModels/AIRun";
 import Project from "../../../../Models/DatabaseModels/Project";
@@ -9,6 +15,7 @@ import AIRunService from "../../../Services/AIRunService";
 import ProjectService from "../../../Services/ProjectService";
 import FixRunBudget, { FixRunBudgetDecision } from "../CodeFix/FixRunBudget";
 import SubjectCodeFixRun from "./SubjectCodeFixRun";
+import PostedRootCause from "./PostedRootCause";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
@@ -17,28 +24,32 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  * capability, vision §4.8), in two forms:
  *
  * HUMAN-TRIGGERED: after an AI investigation posts a root-cause analysis on
- * an incident or alert, the user can click "Open Fix PR from this analysis"
- * on the investigation panel: the agent takes the posted analysis (the
- * RootCause feed item) as its entire context and opens a fix pull request.
- * No project opt-in flag is needed for this form: the human in the loop IS
- * the gate (G11 posture preserved). It runs inside a user-facing endpoint
- * (POST /ai-investigation/create-fix-task) and FAILS EARLY with a clear
- * message when a gate is not met.
+ * an incident or alert and records a Recommended code-fix decision, the user
+ * can click "Open Fix PR from this analysis" on the investigation panel: the
+ * agent takes the immutable stored copy of that posted analysis as its
+ * context and opens a fix pull request. No project opt-in flag is needed for
+ * this form, but the durable recommendation remains a server-side gate so a
+ * stale or forged client cannot turn a non-code investigation into a PR.
  *
- * AUTOMATIC: for projects that opted in (Project.enableAutomaticCodeFixes,
- * default FALSE — G11 posture: autonomous PR creation is opt-in only), an
- * investigation that ends with a POSITIVE confident classification (per the
- * structured G6 confidence signal, never the analysis prose) enqueues the
- * same FixFromIncident task with no human click. Like its
- * InstrumentationTaskTrigger sibling it is fire-and-forget inside the
- * investigation's postAnalysis and never throws. The PR still opens as a
+ * AUTOMATIC: for projects that opted in through the subject lane's incident
+ * or alert automatic-code-fix setting (default FALSE — G11 posture:
+ * autonomous PR creation is opt-in only), an
+ * investigation that ends with a POSITIVE code-fix classification (per the
+ * structured G6 signal, never a regex over the analysis prose) enqueues the
+ * same FixFromIncident task with no human click. Confidence alone is not
+ * sufficient: operational and other non-code causes do not open PRs. Like its
+ * InstrumentationTaskTrigger sibling it runs only after the Recommended
+ * decision and snapshot are durable, and it never throws. The PR still opens as a
  * DRAFT and is always human-reviewed — the opt-in moves the human gate from
  * PR creation to PR review, never past it.
  */
 
 export interface AutoFixTaskGateInput {
-  // The project row with enableAi + enableAutomaticCodeFixes selected.
+  // The project row with enableAi + the subject lane's opt-in selected.
   project: Project | null;
+  // Exactly one subject selects the incident or alert opt-in.
+  incidentId?: ObjectID | undefined;
+  alertId?: ObjectID | undefined;
   // Whether the project has at least one GitHub-App-connected repository.
   hasConnectedRepository: boolean;
   // A non-terminal FixFromIncident run for the same subject, if any.
@@ -59,19 +70,27 @@ export default class FixFromIncidentTaskTrigger {
    * incidentId/alertId is expected; userId is the clicking user
    * (attribution on the run).
    *
-   * Throws BadDataException naming the failed gate: no completed
-   * investigation, no GitHub-App repository, or a duplicate active run.
+   * Throws BadDataException naming the failed gate: no latest
+   * completed/recommended investigation, no GitHub-App repository, or a
+   * duplicate active run.
    */
   @CaptureSpan()
   public static async createFixTaskFromInvestigation(data: {
     projectId: ObjectID;
+    investigationRunId: ObjectID;
     incidentId?: ObjectID | undefined;
     alertId?: ObjectID | undefined;
     userId: ObjectID;
   }): Promise<AIRun> {
-    if (!data.incidentId && !data.alertId) {
+    if (!data.investigationRunId) {
       throw new BadDataException(
-        "An incident or alert subject is required to create a fix task.",
+        "An investigation run id is required to create a fix task.",
+      );
+    }
+
+    if (Boolean(data.incidentId) === Boolean(data.alertId)) {
+      throw new BadDataException(
+        "Exactly one incident or alert subject is required to create a fix task.",
       );
     }
 
@@ -80,28 +99,119 @@ export default class FixFromIncidentTaskTrigger {
       : "alert";
 
     /*
-     * Gate 1: a COMPLETED investigation must exist for the subject — its
-     * posted analysis (the RootCause feed item) is the fix task's entire
-     * context. Without one the worker would have nothing to work from.
+     * Gate 1: inspect the LATEST investigation for the subject, matching the
+     * run shown by the dashboard panel. Its durable recommendation is the
+     * server-authored decision about whether a code fix is appropriate, and
+     * its taskContext carries the exact posted analysis that decision covered.
+     * An older Recommended run must never override a newer NotRecommended,
+     * Running or failed result.
      */
-    const completedInvestigation: AIRun | null = await AIRunService.findOneBy({
+    const latestInvestigation: AIRun | null = await AIRunService.findOneBy({
       query: {
+        projectId: data.projectId,
         runType: AIRunType.Investigation,
-        status: AIRunStatus.Completed,
         ...(data.incidentId
           ? { triggeredByIncidentId: data.incidentId }
           : { triggeredByAlertId: data.alertId! }),
       },
-      select: { _id: true },
+      select: {
+        _id: true,
+        completedAt: true,
+        status: true,
+        codeFixRecommendation: true,
+        taskContext: true,
+      },
+      sort: { createdAt: SortOrder.Descending },
       props: { isRoot: true },
     });
 
-    if (!completedInvestigation) {
+    if (!latestInvestigation) {
       throw new BadDataException(
         `No completed AI investigation exists for this ${subjectLabel} — the fix task uses the investigation's posted analysis as its context. Wait for the investigation to complete, or enable AI investigations in the AI settings.`,
       );
     }
 
+    if (
+      latestInvestigation.id?.toString() !== data.investigationRunId.toString()
+    ) {
+      throw new BadDataException(
+        `A newer AI investigation is now available for this ${subjectLabel}. Refresh the page before creating a fix task.`,
+      );
+    }
+
+    if (latestInvestigation.status !== AIRunStatus.Completed) {
+      throw new BadDataException(
+        `The latest AI investigation for this ${subjectLabel} has not completed, so a fix task cannot be created from an older analysis. Wait for the latest investigation to complete successfully.`,
+      );
+    }
+
+    if (
+      latestInvestigation.codeFixRecommendation ===
+      AIRunCodeFixRecommendation.Pending
+    ) {
+      throw new BadDataException(
+        `The latest AI investigation for this ${subjectLabel} is still deciding whether a code fix is appropriate. Wait for that recommendation to finish before creating a fix task.`,
+      );
+    }
+
+    if (
+      latestInvestigation.codeFixRecommendation !==
+      AIRunCodeFixRecommendation.Recommended
+    ) {
+      throw new BadDataException(
+        `The latest AI investigation for this ${subjectLabel} did not recommend a code fix, so a fix pull request task cannot be created from it.`,
+      );
+    }
+
+    /*
+     * Gate the same immutable pair the worker will consume. A Recommended
+     * row without a complete snapshot (or whose snapshot names another run)
+     * is unsafe: looking up the subject's latest feed item would allow a
+     * later investigation to change the task after this authorization check.
+     */
+    const investigationSnapshot: InvestigationCodeFixTaskSnapshot | null =
+      getInvestigationCodeFixTaskSnapshot(latestInvestigation.taskContext);
+
+    if (
+      !latestInvestigation.id ||
+      !investigationSnapshot ||
+      investigationSnapshot.investigationRunId !==
+        latestInvestigation.id.toString()
+    ) {
+      throw new BadDataException(
+        `The latest AI investigation for this ${subjectLabel} has no complete stored analysis snapshot, so a fix task cannot be created safely. Run a new investigation and try again.`,
+      );
+    }
+
+    /*
+     * The feed item is the published report the user actually saw. Resolve
+     * it by the exact run association added to the feed model, then require
+     * it to match the durable recommendation snapshot. A missing or
+     * mismatched report fails closed instead of silently borrowing a newer
+     * subject-level RootCause.
+     */
+    const analysisMarkdown: string | null =
+      await PostedRootCause.getForInvestigation({
+        incidentId: data.incidentId,
+        alertId: data.alertId,
+        aiRunId: data.investigationRunId,
+        runCompletedAt: latestInvestigation.completedAt,
+      });
+
+    if (!analysisMarkdown) {
+      throw new BadDataException(
+        `No posted investigation analysis exists for this ${subjectLabel} and investigation run. Refresh the page and wait for the report to finish publishing before creating a fix task.`,
+      );
+    }
+
+    if (
+      analysisMarkdown !== investigationSnapshot.investigationAnalysisMarkdown
+    ) {
+      throw new BadDataException(
+        `The posted investigation analysis for this ${subjectLabel} does not match its durable recommendation snapshot. Run a new investigation before creating a fix task.`,
+      );
+    }
+
     // Gate 2: a repository the agent can actually open a PR against.
     const hasConnectedRepository: boolean =
       await SubjectCodeFixRun.hasGitHubAppConnectedRepository(data.projectId);
@@ -136,6 +246,10 @@ export default class FixFromIncidentTaskTrigger {
       incidentId: data.incidentId,
       alertId: data.alertId,
       userId: data.userId,
+      taskContext: {
+        sourceInvestigationRunId: data.investigationRunId.toString(),
+        sourceInvestigationAnalysisMarkdown: analysisMarkdown,
+      },
     });
   }
 
@@ -143,9 +257,8 @@ export default class FixFromIncidentTaskTrigger {
    * The pure trigger decision for the AUTOMATIC form, separated from IO so
    * it can be tested directly: strict opt-in (default FALSE), a repository
    * the agent can actually open a PR against, and the per-subject dedupe
-   * guard. The caller has already established the confident-investigation
-   * prerequisite (this runs inside the investigation's postAnalysis, gated
-   * by AIConfidenceSignal.shouldAutoEnqueueCodeFixTask).
+   * guard. The caller has already established and durably persisted the
+   * code-fix-recommended investigation prerequisite.
    */
   public static shouldAutoEnqueueFixTask(
     input: AutoFixTaskGateInput,
@@ -162,10 +275,21 @@ export default class FixFromIncidentTaskTrigger {
      * Strict opt-in — the column defaults to false, so unset/legacy rows
      * never enqueue. Autonomous PR creation must never be default-on.
      */
-    if (input.project.enableAutomaticCodeFixes !== true) {
+    if (Boolean(input.incidentId) === Boolean(input.alertId)) {
+      return {
+        enqueue: false,
+        reason: "exactly one incident or alert subject is required",
+      };
+    }
+
+    const automaticCodeFixesEnabled: boolean = input.incidentId
+      ? input.project.enableAutomaticIncidentCodeFixes === true
+      : input.project.enableAutomaticAlertCodeFixes === true;
+
+    if (!automaticCodeFixesEnabled) {
       return {
         enqueue: false,
-        reason: "project has not opted in to automatic code fixes",
+        reason: `project has not opted in to automatic ${input.incidentId ? "incident" : "alert"} code fixes`,
       };
     }
 
@@ -197,31 +321,52 @@ export default class FixFromIncidentTaskTrigger {
 
   /*
    * The AUTOMATIC form: enqueue a FixFromIncident run for a subject whose
-   * investigation just posted a CONFIDENT analysis. Exactly one of
+   * latest investigation durably recommends a repository code fix. Exactly one of
    * incidentId/alertId is expected. NEVER throws — every failure is logged
-   * and swallowed, because this runs inside the investigation's
-   * postAnalysis and must not fail it. No userId: the run stays
+   * and swallowed, because this is a best-effort post-recommendation
+   * follow-up and must not fail the investigation. No userId: the run stays
    * system-authored.
    */
   @CaptureSpan()
-  public static async autoEnqueueFromConfidentInvestigation(data: {
+  public static async autoEnqueueFromRecommendedInvestigation(data: {
     projectId: ObjectID;
+    investigationRunId: ObjectID;
+    analysisMarkdown: string;
     incidentId?: ObjectID | undefined;
     alertId?: ObjectID | undefined;
   }): Promise<void> {
     const { projectId } = data;
 
     try {
-      if (!data.incidentId && !data.alertId) {
+      if (Boolean(data.incidentId) === Boolean(data.alertId)) {
+        return;
+      }
+
+      const taskContext: CodeFixTaskContext = {
+        sourceInvestigationRunId: data.investigationRunId?.toString(),
+        sourceInvestigationAnalysisMarkdown: data.analysisMarkdown,
+      };
+      const investigationSnapshot: InvestigationCodeFixTaskSnapshot | null =
+        getInvestigationCodeFixTaskSnapshot(taskContext);
+
+      if (!investigationSnapshot) {
+        logger.error(
+          `AI: not auto-enqueueing fix task for project ${projectId.toString()} — the recommended investigation has no complete stored analysis snapshot.`,
+        );
         return;
       }
 
       const project: Project | null = await ProjectService.findOneById({
         id: projectId,
-        select: {
-          enableAi: true,
-          enableAutomaticCodeFixes: true,
-        },
+        select: data.incidentId
+          ? {
+              enableAi: true,
+              enableAutomaticIncidentCodeFixes: true,
+            }
+          : {
+              enableAi: true,
+              enableAutomaticAlertCodeFixes: true,
+            },
         props: { isRoot: true },
       });
 
@@ -232,6 +377,8 @@ export default class FixFromIncidentTaskTrigger {
       const optInDecision: AutoFixTaskGateDecision =
         this.shouldAutoEnqueueFixTask({
           project,
+          incidentId: data.incidentId,
+          alertId: data.alertId,
           hasConnectedRepository: true,
           existingRun: null,
         });
@@ -243,17 +390,73 @@ export default class FixFromIncidentTaskTrigger {
         return;
       }
 
+      /*
+       * Re-read the authoritative latest source after the engine has durably
+       * settled it. This prevents an older overlapping investigation from
+       * auto-authorizing a task after a newer run became authoritative, and
+       * guarantees a task is never claimable while its source is Pending.
+       */
+      const latestInvestigation: AIRun | null = await AIRunService.findOneBy({
+        query: {
+          projectId,
+          runType: AIRunType.Investigation,
+          ...(data.incidentId
+            ? { triggeredByIncidentId: data.incidentId }
+            : { triggeredByAlertId: data.alertId! }),
+        },
+        select: {
+          _id: true,
+          status: true,
+          codeFixRecommendation: true,
+          taskContext: true,
+        },
+        sort: { createdAt: SortOrder.Descending },
+        props: { isRoot: true },
+      });
+      const persistedSnapshot: InvestigationCodeFixTaskSnapshot | null =
+        getInvestigationCodeFixTaskSnapshot(latestInvestigation?.taskContext);
+
+      if (
+        !latestInvestigation?.id ||
+        latestInvestigation.id.toString() !==
+          data.investigationRunId.toString() ||
+        latestInvestigation.status !== AIRunStatus.Completed ||
+        latestInvestigation.codeFixRecommendation !==
+          AIRunCodeFixRecommendation.Recommended ||
+        !persistedSnapshot ||
+        persistedSnapshot.investigationRunId !==
+          data.investigationRunId.toString() ||
+        persistedSnapshot.investigationAnalysisMarkdown !==
+          investigationSnapshot.investigationAnalysisMarkdown
+      ) {
+        logger.debug(
+          `AI: not auto-enqueueing fix task for project ${projectId.toString()} — the source is no longer the latest durably Recommended investigation.`,
+        );
+        return;
+      }
+
       /*
        * G11 guardrail: the per-project daily fix-run budget. For this
        * AUTOMATIC trigger, over-budget is a logged skip — never an error
        * thrown into the investigation.
        */
-      const budget: FixRunBudgetDecision =
-        await FixRunBudget.getBudgetStatus(projectId);
+      const budget: FixRunBudgetDecision = await FixRunBudget.getBudgetStatus(
+        projectId,
+        {
+          incidentId: data.incidentId,
+          alertId: data.alertId,
+        },
+      );
 
       if (!budget.allowed) {
         logger.debug(
-          `AI: not auto-enqueueing fix task for project ${projectId.toString()} — ${FixRunBudget.describeRejection(budget)}`,
+          `AI: not auto-enqueueing fix task for project ${projectId.toString()} — ${FixRunBudget.describeRejection(
+            budget,
+            {
+              incidentId: data.incidentId,
+              alertId: data.alertId,
+            },
+          )}`,
         );
         return;
       }
@@ -270,6 +473,8 @@ export default class FixFromIncidentTaskTrigger {
 
       const decision: AutoFixTaskGateDecision = this.shouldAutoEnqueueFixTask({
         project,
+        incidentId: data.incidentId,
+        alertId: data.alertId,
         hasConnectedRepository,
         existingRun,
       });
@@ -287,6 +492,10 @@ export default class FixFromIncidentTaskTrigger {
           taskType: CodeFixTaskType.FixFromIncident,
           incidentId: data.incidentId,
           alertId: data.alertId,
+          taskContext: {
+            sourceInvestigationRunId: data.investigationRunId.toString(),
+            sourceInvestigationAnalysisMarkdown: data.analysisMarkdown,
+          },
         });
 
       logger.debug(

package/Server/Utils/AI/SRE/IncidentInvestigationRunner.ts

@@ -216,6 +216,8 @@ export default class AIIncidentInvestigationRunner {
             projectId,
             monitorId: QueryHelper.any(monitorIds),
             runType: AIRunType.Investigation,
+            triggeredByIncidentId: QueryHelper.notNull(),
+            triggeredByAlertId: QueryHelper.isNull(),
             createdAt: QueryHelper.greaterThan(windowStart),
           },
           props: { isRoot: true },
@@ -349,7 +351,10 @@ export default class AIIncidentInvestigationRunner {
       request: {
         // Persisted LlmLog label — owned by AIService (G4 budget list).
         feature: AI_INCIDENT_INVESTIGATION_FEATURE,
+        incidentId,
         contextSummary,
+        persistCodeFixRecommendation: true,
+        persistAnalysisTldr: true,
         postAnalysis: async (postData: {
           analysisMarkdown: string;
           confidence: ConfidenceSignal;
@@ -358,6 +363,7 @@ export default class AIIncidentInvestigationRunner {
           await IncidentFeedService.createIncidentFeedItem({
             incidentId,
             projectId,
+            aiRunId,
             incidentFeedEventType: IncidentFeedEventType.RootCause,
             displayColor: Blue500,
             feedInfoInMarkdown: postData.analysisMarkdown,
@@ -421,7 +427,7 @@ export default class AIIncidentInvestigationRunner {
 
           /*
            * Inconclusive means the telemetry was insufficient — for
-           * opted-in projects (Project.enableInstrumentationFixTasks,
+           * opted-in projects (the incident instrumentation-fix setting,
            * default false), queue an ImproveInstrumentation fix task that
            * opens a PR adding the missing observability. Runs strictly
            * AFTER the analysis is posted, and the trigger never throws, so
@@ -442,29 +448,25 @@ export default class AIIncidentInvestigationRunner {
               },
             );
           }
-
-          /*
-           * The confident twin: a POSITIVE confident classification means
-           * the posted analysis asserts an evidenced root cause — for
-           * opted-in projects (Project.enableAutomaticCodeFixes, default
-           * false), automatically queue the FixFromIncident task that
-           * opens a draft fix PR from this analysis (the automatic form
-           * of the "Open Fix PR from this analysis" button). Runs strictly
-           * AFTER the analysis is posted because the posted RootCause feed
-           * item IS the fix task's entire context, and the trigger never
-           * throws. Fail direction (G6): the deterministic floor and a
-           * failed classification never auto-open a PR.
-           */
-          if (
-            AIConfidenceSignal.shouldAutoEnqueueCodeFixTask(postData.confidence)
-          ) {
-            await FixFromIncidentTaskTrigger.autoEnqueueFromConfidentInvestigation(
-              {
-                projectId,
-                incidentId,
-              },
-            );
-          }
+        },
+        /*
+         * The engine calls this only after it atomically persists Recommended
+         * with the exact posted analysis snapshot. That durable decision is
+         * the prerequisite for the optional automatic FixFromIncident task;
+         * the trigger then applies the incident lane's independent opt-in and
+         * remaining repository, budget and dedupe gates.
+         */
+        onCodeFixRecommended: async (data: {
+          analysisMarkdown: string;
+        }): Promise<void> => {
+          await FixFromIncidentTaskTrigger.autoEnqueueFromRecommendedInvestigation(
+            {
+              projectId,
+              investigationRunId: aiRunId,
+              analysisMarkdown: data.analysisMarkdown,
+              incidentId,
+            },
+          );
         },
         /*
          * RCA-first ordering: auto-remediation for this incident was

package/Server/Utils/AI/SRE/Insights/Detectors/ExceptionIdentity.ts

@@ -0,0 +1,125 @@
+import ObjectID from "../../../../../../Types/ObjectID";
+import Crypto from "../../../../../../Utils/Crypto";
+import { normalizeExceptionText } from "../../../../Telemetry/ExceptionSanitizer";
+
+/*
+ * Shared exception identity for the two exception detectors (NewException and
+ * ExceptionSpike).
+ *
+ * THE PROBLEM THIS SOLVES. A TelemetryException GROUP is keyed at ingest on
+ * sha256(projectId + primaryEntityId + normalizedMessage + normalizedStackTrace
+ * + exceptionType) — the STACK TRACE is part of that key. One logical failure
+ * raised from several call sites is therefore several groups, and a detector
+ * that emits one insight per group files the same finding several times over.
+ * Worse, the insight TITLE used to be the exceptionType alone, so those rows
+ * were not merely duplicated, they were indistinguishable: a service whose
+ * instrumentation reports `exception.type = "401"` produced a wall of
+ * identical "New exception: 401 in api" rows.
+ *
+ * THE FAILURE MODE is the unit a human actually triages: one service, one
+ * exception type, one normalized message. Everything below that (which call
+ * stack it surfaced on) is detail that belongs inside the insight, not a
+ * separate row in the inbox.
+ */
+
+/*
+ * How much of the message rides in the insight title. Titles must stay
+ * scannable in a list, but they must also carry enough of the message to tell
+ * two findings apart — the type alone frequently cannot (HTTP status codes,
+ * bare `Error`, `Exception`).
+ */
+export const EXCEPTION_LABEL_MESSAGE_MAX_LENGTH: number = 80;
+
+/*
+ * Human label for an exception: the type AND the message, because either one
+ * alone collides. `TypeError` is not a finding; `TypeError: Cannot read
+ * properties of undefined (reading 'user')` is.
+ *
+ * Only the message is truncated (the type is short by construction), so a
+ * label is never longer than the type plus the cap plus the separator.
+ */
+export function buildExceptionLabel(
+  exceptionType: string | undefined,
+  message: string | undefined,
+  messageMaxLength: number = EXCEPTION_LABEL_MESSAGE_MAX_LENGTH,
+): string {
+  const type: string = (exceptionType || "").trim();
+  const fullMessage: string = (message || "").trim();
+
+  if (!type && !fullMessage) {
+    return "Unknown exception";
+  }
+
+  if (!fullMessage) {
+    return type;
+  }
+
+  const shortMessage: string =
+    fullMessage.length <= messageMaxLength
+      ? fullMessage
+      : `${fullMessage.slice(0, messageMaxLength)}…`;
+
+  if (!type) {
+    return shortMessage;
+  }
+
+  /*
+   * Runtimes routinely prefix the message with the type ("Error: connect
+   * ECONNREFUSED", "ValueError: invalid literal"), and some SDKs report the
+   * message as the type verbatim. Saying it twice reads as a bug.
+   */
+  const lowerType: string = type.toLowerCase();
+  const lowerMessage: string = fullMessage.toLowerCase();
+
+  if (
+    lowerMessage === lowerType ||
+    lowerMessage.startsWith(`${lowerType}:`) ||
+    lowerMessage.startsWith(`${lowerType} `)
+  ) {
+    return shortMessage;
+  }
+
+  return `${type}: ${shortMessage}`;
+}
+
+/*
+ * Stable identity of a FAILURE MODE — the dedupe key insights are filed under.
+ *
+ * (entity, type, normalized message) and deliberately NOT the stack trace:
+ * collapsing stack-trace-only variants of one failure into one insight is the
+ * entire point. The entity id stays in the clear so a fingerprint is greppable
+ * back to a service, and it guarantees the key can never collide across
+ * services; the type+message half is hashed so an unbounded telemetry string
+ * cannot blow past the fingerprint column.
+ *
+ * `${exceptionType}|${normalizeExceptionText(message)}` is the SAME signature
+ * TelemetryExceptionService.validateNoOpenFixForSimilarException already uses
+ * to stop the fix lane opening near-identical pull requests for one throw site
+ * ("observed: 15 near-identical PRs for one invalid-uuid throw site"). The
+ * insight lane had no equivalent — this is that same rule, applied one stage
+ * earlier so the duplicates never reach the inbox in the first place.
+ *
+ * The RAW message is normalized here rather than the sanitized one: the
+ * fingerprint is an identity that MUST stay byte-for-byte stable across ticks
+ * and releases (a key that drifts files a duplicate on every drift), and
+ * normalizeExceptionText carries exactly that stability contract, while the
+ * secret redactor layered on top of it for display does not.
+ */
+export function buildExceptionFailureModeKey(data: {
+  primaryEntityId: ObjectID | undefined;
+  exceptionType: string | undefined;
+  message: string | undefined;
+}): string {
+  const entityId: string =
+    data.primaryEntityId?.toString() || "unattributed-entity";
+  const type: string = (data.exceptionType || "").trim();
+  const normalizedMessage: string = normalizeExceptionText(
+    (data.message || "").trim(),
+  );
+
+  const contentHash: string = Crypto.getSha256Hash(
+    `${type}|${normalizedMessage}`,
+  );
+
+  return `${entityId}:${contentHash}`;
+}