@oneuptime/common 11.5.11 → 11.5.13

package/Server/Utils/Telemetry/ExceptionSanitizer.ts

@@ -0,0 +1,222 @@
+import ToolResultSerializer from "../AI/Toolbox/Serializer";
+
+/*
+ * Shared exception-text sanitization.
+ *
+ * normalizeExceptionText is the fingerprint normalizer that used to live
+ * in App/FeatureSet/Telemetry/Utils/Exception.ts — it was moved here (the
+ * App util now delegates) so server-side consumers in Common (the AI
+ * agent data API) can sanitize exception messages before they reach LLM
+ * prompts, pull-request text, and commit messages. The replacement
+ * behavior MUST stay byte-for-byte stable: fingerprints are computed from
+ * its output, and changing it regroups every existing exception.
+ */
+
+/**
+ * Normalizes a string by replacing dynamic values with placeholders.
+ * This ensures that exceptions with the same root cause but different
+ * dynamic values (like IDs, timestamps, etc.) get the same fingerprint.
+ *
+ * @param text - The text to normalize (message or stack trace)
+ * @returns The normalized text with dynamic values replaced
+ */
+export function normalizeExceptionText(text: string): string {
+  if (!text) {
+    return "";
+  }
+
+  let normalized: string = text;
+
+  // Order matters! More specific patterns should come before generic ones.
+
+  // 1. UUIDs (e.g., 550e8400-e29b-41d4-a716-446655440000)
+  normalized = normalized.replace(
+    /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi,
+    "<UUID>",
+  );
+
+  // 2. MongoDB ObjectIDs (24 hex characters)
+  normalized = normalized.replace(/\b[0-9a-f]{24}\b/gi, "<OBJECT_ID>");
+
+  /*
+   * 3. Stripe-style IDs (e.g., sub_xxx, cus_xxx, pi_xxx, ch_xxx, etc.)
+   * These have a prefix followed by underscore and alphanumeric characters
+   */
+  normalized = normalized.replace(
+    /\b(sub|cus|pi|ch|pm|card|price|prod|inv|txn|evt|req|acct|payout|ba|btok|src|tok|seti|si|cs|link|file|dp|icr|ii|il|is|isci|mbur|or|po|qt|rcpt|re|refund|sku|tax|txi|tr|us|wh)_[A-Za-z0-9]{10,32}\b/g,
+    "<STRIPE_ID>",
+  );
+
+  /*
+   * 4. Generic API/Service IDs - alphanumeric strings that look like IDs
+   * Matches patterns like: prefix_alphanumeric or just long alphanumeric strings
+   * Common in many services (AWS, GCP, etc.)
+   */
+  normalized = normalized.replace(
+    /\b[a-z]{2,10}_[A-Za-z0-9]{8,}\b/g,
+    "<SERVICE_ID>",
+  );
+
+  // 5. JWT tokens (three base64 segments separated by dots)
+  normalized = normalized.replace(
+    /\beyJ[A-Za-z0-9_-]*\.eyJ[A-Za-z0-9_-]*\.[A-Za-z0-9_-]+\b/g,
+    "<JWT>",
+  );
+
+  // 6. Base64 encoded strings (long sequences, likely tokens or encoded data)
+  normalized = normalized.replace(/\b[A-Za-z0-9+/]{40,}={0,2}\b/g, "<BASE64>");
+
+  // 7. IP addresses (IPv4)
+  normalized = normalized.replace(
+    /\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b/g,
+    "<IP>",
+  );
+
+  // 8. IP addresses (IPv6) - simplified pattern
+  normalized = normalized.replace(
+    /\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b/g,
+    "<IPV6>",
+  );
+  normalized = normalized.replace(/\b::1\b/g, "<IPV6>"); // localhost IPv6
+
+  // 9. Email addresses
+  normalized = normalized.replace(
+    /\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/g,
+    "<EMAIL>",
+  );
+
+  /*
+   * 10. URLs with dynamic paths/query params (normalize the dynamic parts)
+   * Keep the domain but normalize path segments that look like IDs
+   */
+  normalized = normalized.replace(
+    /\/[0-9a-f]{8,}(?=\/|$|\?|#|\s|'|")/gi,
+    "/<ID>",
+  );
+
+  /*
+   * 11. Timestamps in various formats
+   * ISO 8601 timestamps
+   */
+  normalized = normalized.replace(
+    /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?/g,
+    "<TIMESTAMP>",
+  );
+  // Unix timestamps (10 or 13 digits)
+  normalized = normalized.replace(/\b1[0-9]{9,12}\b/g, "<TIMESTAMP>");
+
+  // 12. Date formats (YYYY-MM-DD, MM/DD/YYYY, etc.)
+  normalized = normalized.replace(/\b\d{4}[-/]\d{2}[-/]\d{2}\b/g, "<DATE>");
+  normalized = normalized.replace(/\b\d{2}[-/]\d{2}[-/]\d{4}\b/g, "<DATE>");
+
+  // 13. Time formats (HH:MM:SS, HH:MM)
+  normalized = normalized.replace(/\b\d{2}:\d{2}(?::\d{2})?\b/g, "<TIME>");
+
+  // 14. Memory addresses (0x followed by hex)
+  normalized = normalized.replace(/\b0x[0-9a-fA-F]+\b/g, "<MEMORY_ADDR>");
+
+  // 15. Session IDs (common patterns) - MUST come before hex ID pattern
+  normalized = normalized.replace(
+    /\bsession[_-]?id[=:\s]*['"]?[A-Za-z0-9_-]+['"]?/gi,
+    "session_id=<SESSION>",
+  );
+
+  // 16. Request IDs (common patterns) - MUST come before hex ID pattern
+  normalized = normalized.replace(
+    /\brequest[_-]?id[=:\s]*['"]?[A-Za-z0-9_-]+['"]?/gi,
+    "request_id=<REQUEST>",
+  );
+
+  // 17. Correlation IDs - MUST come before hex ID pattern
+  normalized = normalized.replace(
+    /\bcorrelation[_-]?id[=:\s]*['"]?[A-Za-z0-9_-]+['"]?/gi,
+    "correlation_id=<CORRELATION>",
+  );
+
+  // 18. Transaction IDs - MUST come before hex ID pattern
+  normalized = normalized.replace(
+    /\btransaction[_-]?id[=:\s]*['"]?[A-Za-z0-9_-]+['"]?/gi,
+    "transaction_id=<TRANSACTION>",
+  );
+
+  // 19. Hex strings that are likely IDs (8+ chars)
+  normalized = normalized.replace(/\b[0-9a-f]{8,}\b/gi, "<HEX_ID>");
+
+  /*
+   * 20. Quoted strings containing IDs or dynamic values
+   * Match strings in single or double quotes that look like IDs
+   */
+  normalized = normalized.replace(/'[A-Za-z0-9_-]{16,}'/g, "'<ID>'");
+  normalized = normalized.replace(/"[A-Za-z0-9_-]{16,}"/g, '"<ID>"');
+
+  // 21. Port numbers in URLs or connection strings
+  normalized = normalized.replace(/:(\d{4,5})(?=\/|$|\s)/g, ":<PORT>");
+
+  /*
+   * 22. Line numbers in stack traces (keep for context, but normalize large numbers)
+   * This normalizes specific line/column references that might vary
+   */
+  normalized = normalized.replace(/:\d+:\d+\)?$/gm, ":<LINE>:<COL>)");
+
+  // 23. Process/Thread IDs
+  normalized = normalized.replace(/\bPID[:\s]*\d+\b/gi, "PID:<PID>");
+  normalized = normalized.replace(/\bTID[:\s]*\d+\b/gi, "TID:<TID>");
+
+  // 24. Numeric IDs in common patterns (id=123, id: 123, etc.)
+  normalized = normalized.replace(/\bid[=:\s]*['"]?\d+['"]?/gi, "id=<ID>");
+
+  // 25. Large numbers that are likely IDs (more than 6 digits)
+  normalized = normalized.replace(/\b\d{7,}\b/g, "<NUMBER>");
+
+  return normalized;
+}
+
+/**
+ * Sanitize an exception message for surfaces that leave the platform:
+ * LLM prompts, pull-request titles/bodies, and commit messages.
+ *
+ * Normalization replaces the dynamic tokens (UUIDs, emails, IPs, IDs,
+ * timestamps...) that carry user data, then the secret redactor sweeps
+ * whatever is left (bearer tokens, API keys, cards...). The message
+ * keeps its structure — "invalid input syntax for type uuid: <HEX_ID>"
+ * is still perfectly actionable for a fix.
+ */
+export function sanitizeExceptionMessage(message: string): string {
+  if (!message) {
+    return "";
+  }
+  return ToolResultSerializer.redact(normalizeExceptionText(message)).text;
+}
+
+/**
+ * Sanitize a stack trace for the same surfaces.
+ *
+ * Runtimes prefix the native stack string with the exception MESSAGE
+ * ("Error: <message>" in Node, "ValueError: <message>" in Python
+ * headers, "Caused by: ..." in Java) — so a redact-only pass would leak
+ * the very identifiers sanitizeExceptionMessage strips. Frame lines are
+ * indented ("    at fn (file:42:7)", '  File "x.py", line 3', "\tat ...")
+ * and must keep exact file:line references for the code agent, so the
+ * split is: unindented header/message lines get full normalization,
+ * indented frame lines are left intact, and the secret redactor sweeps
+ * everything at the end.
+ */
+// Frame lines are indented; header/message lines are not.
+const INDENTED_FRAME_LINE_REGEX: RegExp = /^\s/;
+
+export function sanitizeStackTrace(stackTrace: string): string {
+  if (!stackTrace) {
+    return "";
+  }
+
+  const normalized: string = stackTrace
+    .split("\n")
+    .map((line: string): string => {
+      return INDENTED_FRAME_LINE_REGEX.test(line)
+        ? line
+        : normalizeExceptionText(line);
+    })
+    .join("\n");
+
+  return ToolResultSerializer.redact(normalized).text;
+}

package/Tests/Server/Services/MonitorStatusTimelineService.test.ts

@@ -0,0 +1,235 @@
+import ObjectID from "../../../Types/ObjectID";
+
+/*
+ * These tests pin the ownership of the per-monitor mutex in
+ * MonitorStatusTimelineService.create() - the prevention layer that stops new
+ * orphaned (endsAt = NULL) timeline rows at the source:
+ *
+ *   - fail-closed: a lock that cannot be acquired refuses the write with the
+ *     exact MONITOR_STATUS_TIMELINE_LOCK_ERROR_MESSAGE callers match on,
+ *     instead of falling through and inserting unlocked (the original bug),
+ *   - release-on-every-path: the mutex is released whether the create
+ *     succeeds or throws (a leaked redis-semaphore mutex refreshes its own
+ *     key for the life of the process, blocking every later create for the
+ *     monitor until acquireTimeout),
+ *   - the feed/workspace-notification side effects run only AFTER release,
+ *     so third-party HTTP latency (Slack/Teams) can never extend the
+ *     critical section,
+ *   - the ignoreHooks and missing-monitorId paths skip locking entirely.
+ *
+ * Semaphore is mocked at the module boundary; DatabaseService.prototype.create
+ * is spied so super.create() is observable without a database.
+ */
+
+const lockMock: jest.Mock = jest.fn();
+const releaseMock: jest.Mock = jest.fn();
+
+jest.mock("../../../Server/Infrastructure/Semaphore", () => {
+  return {
+    __esModule: true,
+    default: {
+      lock: (...args: Array<unknown>) => {
+        return lockMock(...args);
+      },
+      release: (...args: Array<unknown>) => {
+        return releaseMock(...args);
+      },
+    },
+  };
+});
+
+import MonitorStatusTimelineService, {
+  MONITOR_STATUS_TIMELINE_LOCK_ERROR_MESSAGE,
+} from "../../../Server/Services/MonitorStatusTimelineService";
+import DatabaseService from "../../../Server/Services/DatabaseService";
+import MonitorStatusTimeline from "../../../Models/DatabaseModels/MonitorStatusTimeline";
+import CreateBy from "../../../Server/Types/Database/CreateBy";
+import ServerException from "../../../Types/Exception/ServerException";
+
+const MONITOR_ID: ObjectID = new ObjectID(
+  "11111111-1111-4111-8111-111111111111",
+);
+const PROJECT_ID: ObjectID = new ObjectID(
+  "22222222-2222-4222-8222-222222222222",
+);
+const STATUS_ID: ObjectID = new ObjectID(
+  "33333333-3333-4333-8333-333333333333",
+);
+
+type MakeCreateByFunction = (data?: {
+  ignoreHooks?: boolean;
+  omitMonitorId?: boolean;
+}) => CreateBy<MonitorStatusTimeline>;
+
+const makeCreateBy: MakeCreateByFunction = (data?: {
+  ignoreHooks?: boolean;
+  omitMonitorId?: boolean;
+}): CreateBy<MonitorStatusTimeline> => {
+  const timeline: MonitorStatusTimeline = new MonitorStatusTimeline();
+
+  if (!data?.omitMonitorId) {
+    timeline.monitorId = MONITOR_ID;
+  }
+
+  timeline.projectId = PROJECT_ID;
+  timeline.monitorStatusId = STATUS_ID;
+
+  return {
+    data: timeline,
+    props: {
+      isRoot: true,
+      ignoreHooks: data?.ignoreHooks || false,
+    },
+  };
+};
+
+describe("MonitorStatusTimelineService.create mutex ownership", () => {
+  let superCreateSpy: jest.SpyInstance;
+  let feedItemSpy: jest.SpyInstance;
+  let createdItem: MonitorStatusTimeline;
+
+  // a unique object standing in for the redis-semaphore mutex.
+  const fakeMutex: { id: string } = { id: "fake-mutex" };
+
+  beforeEach(() => {
+    lockMock.mockReset();
+    releaseMock.mockReset();
+
+    lockMock.mockResolvedValue(fakeMutex);
+    releaseMock.mockResolvedValue(undefined);
+
+    createdItem = new MonitorStatusTimeline();
+    createdItem.monitorId = MONITOR_ID;
+    createdItem.projectId = PROJECT_ID;
+    createdItem.monitorStatusId = STATUS_ID;
+
+    /*
+     * super.create - everything between lock and release (hooks, validation,
+     * the INSERT) is DatabaseService.create's responsibility and is not under
+     * test here.
+     */
+    superCreateSpy = jest
+      .spyOn(DatabaseService.prototype, "create")
+      .mockResolvedValue(createdItem);
+
+    /*
+     * The feed side effect does DB lookups and third-party HTTP; stub it and
+     * assert WHEN it runs relative to the release.
+     */
+    feedItemSpy = jest
+      .spyOn(
+        MonitorStatusTimelineService as unknown as {
+          createStatusChangeFeedItem: () => Promise<void>;
+        },
+        "createStatusChangeFeedItem",
+      )
+      .mockResolvedValue(undefined);
+  });
+
+  afterEach(() => {
+    superCreateSpy.mockRestore();
+    feedItemSpy.mockRestore();
+  });
+
+  it("acquires the per-monitor lock, creates, then releases", async () => {
+    const result: MonitorStatusTimeline =
+      await MonitorStatusTimelineService.create(makeCreateBy());
+
+    expect(result).toBe(createdItem);
+
+    expect(lockMock).toHaveBeenCalledTimes(1);
+    expect(lockMock).toHaveBeenCalledWith({
+      key: MONITOR_ID.toString(),
+      namespace: "MonitorStatusTimeline.create",
+    });
+
+    expect(superCreateSpy).toHaveBeenCalledTimes(1);
+    expect(releaseMock).toHaveBeenCalledTimes(1);
+    expect(releaseMock).toHaveBeenCalledWith(fakeMutex);
+
+    // lock -> create -> release, in that order.
+    expect(lockMock.mock.invocationCallOrder[0]!).toBeLessThan(
+      superCreateSpy.mock.invocationCallOrder[0]!,
+    );
+    expect(superCreateSpy.mock.invocationCallOrder[0]!).toBeLessThan(
+      releaseMock.mock.invocationCallOrder[0]!,
+    );
+  });
+
+  it("runs the feed side effect only after the mutex is released", async () => {
+    await MonitorStatusTimelineService.create(makeCreateBy());
+
+    expect(feedItemSpy).toHaveBeenCalledTimes(1);
+    expect(releaseMock.mock.invocationCallOrder[0]!).toBeLessThan(
+      feedItemSpy.mock.invocationCallOrder[0]!,
+    );
+  });
+
+  it("fails closed with the exact lock error message and never inserts unlocked", async () => {
+    lockMock.mockRejectedValue(new Error("redis unavailable"));
+
+    await expect(
+      MonitorStatusTimelineService.create(makeCreateBy()),
+    ).rejects.toThrow(MONITOR_STATUS_TIMELINE_LOCK_ERROR_MESSAGE);
+
+    await expect(
+      MonitorStatusTimelineService.create(makeCreateBy()),
+    ).rejects.toBeInstanceOf(ServerException);
+
+    /*
+     * The whole point of fail-closed: the write must NOT happen without the
+     * lock. The pre-fix code logged and fell through unlocked, which is what
+     * produced permanently orphaned endsAt = NULL rows.
+     */
+    expect(superCreateSpy).not.toHaveBeenCalled();
+    expect(releaseMock).not.toHaveBeenCalled();
+  });
+
+  it("releases the mutex when the create itself throws", async () => {
+    superCreateSpy.mockRejectedValue(new Error("insert failed"));
+
+    await expect(
+      MonitorStatusTimelineService.create(makeCreateBy()),
+    ).rejects.toThrow("insert failed");
+
+    // release must run on the throw path - a leaked mutex never expires.
+    expect(releaseMock).toHaveBeenCalledTimes(1);
+    expect(releaseMock).toHaveBeenCalledWith(fakeMutex);
+
+    // and the feed side effect must not run for a failed create.
+    expect(feedItemSpy).not.toHaveBeenCalled();
+  });
+
+  it("does not fail a successful create when the release itself fails", async () => {
+    releaseMock.mockRejectedValue(new Error("redis blip on release"));
+
+    const result: MonitorStatusTimeline =
+      await MonitorStatusTimelineService.create(makeCreateBy());
+
+    expect(result).toBe(createdItem);
+    expect(superCreateSpy).toHaveBeenCalledTimes(1);
+  });
+
+  it("skips locking entirely when ignoreHooks is set", async () => {
+    const result: MonitorStatusTimeline =
+      await MonitorStatusTimelineService.create(
+        makeCreateBy({ ignoreHooks: true }),
+      );
+
+    expect(result).toBe(createdItem);
+    expect(lockMock).not.toHaveBeenCalled();
+    expect(releaseMock).not.toHaveBeenCalled();
+    // no hooks -> no predecessor bookkeeping -> no feed side effect either.
+    expect(feedItemSpy).not.toHaveBeenCalled();
+  });
+
+  it("skips locking when monitorId is missing and lets onBeforeCreate reject it", async () => {
+    await MonitorStatusTimelineService.create(
+      makeCreateBy({ omitMonitorId: true }),
+    );
+
+    // super.create's onBeforeCreate throws BadDataException for this case.
+    expect(lockMock).not.toHaveBeenCalled();
+    expect(superCreateSpy).toHaveBeenCalledTimes(1);
+  });
+});

package/Tests/Server/Services/TelemetryExceptionCodeFixRun.test.ts

@@ -6,6 +6,7 @@ import ServiceService from "../../../Server/Services/ServiceService";
 import CodeRepositoryService from "../../../Server/Services/CodeRepositoryService";
 import AIAgentService from "../../../Server/Services/AIAgentService";
 import AIRunService from "../../../Server/Services/AIRunService";
+import AIAgentTaskPullRequestService from "../../../Server/Services/AIAgentTaskPullRequestService";
 import { RepoResolution } from "../../../Server/Utils/CodeRepository/StackTraceRepoResolver";
 import FindOneBy from "../../../Server/Types/Database/FindOneBy";
 import TelemetryException from "../../../Models/DatabaseModels/TelemetryException";
@@ -37,12 +38,19 @@ const projectId: ObjectID = ObjectID.generate();
 const exceptionId: ObjectID = ObjectID.generate();
 const serviceId: ObjectID = ObjectID.generate();
 
-function fakeException(): TelemetryException {
+function fakeException(
+  overrides: Partial<TelemetryException> = {},
+): TelemetryException {
   return {
     id: exceptionId,
     projectId: projectId,
     primaryEntityId: serviceId,
+    message: "Failed to charge card for user 123",
+    exceptionType: "PaymentError",
     stackTrace: "at charge (/app/src/billing/charge.ts:12:5)",
+    isResolved: false,
+    isArchived: false,
+    ...overrides,
   } as unknown as TelemetryException;
 }
 
@@ -91,6 +99,14 @@ function mockReadinessOk(): void {
     limitInTokens: null,
     usedTokensToday: 0,
   });
+
+  /*
+   * Cross-group duplicate guard collaborators: no non-terminal runs and no
+   * open AI pull requests unless a test overrides these. findBy on the
+   * exception service itself is only reached when candidates exist.
+   */
+  jest.spyOn(AIRunService, "findBy").mockResolvedValue([]);
+  jest.spyOn(AIAgentTaskPullRequestService, "findBy").mockResolvedValue([]);
 }
 
 describe("TelemetryExceptionService.createCodeFixRunForException", () => {
@@ -283,6 +299,194 @@ describe("TelemetryExceptionService.createCodeFixRunForException", () => {
       }),
     );
   });
+
+  test("rejects resolved and archived exceptions server-side", async () => {
+    mockReadinessOk();
+    const create: jest.SpyInstance = jest.spyOn(AIRunService, "create");
+
+    jest
+      .spyOn(TelemetryExceptionService, "findOneById")
+      .mockResolvedValue(fakeException({ isResolved: true }));
+
+    await expect(
+      TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true },
+      }),
+    ).rejects.toThrow(/marked as resolved/);
+
+    jest
+      .spyOn(TelemetryExceptionService, "findOneById")
+      .mockResolvedValue(fakeException({ isArchived: true }));
+
+    await expect(
+      TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true },
+      }),
+    ).rejects.toThrow(/archived/);
+
+    expect(create).not.toHaveBeenCalled();
+  });
+
+  test("a declined AI fix (aiFixDeclinedAt) blocks the automatic lane but a user click clears the stamp and proceeds", async () => {
+    mockReadinessOk();
+
+    jest
+      .spyOn(TelemetryExceptionService, "findOneById")
+      .mockResolvedValue(fakeException({ aiFixDeclinedAt: new Date() }));
+    jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(null);
+    const create: jest.SpyInstance = jest
+      .spyOn(AIRunService, "create")
+      .mockResolvedValue({ id: ObjectID.generate() } as unknown as AIRun);
+    const clearStamp: jest.SpyInstance = jest
+      .spyOn(TelemetryExceptionService, "updateOneById")
+      .mockResolvedValue(undefined as never);
+
+    // System-triggered (no userId): blocked.
+    await expect(
+      TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true },
+      }),
+    ).rejects.toThrow(/closed without merging/);
+    expect(create).not.toHaveBeenCalled();
+
+    // A human click: clears the stamp and creates the run.
+    await expect(
+      TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true, userId: ObjectID.generate() },
+      }),
+    ).resolves.toBeDefined();
+
+    expect(clearStamp).toHaveBeenCalledWith(
+      expect.objectContaining({
+        id: exceptionId,
+        data: expect.objectContaining({ aiFixDeclinedAt: null }),
+      }),
+    );
+    expect(create).toHaveBeenCalled();
+  });
+
+  test("the decline stamp is scoped to FixException: other recipes are neither blocked by it nor clear it", async () => {
+    mockReadinessOk();
+
+    jest
+      .spyOn(TelemetryExceptionService, "findOneById")
+      .mockResolvedValue(fakeException({ aiFixDeclinedAt: new Date() }));
+    jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(null);
+    const create: jest.SpyInstance = jest
+      .spyOn(AIRunService, "create")
+      .mockResolvedValue({ id: ObjectID.generate() } as unknown as AIRun);
+    const exceptionUpdates: jest.SpyInstance = jest
+      .spyOn(TelemetryExceptionService, "updateOneById")
+      .mockResolvedValue(undefined as never);
+
+    /*
+     * A user asking for a regression TEST on a group whose FIX a human
+     * declined: the test run proceeds, and — critically — the fix-decline
+     * suppression stays in force (the user never overrode the fix).
+     */
+    await expect(
+      TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true, userId: ObjectID.generate() },
+        taskType: CodeFixTaskType.WriteRegressionTest,
+      }),
+    ).resolves.toBeDefined();
+
+    expect(create).toHaveBeenCalled();
+    expect(exceptionUpdates).not.toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: expect.objectContaining({ aiFixDeclinedAt: null }),
+      }),
+    );
+  });
+
+  test("cross-group duplicate guard: a similar exception (same type + normalized message) with an active run blocks creation", async () => {
+    mockReadinessOk();
+
+    /*
+     * The observed flood shape: one throw site interpolating a different
+     * UUID per request splinters into one fingerprint (and one exception
+     * group) per value. Normalization collapses both messages to
+     * `...uuid: "<UUID>"`, which is what the guard compares.
+     */
+    jest.spyOn(TelemetryExceptionService, "findOneById").mockResolvedValue(
+      fakeException({
+        message:
+          'invalid input syntax for type uuid: "550e8400-e29b-41d4-a716-446655440000"',
+        exceptionType: "QueryFailedError",
+      }),
+    );
+
+    // Per-exception guard finds nothing for THIS exception...
+    jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(null);
+
+    /*
+     * ...but another exception group — same root cause, different
+     * interpolated value — has a non-terminal FixException run.
+     */
+    const similarExceptionId: ObjectID = ObjectID.generate();
+    jest.spyOn(AIRunService, "findBy").mockResolvedValue([
+      {
+        triggeredByTelemetryExceptionId: similarExceptionId,
+      } as unknown as AIRun,
+    ]);
+    jest.spyOn(TelemetryExceptionService, "findBy").mockResolvedValue([
+      {
+        _id: similarExceptionId.toString(),
+        message:
+          'invalid input syntax for type uuid: "123e4567-e89b-12d3-a456-426614174000"',
+        exceptionType: "QueryFailedError",
+      } as unknown as TelemetryException,
+    ]);
+
+    const create: jest.SpyInstance = jest.spyOn(AIRunService, "create");
+
+    await expect(
+      TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true },
+      }),
+    ).rejects.toThrow(/similar exception/);
+
+    expect(create).not.toHaveBeenCalled();
+  });
+
+  test("cross-group duplicate guard: a DIFFERENT normalized message does not block", async () => {
+    mockReadinessOk();
+
+    jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(null);
+
+    const otherExceptionId: ObjectID = ObjectID.generate();
+    jest.spyOn(AIRunService, "findBy").mockResolvedValue([
+      {
+        triggeredByTelemetryExceptionId: otherExceptionId,
+      } as unknown as AIRun,
+    ]);
+    jest.spyOn(TelemetryExceptionService, "findBy").mockResolvedValue([
+      {
+        _id: otherExceptionId.toString(),
+        message: "Connection refused to redis",
+        exceptionType: "ConnectionError",
+      } as unknown as TelemetryException,
+    ]);
+
+    const create: jest.SpyInstance = jest
+      .spyOn(AIRunService, "create")
+      .mockResolvedValue({ id: ObjectID.generate() } as unknown as AIRun);
+
+    await expect(
+      TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true },
+      }),
+    ).resolves.toBeDefined();
+
+    expect(create).toHaveBeenCalled();
+  });
 });
 
 /*