@oneuptime/common 12.0.4 → 12.0.6

package/Tests/Server/Utils/AI/InvestigationTldrWiring.test.ts

@@ -0,0 +1,98 @@
+import AIIncidentInvestigationRunner from "../../../../Server/Utils/AI/SRE/IncidentInvestigationRunner";
+import AIAlertInvestigationRunner from "../../../../Server/Utils/AI/SRE/AlertInvestigationRunner";
+import AIInvestigationEngine, {
+  InvestigationRequest,
+} from "../../../../Server/Utils/AI/SRE/AIInvestigationEngine";
+import IncidentAIContextBuilder, {
+  IncidentContextData,
+} from "../../../../Server/Utils/AI/IncidentAIContextBuilder";
+import AlertAIContextBuilder, {
+  AlertContextData,
+} from "../../../../Server/Utils/AI/AlertAIContextBuilder";
+import AIMemory from "../../../../Server/Utils/AI/SRE/AIMemory";
+import Alert from "../../../../Models/DatabaseModels/Alert";
+import Incident from "../../../../Models/DatabaseModels/Incident";
+import ObjectID from "../../../../Types/ObjectID";
+import { afterEach, beforeEach, describe, expect, test } from "@jest/globals";
+
+/*
+ * The panel's TL;DR only exists for runs whose runner asked for one. The
+ * opt-in is a single boolean on the engine request, and losing it is a silent
+ * regression: reports keep publishing, they just quietly stop carrying a
+ * summary. These tests pin the wiring for both subject lanes.
+ */
+
+const projectId: ObjectID = ObjectID.generate();
+const aiRunId: ObjectID = ObjectID.generate();
+const incidentId: ObjectID = ObjectID.generate();
+const alertId: ObjectID = ObjectID.generate();
+
+function sentRequest(executeRun: jest.SpyInstance): InvestigationRequest {
+  return (executeRun.mock.calls[0]![0] as { request: InvestigationRequest })
+    .request;
+}
+
+describe("Investigation runners request the AI TL;DR", () => {
+  let executeRun: jest.SpyInstance;
+
+  beforeEach(() => {
+    executeRun = jest
+      .spyOn(AIInvestigationEngine, "executeRun")
+      .mockResolvedValue(undefined);
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("an incident investigation opts in alongside the code-fix decision", async () => {
+    const incident: Incident = new Incident(incidentId);
+    incident.title = "Checkout 500s";
+    jest
+      .spyOn(IncidentAIContextBuilder, "buildIncidentContext")
+      .mockResolvedValue({
+        incident,
+        stateTimeline: [],
+        internalNotes: [],
+        publicNotes: [],
+        workspaceMessages: [],
+      } as unknown as IncidentContextData);
+    jest
+      .spyOn(AIMemory, "getPriorSimilarIncidentsContext")
+      .mockResolvedValue("");
+
+    await AIIncidentInvestigationRunner.executeInvestigation({
+      aiRunId,
+      projectId,
+      incidentId,
+      attemptCount: 1,
+    });
+
+    expect(executeRun).toHaveBeenCalledTimes(1);
+    expect(sentRequest(executeRun).persistAnalysisTldr).toBe(true);
+    expect(sentRequest(executeRun).persistCodeFixRecommendation).toBe(true);
+    expect(sentRequest(executeRun).incidentId).toEqual(incidentId);
+  });
+
+  test("an alert investigation opts in too", async () => {
+    const alert: Alert = new Alert(alertId);
+    alert.title = "Error rate above threshold";
+    jest.spyOn(AlertAIContextBuilder, "buildAlertContext").mockResolvedValue({
+      alert,
+      stateTimeline: [],
+      internalNotes: [],
+    } as unknown as AlertContextData);
+
+    await AIAlertInvestigationRunner.executeInvestigation({
+      aiRunId,
+      projectId,
+      alertId,
+      attemptCount: 1,
+    });
+
+    expect(executeRun).toHaveBeenCalledTimes(1);
+    expect(sentRequest(executeRun).persistAnalysisTldr).toBe(true);
+    expect(sentRequest(executeRun).persistCodeFixRecommendation).toBe(true);
+    expect(sentRequest(executeRun).alertId).toEqual(alertId);
+  });
+});

package/Tests/Server/Utils/AI/PostedRootCause.test.ts

@@ -10,6 +10,7 @@ import IncidentFeed, {
 import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
 import ObjectID from "../../../../Types/ObjectID";
 import { afterEach, describe, expect, it } from "@jest/globals";
+import { FindOperator } from "typeorm";
 
 /*
  * The one reader of the analysis an AI investigation posted.
@@ -31,6 +32,9 @@ const INCIDENT_ID: ObjectID = new ObjectID(
   "cccccccc-cccc-4ccc-8ccc-cccccccccccc",
 );
 const ALERT_ID: ObjectID = new ObjectID("dddddddd-dddd-4ddd-8ddd-dddddddddddd");
+const AI_RUN_ID: ObjectID = new ObjectID(
+  "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee",
+);
 
 function incidentFeedItem(markdown: string | null): IncidentFeed {
   return {
@@ -44,6 +48,23 @@ function alertFeedItem(markdown: string | null): AlertFeed {
   } as unknown as AlertFeed;
 }
 
+interface RawFindOperatorInternals {
+  _getSql: (columnName: string) => string;
+  _objectLiteralParameters: Record<string, unknown>;
+}
+
+function expectCreatedAtCutoff(
+  createdAtFilter: unknown,
+  expectedCutoff: Date,
+): void {
+  expect(createdAtFilter).toBeInstanceOf(FindOperator);
+
+  const raw: RawFindOperatorInternals =
+    createdAtFilter as RawFindOperatorInternals;
+  expect(raw._getSql("COLUMN")).toContain("COLUMN >=");
+  expect(Object.values(raw._objectLiteralParameters)).toEqual([expectedCutoff]);
+}
+
 describe("PostedRootCause.getForIncident", () => {
   afterEach(() => {
     jest.restoreAllMocks();
@@ -85,6 +106,73 @@ describe("PostedRootCause.getForIncident", () => {
         }),
       }),
     );
+    const query: Record<string, unknown> = find.mock.calls[0]![0]
+      .query as Record<string, unknown>;
+    expect(Object.prototype.hasOwnProperty.call(query, "createdAt")).toBe(
+      false,
+    );
+    expect(Object.prototype.hasOwnProperty.call(query, "aiRunId")).toBe(false);
+  });
+
+  it("can select the RootCause explicitly owned by one investigation run", async () => {
+    const find: jest.SpyInstance = jest
+      .spyOn(IncidentFeedService, "findOneBy")
+      .mockResolvedValue(null);
+
+    await PostedRootCause.getForIncident(INCIDENT_ID, {
+      aiRunId: AI_RUN_ID,
+    });
+
+    expect(find).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.objectContaining({
+          incidentId: INCIDENT_ID,
+          incidentFeedEventType: IncidentFeedEventType.RootCause,
+          aiRunId: AI_RUN_ID,
+        }),
+      }),
+    );
+  });
+
+  it("can restrict an upgrade lookup to rows without an AI run association", async () => {
+    const find: jest.SpyInstance = jest
+      .spyOn(IncidentFeedService, "findOneBy")
+      .mockResolvedValue(null);
+
+    await PostedRootCause.getForIncident(INCIDENT_ID, {
+      withoutAIRunId: true,
+    });
+
+    const aiRunIdFilter: unknown = find.mock.calls[0]![0].query.aiRunId;
+    expect(aiRunIdFilter).toBeInstanceOf(FindOperator);
+    expect(
+      (aiRunIdFilter as RawFindOperatorInternals)._getSql("COLUMN"),
+    ).toContain("COLUMN IS NULL");
+  });
+
+  it("can scope the RootCause item to feed rows created by this run", async () => {
+    const runCompletedAt: Date = new Date("2026-08-07T10:00:00.000Z");
+    const find: jest.SpyInstance = jest
+      .spyOn(IncidentFeedService, "findOneBy")
+      .mockResolvedValue(null);
+
+    await PostedRootCause.getForIncident(INCIDENT_ID, {
+      createdAtOrAfter: runCompletedAt,
+    });
+
+    expect(find).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.objectContaining({
+          incidentId: INCIDENT_ID,
+          incidentFeedEventType: IncidentFeedEventType.RootCause,
+          createdAt: expect.any(FindOperator),
+        }),
+      }),
+    );
+    expectCreatedAtCutoff(
+      find.mock.calls[0]![0].query.createdAt,
+      runCompletedAt,
+    );
   });
 
   it("returns null when no analysis has been posted", async () => {
@@ -166,6 +254,57 @@ describe("PostedRootCause.getForAlert", () => {
         }),
       }),
     );
+    const query: Record<string, unknown> = find.mock.calls[0]![0]
+      .query as Record<string, unknown>;
+    expect(Object.prototype.hasOwnProperty.call(query, "createdAt")).toBe(
+      false,
+    );
+    expect(Object.prototype.hasOwnProperty.call(query, "aiRunId")).toBe(false);
+  });
+
+  it("can select the RootCause explicitly owned by one alert investigation", async () => {
+    const find: jest.SpyInstance = jest
+      .spyOn(AlertFeedService, "findOneBy")
+      .mockResolvedValue(null);
+
+    await PostedRootCause.getForAlert(ALERT_ID, {
+      aiRunId: AI_RUN_ID,
+    });
+
+    expect(find).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.objectContaining({
+          alertId: ALERT_ID,
+          alertFeedEventType: AlertFeedEventType.RootCause,
+          aiRunId: AI_RUN_ID,
+        }),
+      }),
+    );
+  });
+
+  it("can scope the RootCause item to alert feed rows created by this run", async () => {
+    const runCompletedAt: Date = new Date("2026-08-07T11:00:00.000Z");
+    const find: jest.SpyInstance = jest
+      .spyOn(AlertFeedService, "findOneBy")
+      .mockResolvedValue(null);
+
+    await PostedRootCause.getForAlert(ALERT_ID, {
+      createdAtOrAfter: runCompletedAt,
+    });
+
+    expect(find).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.objectContaining({
+          alertId: ALERT_ID,
+          alertFeedEventType: AlertFeedEventType.RootCause,
+          createdAt: expect.any(FindOperator),
+        }),
+      }),
+    );
+    expectCreatedAtCutoff(
+      find.mock.calls[0]![0].query.createdAt,
+      runCompletedAt,
+    );
   });
 
   it("returns null when no analysis has been posted", async () => {
@@ -196,6 +335,43 @@ describe("PostedRootCause.getForSubject", () => {
     expect(alertFind).not.toHaveBeenCalled();
   });
 
+  it("forwards the run boundary to the subject-specific reader", async () => {
+    const runCompletedAt: Date = new Date("2026-08-07T12:00:00.000Z");
+    const getForIncident: jest.SpyInstance = jest
+      .spyOn(PostedRootCause, "getForIncident")
+      .mockResolvedValue("current analysis");
+
+    expect(
+      await PostedRootCause.getForSubject({
+        incidentId: INCIDENT_ID,
+        createdAtOrAfter: runCompletedAt,
+      }),
+    ).toBe("current analysis");
+    expect(getForIncident).toHaveBeenCalledWith(INCIDENT_ID, {
+      aiRunId: undefined,
+      withoutAIRunId: undefined,
+      createdAtOrAfter: runCompletedAt,
+    });
+  });
+
+  it("forwards the exact run id to the subject-specific reader", async () => {
+    const getForAlert: jest.SpyInstance = jest
+      .spyOn(PostedRootCause, "getForAlert")
+      .mockResolvedValue("current alert analysis");
+
+    expect(
+      await PostedRootCause.getForSubject({
+        alertId: ALERT_ID,
+        aiRunId: AI_RUN_ID,
+      }),
+    ).toBe("current alert analysis");
+    expect(getForAlert).toHaveBeenCalledWith(ALERT_ID, {
+      aiRunId: AI_RUN_ID,
+      withoutAIRunId: undefined,
+      createdAtOrAfter: undefined,
+    });
+  });
+
   it("reads the alert feed for an alert subject", async () => {
     const alertFind: jest.SpyInstance = jest
       .spyOn(AlertFeedService, "findOneBy")
@@ -252,3 +428,81 @@ describe("PostedRootCause.getForSubject", () => {
     expect(alertFind).not.toHaveBeenCalled();
   });
 });
+
+describe("PostedRootCause.getForInvestigation", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it("returns the exact run-associated report without a legacy lookup", async () => {
+    const getForSubject: jest.SpyInstance = jest
+      .spyOn(PostedRootCause, "getForSubject")
+      .mockResolvedValue("exact report");
+
+    expect(
+      await PostedRootCause.getForInvestigation({
+        incidentId: INCIDENT_ID,
+        aiRunId: AI_RUN_ID,
+      }),
+    ).toBe("exact report");
+    expect(getForSubject).toHaveBeenCalledTimes(1);
+    expect(getForSubject).toHaveBeenCalledWith({
+      incidentId: INCIDENT_ID,
+      alertId: undefined,
+      aiRunId: AI_RUN_ID,
+    });
+  });
+
+  it("recovers a branded null-associated report across a deployment-safe upgrade window", async () => {
+    const runCompletedAt: Date = new Date("2026-08-08T10:00:00.000Z");
+    const legacyReport: string =
+      "## AI — Automated Root Cause Analysis\n\nHistorical evidence.";
+    const getForSubject: jest.SpyInstance = jest
+      .spyOn(PostedRootCause, "getForSubject")
+      .mockResolvedValueOnce(null)
+      .mockResolvedValueOnce(legacyReport);
+
+    expect(
+      await PostedRootCause.getForInvestigation({
+        alertId: ALERT_ID,
+        aiRunId: AI_RUN_ID,
+        runCompletedAt,
+      }),
+    ).toBe(legacyReport);
+    expect(getForSubject).toHaveBeenNthCalledWith(2, {
+      incidentId: undefined,
+      alertId: ALERT_ID,
+      withoutAIRunId: true,
+      createdAtOrAfter: runCompletedAt,
+    });
+  });
+
+  it("rejects ordinary RootCause markdown from the legacy path", async () => {
+    jest
+      .spyOn(PostedRootCause, "getForSubject")
+      .mockResolvedValueOnce(null)
+      .mockResolvedValueOnce("## Root cause\n\nWritten by an engineer.");
+
+    expect(
+      await PostedRootCause.getForInvestigation({
+        incidentId: INCIDENT_ID,
+        aiRunId: AI_RUN_ID,
+        runCompletedAt: new Date("2026-08-08T10:00:00.000Z"),
+      }),
+    ).toBeNull();
+  });
+
+  it("never falls back to an unassociated row without a reliable completion boundary", async () => {
+    const getForSubject: jest.SpyInstance = jest
+      .spyOn(PostedRootCause, "getForSubject")
+      .mockResolvedValue(null);
+
+    expect(
+      await PostedRootCause.getForInvestigation({
+        incidentId: INCIDENT_ID,
+        aiRunId: AI_RUN_ID,
+      }),
+    ).toBeNull();
+    expect(getForSubject).toHaveBeenCalledTimes(1);
+  });
+});

package/Tests/Server/Utils/AI/RemediationExecutionRunner.test.ts

@@ -547,6 +547,8 @@ describe("RemediationExecutionRunner.executeRemediation", () => {
       );
       // Persisted budget-ledger string — pinned as a literal on purpose.
       expect(request.get().feature).toBe("AI Remediation Execution");
+      expect(request.get().incidentId).toBe(INCIDENT_ID);
+      expect(request.get().alertId).toBeUndefined();
     });
 
     it("runs FullAuto with the execute tool and the allowlist in context when every condition holds", async () => {
@@ -859,6 +861,8 @@ describe("RemediationExecutionRunner.executeRemediation", () => {
         '<untrusted_context source="alert_text">',
       );
       expect(captured!.contextSummary).toContain("Disk almost full on db-1");
+      expect(captured!.incidentId).toBeUndefined();
+      expect(captured!.alertId).toBe(ALERT_ID);
       expect(incidentFeed).not.toHaveBeenCalled();
       expect(alertFeed).toHaveBeenCalledWith(
         expect.objectContaining({

package/Tests/Server/Utils/AI/RemediationPlanRunner.test.ts

@@ -260,6 +260,8 @@ describe("RemediationPlanRunner.executePlan", () => {
         request: expect.objectContaining({
           // Persisted budget-ledger string — pinned as a literal on purpose.
           feature: "AI Remediation Planning",
+          incidentId: INCIDENT_ID,
+          alertId: undefined,
         }),
       }),
     );

package/Tests/Server/Utils/AI/SubjectCodeFixRunDedupe.test.ts

@@ -1,4 +1,5 @@
 import SubjectCodeFixRun from "../../../../Server/Utils/AI/SRE/SubjectCodeFixRun";
+import FixRunBudget from "../../../../Server/Utils/AI/CodeFix/FixRunBudget";
 import AIRunService from "../../../../Server/Services/AIRunService";
 import QueryHelper from "../../../../Server/Types/Database/QueryHelper";
 import AIRun from "../../../../Models/DatabaseModels/AIRun";
@@ -119,6 +120,27 @@ describe("SubjectCodeFixRun dedupe guards", () => {
   });
 
   describe("findNonTerminalRunForSubject", () => {
+    test.each([
+      { label: "neither", input: {} },
+      { label: "both", input: { incidentId, alertId } },
+    ])(
+      "rejects $label subject type before querying",
+      async ({
+        input,
+      }: {
+        input: { incidentId?: ObjectID; alertId?: ObjectID };
+      }) => {
+        await expect(
+          SubjectCodeFixRun.findNonTerminalRunForSubject({
+            taskType: CodeFixTaskType.FixFromIncident,
+            ...input,
+          }),
+        ).rejects.toThrow(/Exactly one/);
+
+        expect(AIRunService.findOneBy).not.toHaveBeenCalled();
+      },
+    );
+
     test("a NoFixFound run does NOT block a new run for the same subject", async () => {
       storeRun(AIRunStatus.NoFixFound);
 
@@ -176,6 +198,28 @@ describe("SubjectCodeFixRun dedupe guards", () => {
     });
   });
 
+  describe("enqueueSubjectCodeFixRun", () => {
+    test("rejects a dual-subject run before the budget check or write", async () => {
+      const assertWithinBudget: jest.SpyInstance = jest.spyOn(
+        FixRunBudget,
+        "assertWithinBudget",
+      );
+      const create: jest.SpyInstance = jest.spyOn(AIRunService, "create");
+
+      await expect(
+        SubjectCodeFixRun.enqueueSubjectCodeFixRun({
+          projectId,
+          taskType: CodeFixTaskType.FixFromIncident,
+          incidentId,
+          alertId,
+        }),
+      ).rejects.toThrow(/both an incident and an alert/);
+
+      expect(assertWithinBudget).not.toHaveBeenCalled();
+      expect(create).not.toHaveBeenCalled();
+    });
+  });
+
   describe("findNonTerminalPerformanceFixRunForTrace", () => {
     test("a NoFixFound run does NOT block a new run for the same trace", async () => {
       storeRun(AIRunStatus.NoFixFound);

package/Tests/Server/Utils/AnalyticsDatabase/ClusterAwareSchema.test.ts

@@ -8,6 +8,7 @@ import {
   adaptTableSettingsForStorage,
   applyClusterToMaterializedViewQuery,
   getClickhouseClusterName,
+  getDistributedDdlTaskTimeoutSeconds,
   getDistributedEngine,
   getStorageEngine,
   getStorageTableName,
@@ -33,6 +34,8 @@ import MetricBaselineHourly from "../../../../Models/AnalyticsModels/MetricBasel
 
 const CLUSTER_ENV_KEY: string = "CLICKHOUSE_CLUSTER_NAME";
 const SHARDING_ENV_KEY: string = "CLICKHOUSE_SHARDING_KEY";
+const DDL_TIMEOUT_ENV_KEY: string =
+  "CLICKHOUSE_DISTRIBUTED_DDL_TASK_TIMEOUT_SECONDS";
 
 /*
  * Flatten a Statement into a single string (raw SQL + serialized identifier
@@ -49,6 +52,8 @@ function fullText(statement: Statement | string): string {
 describe("ClickHouse cluster-aware schema (always-on)", () => {
   const originalCluster: string | undefined = process.env[CLUSTER_ENV_KEY];
   const originalSharding: string | undefined = process.env[SHARDING_ENV_KEY];
+  const originalDdlTimeout: string | undefined =
+    process.env[DDL_TIMEOUT_ENV_KEY];
 
   afterEach(() => {
     if (originalCluster === undefined) {
@@ -61,6 +66,11 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
     } else {
       process.env[SHARDING_ENV_KEY] = originalSharding;
     }
+    if (originalDdlTimeout === undefined) {
+      delete process.env[DDL_TIMEOUT_ENV_KEY];
+    } else {
+      process.env[DDL_TIMEOUT_ENV_KEY] = originalDdlTimeout;
+    }
   });
 
   describe("ClusterConfig helpers", () => {
@@ -101,6 +111,26 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
       );
     });
 
+    test("distributed DDL task timeout: default 180, overridable, garbage-safe", () => {
+      delete process.env[DDL_TIMEOUT_ENV_KEY];
+      expect(getDistributedDdlTaskTimeoutSeconds()).toBe(180);
+
+      process.env[DDL_TIMEOUT_ENV_KEY] = "1800";
+      expect(getDistributedDdlTaskTimeoutSeconds()).toBe(1800);
+
+      // ClickHouse semantics pass through: 0 = async, negative = no server cap.
+      process.env[DDL_TIMEOUT_ENV_KEY] = "0";
+      expect(getDistributedDdlTaskTimeoutSeconds()).toBe(0);
+      process.env[DDL_TIMEOUT_ENV_KEY] = "-1";
+      expect(getDistributedDdlTaskTimeoutSeconds()).toBe(-1);
+
+      // Garbage and whitespace fall back to the default.
+      process.env[DDL_TIMEOUT_ENV_KEY] = "not-a-number";
+      expect(getDistributedDdlTaskTimeoutSeconds()).toBe(180);
+      process.env[DDL_TIMEOUT_ENV_KEY] = "   ";
+      expect(getDistributedDdlTaskTimeoutSeconds()).toBe(180);
+    });
+
     test("distributed engine: model key, with global override winning", () => {
       delete process.env[SHARDING_ENV_KEY];
       // model sharding key is used