@oneuptime/common 12.0.3 → 12.0.5

package/Tests/Server/Utils/AI/PostedRootCause.test.ts

@@ -10,6 +10,7 @@ import IncidentFeed, {
 import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
 import ObjectID from "../../../../Types/ObjectID";
 import { afterEach, describe, expect, it } from "@jest/globals";
+import { FindOperator } from "typeorm";
 
 /*
  * The one reader of the analysis an AI investigation posted.
@@ -31,6 +32,9 @@ const INCIDENT_ID: ObjectID = new ObjectID(
   "cccccccc-cccc-4ccc-8ccc-cccccccccccc",
 );
 const ALERT_ID: ObjectID = new ObjectID("dddddddd-dddd-4ddd-8ddd-dddddddddddd");
+const AI_RUN_ID: ObjectID = new ObjectID(
+  "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee",
+);
 
 function incidentFeedItem(markdown: string | null): IncidentFeed {
   return {
@@ -44,6 +48,23 @@ function alertFeedItem(markdown: string | null): AlertFeed {
   } as unknown as AlertFeed;
 }
 
+interface RawFindOperatorInternals {
+  _getSql: (columnName: string) => string;
+  _objectLiteralParameters: Record<string, unknown>;
+}
+
+function expectCreatedAtCutoff(
+  createdAtFilter: unknown,
+  expectedCutoff: Date,
+): void {
+  expect(createdAtFilter).toBeInstanceOf(FindOperator);
+
+  const raw: RawFindOperatorInternals =
+    createdAtFilter as RawFindOperatorInternals;
+  expect(raw._getSql("COLUMN")).toContain("COLUMN >=");
+  expect(Object.values(raw._objectLiteralParameters)).toEqual([expectedCutoff]);
+}
+
 describe("PostedRootCause.getForIncident", () => {
   afterEach(() => {
     jest.restoreAllMocks();
@@ -85,6 +106,73 @@ describe("PostedRootCause.getForIncident", () => {
         }),
       }),
     );
+    const query: Record<string, unknown> = find.mock.calls[0]![0]
+      .query as Record<string, unknown>;
+    expect(Object.prototype.hasOwnProperty.call(query, "createdAt")).toBe(
+      false,
+    );
+    expect(Object.prototype.hasOwnProperty.call(query, "aiRunId")).toBe(false);
+  });
+
+  it("can select the RootCause explicitly owned by one investigation run", async () => {
+    const find: jest.SpyInstance = jest
+      .spyOn(IncidentFeedService, "findOneBy")
+      .mockResolvedValue(null);
+
+    await PostedRootCause.getForIncident(INCIDENT_ID, {
+      aiRunId: AI_RUN_ID,
+    });
+
+    expect(find).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.objectContaining({
+          incidentId: INCIDENT_ID,
+          incidentFeedEventType: IncidentFeedEventType.RootCause,
+          aiRunId: AI_RUN_ID,
+        }),
+      }),
+    );
+  });
+
+  it("can restrict an upgrade lookup to rows without an AI run association", async () => {
+    const find: jest.SpyInstance = jest
+      .spyOn(IncidentFeedService, "findOneBy")
+      .mockResolvedValue(null);
+
+    await PostedRootCause.getForIncident(INCIDENT_ID, {
+      withoutAIRunId: true,
+    });
+
+    const aiRunIdFilter: unknown = find.mock.calls[0]![0].query.aiRunId;
+    expect(aiRunIdFilter).toBeInstanceOf(FindOperator);
+    expect(
+      (aiRunIdFilter as RawFindOperatorInternals)._getSql("COLUMN"),
+    ).toContain("COLUMN IS NULL");
+  });
+
+  it("can scope the RootCause item to feed rows created by this run", async () => {
+    const runCompletedAt: Date = new Date("2026-08-07T10:00:00.000Z");
+    const find: jest.SpyInstance = jest
+      .spyOn(IncidentFeedService, "findOneBy")
+      .mockResolvedValue(null);
+
+    await PostedRootCause.getForIncident(INCIDENT_ID, {
+      createdAtOrAfter: runCompletedAt,
+    });
+
+    expect(find).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.objectContaining({
+          incidentId: INCIDENT_ID,
+          incidentFeedEventType: IncidentFeedEventType.RootCause,
+          createdAt: expect.any(FindOperator),
+        }),
+      }),
+    );
+    expectCreatedAtCutoff(
+      find.mock.calls[0]![0].query.createdAt,
+      runCompletedAt,
+    );
   });
 
   it("returns null when no analysis has been posted", async () => {
@@ -166,6 +254,57 @@ describe("PostedRootCause.getForAlert", () => {
         }),
       }),
     );
+    const query: Record<string, unknown> = find.mock.calls[0]![0]
+      .query as Record<string, unknown>;
+    expect(Object.prototype.hasOwnProperty.call(query, "createdAt")).toBe(
+      false,
+    );
+    expect(Object.prototype.hasOwnProperty.call(query, "aiRunId")).toBe(false);
+  });
+
+  it("can select the RootCause explicitly owned by one alert investigation", async () => {
+    const find: jest.SpyInstance = jest
+      .spyOn(AlertFeedService, "findOneBy")
+      .mockResolvedValue(null);
+
+    await PostedRootCause.getForAlert(ALERT_ID, {
+      aiRunId: AI_RUN_ID,
+    });
+
+    expect(find).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.objectContaining({
+          alertId: ALERT_ID,
+          alertFeedEventType: AlertFeedEventType.RootCause,
+          aiRunId: AI_RUN_ID,
+        }),
+      }),
+    );
+  });
+
+  it("can scope the RootCause item to alert feed rows created by this run", async () => {
+    const runCompletedAt: Date = new Date("2026-08-07T11:00:00.000Z");
+    const find: jest.SpyInstance = jest
+      .spyOn(AlertFeedService, "findOneBy")
+      .mockResolvedValue(null);
+
+    await PostedRootCause.getForAlert(ALERT_ID, {
+      createdAtOrAfter: runCompletedAt,
+    });
+
+    expect(find).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.objectContaining({
+          alertId: ALERT_ID,
+          alertFeedEventType: AlertFeedEventType.RootCause,
+          createdAt: expect.any(FindOperator),
+        }),
+      }),
+    );
+    expectCreatedAtCutoff(
+      find.mock.calls[0]![0].query.createdAt,
+      runCompletedAt,
+    );
   });
 
   it("returns null when no analysis has been posted", async () => {
@@ -196,6 +335,43 @@ describe("PostedRootCause.getForSubject", () => {
     expect(alertFind).not.toHaveBeenCalled();
   });
 
+  it("forwards the run boundary to the subject-specific reader", async () => {
+    const runCompletedAt: Date = new Date("2026-08-07T12:00:00.000Z");
+    const getForIncident: jest.SpyInstance = jest
+      .spyOn(PostedRootCause, "getForIncident")
+      .mockResolvedValue("current analysis");
+
+    expect(
+      await PostedRootCause.getForSubject({
+        incidentId: INCIDENT_ID,
+        createdAtOrAfter: runCompletedAt,
+      }),
+    ).toBe("current analysis");
+    expect(getForIncident).toHaveBeenCalledWith(INCIDENT_ID, {
+      aiRunId: undefined,
+      withoutAIRunId: undefined,
+      createdAtOrAfter: runCompletedAt,
+    });
+  });
+
+  it("forwards the exact run id to the subject-specific reader", async () => {
+    const getForAlert: jest.SpyInstance = jest
+      .spyOn(PostedRootCause, "getForAlert")
+      .mockResolvedValue("current alert analysis");
+
+    expect(
+      await PostedRootCause.getForSubject({
+        alertId: ALERT_ID,
+        aiRunId: AI_RUN_ID,
+      }),
+    ).toBe("current alert analysis");
+    expect(getForAlert).toHaveBeenCalledWith(ALERT_ID, {
+      aiRunId: AI_RUN_ID,
+      withoutAIRunId: undefined,
+      createdAtOrAfter: undefined,
+    });
+  });
+
   it("reads the alert feed for an alert subject", async () => {
     const alertFind: jest.SpyInstance = jest
       .spyOn(AlertFeedService, "findOneBy")
@@ -252,3 +428,81 @@ describe("PostedRootCause.getForSubject", () => {
     expect(alertFind).not.toHaveBeenCalled();
   });
 });
+
+describe("PostedRootCause.getForInvestigation", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  it("returns the exact run-associated report without a legacy lookup", async () => {
+    const getForSubject: jest.SpyInstance = jest
+      .spyOn(PostedRootCause, "getForSubject")
+      .mockResolvedValue("exact report");
+
+    expect(
+      await PostedRootCause.getForInvestigation({
+        incidentId: INCIDENT_ID,
+        aiRunId: AI_RUN_ID,
+      }),
+    ).toBe("exact report");
+    expect(getForSubject).toHaveBeenCalledTimes(1);
+    expect(getForSubject).toHaveBeenCalledWith({
+      incidentId: INCIDENT_ID,
+      alertId: undefined,
+      aiRunId: AI_RUN_ID,
+    });
+  });
+
+  it("recovers a branded null-associated report across a deployment-safe upgrade window", async () => {
+    const runCompletedAt: Date = new Date("2026-08-08T10:00:00.000Z");
+    const legacyReport: string =
+      "## AI — Automated Root Cause Analysis\n\nHistorical evidence.";
+    const getForSubject: jest.SpyInstance = jest
+      .spyOn(PostedRootCause, "getForSubject")
+      .mockResolvedValueOnce(null)
+      .mockResolvedValueOnce(legacyReport);
+
+    expect(
+      await PostedRootCause.getForInvestigation({
+        alertId: ALERT_ID,
+        aiRunId: AI_RUN_ID,
+        runCompletedAt,
+      }),
+    ).toBe(legacyReport);
+    expect(getForSubject).toHaveBeenNthCalledWith(2, {
+      incidentId: undefined,
+      alertId: ALERT_ID,
+      withoutAIRunId: true,
+      createdAtOrAfter: runCompletedAt,
+    });
+  });
+
+  it("rejects ordinary RootCause markdown from the legacy path", async () => {
+    jest
+      .spyOn(PostedRootCause, "getForSubject")
+      .mockResolvedValueOnce(null)
+      .mockResolvedValueOnce("## Root cause\n\nWritten by an engineer.");
+
+    expect(
+      await PostedRootCause.getForInvestigation({
+        incidentId: INCIDENT_ID,
+        aiRunId: AI_RUN_ID,
+        runCompletedAt: new Date("2026-08-08T10:00:00.000Z"),
+      }),
+    ).toBeNull();
+  });
+
+  it("never falls back to an unassociated row without a reliable completion boundary", async () => {
+    const getForSubject: jest.SpyInstance = jest
+      .spyOn(PostedRootCause, "getForSubject")
+      .mockResolvedValue(null);
+
+    expect(
+      await PostedRootCause.getForInvestigation({
+        incidentId: INCIDENT_ID,
+        aiRunId: AI_RUN_ID,
+      }),
+    ).toBeNull();
+    expect(getForSubject).toHaveBeenCalledTimes(1);
+  });
+});

package/Tests/Server/Utils/AI/RemediationExecutionRunner.test.ts

@@ -547,6 +547,8 @@ describe("RemediationExecutionRunner.executeRemediation", () => {
       );
       // Persisted budget-ledger string — pinned as a literal on purpose.
       expect(request.get().feature).toBe("AI Remediation Execution");
+      expect(request.get().incidentId).toBe(INCIDENT_ID);
+      expect(request.get().alertId).toBeUndefined();
     });
 
     it("runs FullAuto with the execute tool and the allowlist in context when every condition holds", async () => {
@@ -859,6 +861,8 @@ describe("RemediationExecutionRunner.executeRemediation", () => {
         '<untrusted_context source="alert_text">',
       );
       expect(captured!.contextSummary).toContain("Disk almost full on db-1");
+      expect(captured!.incidentId).toBeUndefined();
+      expect(captured!.alertId).toBe(ALERT_ID);
       expect(incidentFeed).not.toHaveBeenCalled();
       expect(alertFeed).toHaveBeenCalledWith(
         expect.objectContaining({

package/Tests/Server/Utils/AI/RemediationPlanRunner.test.ts

@@ -260,6 +260,8 @@ describe("RemediationPlanRunner.executePlan", () => {
         request: expect.objectContaining({
           // Persisted budget-ledger string — pinned as a literal on purpose.
           feature: "AI Remediation Planning",
+          incidentId: INCIDENT_ID,
+          alertId: undefined,
         }),
       }),
     );

package/Tests/Server/Utils/AI/SubjectCodeFixRunDedupe.test.ts

@@ -1,4 +1,5 @@
 import SubjectCodeFixRun from "../../../../Server/Utils/AI/SRE/SubjectCodeFixRun";
+import FixRunBudget from "../../../../Server/Utils/AI/CodeFix/FixRunBudget";
 import AIRunService from "../../../../Server/Services/AIRunService";
 import QueryHelper from "../../../../Server/Types/Database/QueryHelper";
 import AIRun from "../../../../Models/DatabaseModels/AIRun";
@@ -119,6 +120,27 @@ describe("SubjectCodeFixRun dedupe guards", () => {
   });
 
   describe("findNonTerminalRunForSubject", () => {
+    test.each([
+      { label: "neither", input: {} },
+      { label: "both", input: { incidentId, alertId } },
+    ])(
+      "rejects $label subject type before querying",
+      async ({
+        input,
+      }: {
+        input: { incidentId?: ObjectID; alertId?: ObjectID };
+      }) => {
+        await expect(
+          SubjectCodeFixRun.findNonTerminalRunForSubject({
+            taskType: CodeFixTaskType.FixFromIncident,
+            ...input,
+          }),
+        ).rejects.toThrow(/Exactly one/);
+
+        expect(AIRunService.findOneBy).not.toHaveBeenCalled();
+      },
+    );
+
     test("a NoFixFound run does NOT block a new run for the same subject", async () => {
       storeRun(AIRunStatus.NoFixFound);
 
@@ -176,6 +198,28 @@ describe("SubjectCodeFixRun dedupe guards", () => {
     });
   });
 
+  describe("enqueueSubjectCodeFixRun", () => {
+    test("rejects a dual-subject run before the budget check or write", async () => {
+      const assertWithinBudget: jest.SpyInstance = jest.spyOn(
+        FixRunBudget,
+        "assertWithinBudget",
+      );
+      const create: jest.SpyInstance = jest.spyOn(AIRunService, "create");
+
+      await expect(
+        SubjectCodeFixRun.enqueueSubjectCodeFixRun({
+          projectId,
+          taskType: CodeFixTaskType.FixFromIncident,
+          incidentId,
+          alertId,
+        }),
+      ).rejects.toThrow(/both an incident and an alert/);
+
+      expect(assertWithinBudget).not.toHaveBeenCalled();
+      expect(create).not.toHaveBeenCalled();
+    });
+  });
+
   describe("findNonTerminalPerformanceFixRunForTrace", () => {
     test("a NoFixFound run does NOT block a new run for the same trace", async () => {
       storeRun(AIRunStatus.NoFixFound);

package/Tests/Server/Utils/AnalyticsDatabase/ClusterAwareSchema.test.ts

@@ -8,6 +8,7 @@ import {
   adaptTableSettingsForStorage,
   applyClusterToMaterializedViewQuery,
   getClickhouseClusterName,
+  getDistributedDdlTaskTimeoutSeconds,
   getDistributedEngine,
   getStorageEngine,
   getStorageTableName,
@@ -33,6 +34,8 @@ import MetricBaselineHourly from "../../../../Models/AnalyticsModels/MetricBasel
 
 const CLUSTER_ENV_KEY: string = "CLICKHOUSE_CLUSTER_NAME";
 const SHARDING_ENV_KEY: string = "CLICKHOUSE_SHARDING_KEY";
+const DDL_TIMEOUT_ENV_KEY: string =
+  "CLICKHOUSE_DISTRIBUTED_DDL_TASK_TIMEOUT_SECONDS";
 
 /*
  * Flatten a Statement into a single string (raw SQL + serialized identifier
@@ -49,6 +52,8 @@ function fullText(statement: Statement | string): string {
 describe("ClickHouse cluster-aware schema (always-on)", () => {
   const originalCluster: string | undefined = process.env[CLUSTER_ENV_KEY];
   const originalSharding: string | undefined = process.env[SHARDING_ENV_KEY];
+  const originalDdlTimeout: string | undefined =
+    process.env[DDL_TIMEOUT_ENV_KEY];
 
   afterEach(() => {
     if (originalCluster === undefined) {
@@ -61,6 +66,11 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
     } else {
       process.env[SHARDING_ENV_KEY] = originalSharding;
     }
+    if (originalDdlTimeout === undefined) {
+      delete process.env[DDL_TIMEOUT_ENV_KEY];
+    } else {
+      process.env[DDL_TIMEOUT_ENV_KEY] = originalDdlTimeout;
+    }
   });
 
   describe("ClusterConfig helpers", () => {
@@ -101,6 +111,26 @@ describe("ClickHouse cluster-aware schema (always-on)", () => {
       );
     });
 
+    test("distributed DDL task timeout: default 180, overridable, garbage-safe", () => {
+      delete process.env[DDL_TIMEOUT_ENV_KEY];
+      expect(getDistributedDdlTaskTimeoutSeconds()).toBe(180);
+
+      process.env[DDL_TIMEOUT_ENV_KEY] = "1800";
+      expect(getDistributedDdlTaskTimeoutSeconds()).toBe(1800);
+
+      // ClickHouse semantics pass through: 0 = async, negative = no server cap.
+      process.env[DDL_TIMEOUT_ENV_KEY] = "0";
+      expect(getDistributedDdlTaskTimeoutSeconds()).toBe(0);
+      process.env[DDL_TIMEOUT_ENV_KEY] = "-1";
+      expect(getDistributedDdlTaskTimeoutSeconds()).toBe(-1);
+
+      // Garbage and whitespace fall back to the default.
+      process.env[DDL_TIMEOUT_ENV_KEY] = "not-a-number";
+      expect(getDistributedDdlTaskTimeoutSeconds()).toBe(180);
+      process.env[DDL_TIMEOUT_ENV_KEY] = "   ";
+      expect(getDistributedDdlTaskTimeoutSeconds()).toBe(180);
+    });
+
     test("distributed engine: model key, with global override winning", () => {
       delete process.env[SHARDING_ENV_KEY];
       // model sharding key is used

package/Tests/Server/Utils/AnalyticsDatabase/StatementGenerator.test.ts

@@ -24,6 +24,7 @@ import IncludesNone from "../../../../Types/BaseDatabase/IncludesNone";
 import MultiSearch from "../../../../Types/BaseDatabase/MultiSearch";
 import Search from "../../../../Types/BaseDatabase/Search";
 import StartsWith from "../../../../Types/BaseDatabase/StartsWith";
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
 import BadDataException from "../../../../Types/Exception/BadDataException";
 
 function expectStatement(actual: Statement, expected: Statement): void {
@@ -1431,6 +1432,125 @@ describe("StatementGenerator", () => {
     });
   });
 
+  /*
+   * The separator here is load-bearing for pagination. `toSortStatement`
+   * used to append sort keys with no delimiter, so anything past the first
+   * key produced a malformed term (`col_1 DESCcol_2 ASC`) — which meant a
+   * multi-key ORDER BY was unusable and, in practice, every paginated read
+   * ran on a single, non-unique sort column. See the pagination-stability
+   * suite in Tests/Server/Services/AnalyticsDatabasePaginationStability.
+   */
+  describe("toSortStatement", () => {
+    /*
+     * TestModel declares its columns through `tableColumns`, not as TS
+     * properties, so a `Sort<TestModel>` literal would fail excess-property
+     * checking on every key. Funnelling through one helper keeps the cast
+     * in a single place instead of on each call.
+     */
+    const sortStatementFor: (sort: Record<string, SortOrder>) => Statement = (
+      sort: Record<string, SortOrder>,
+    ): Statement => {
+      return generator.toSortStatement(sort as any);
+    };
+
+    test("renders a single ascending key with no trailing separator", () => {
+      const statement: Statement = sortStatementFor({
+        column_1: SortOrder.Ascending,
+      });
+
+      expect(statement.query).toBe("{p0:Identifier} ASC");
+      expect(statement.query_params).toStrictEqual({ p0: "column_1" });
+    });
+
+    test("renders a single descending key", () => {
+      const statement: Statement = sortStatementFor({
+        column_1: SortOrder.Descending,
+      });
+
+      expect(statement.query).toBe("{p0:Identifier} DESC");
+      expect(statement.query_params).toStrictEqual({ p0: "column_1" });
+    });
+
+    test("comma separates two keys", () => {
+      const statement: Statement = sortStatementFor({
+        column_1: SortOrder.Descending,
+        column_2: SortOrder.Ascending,
+      });
+
+      expect(statement.query).toBe("{p0:Identifier} DESC, {p1:Identifier} ASC");
+      expect(statement.query_params).toStrictEqual({
+        p0: "column_1",
+        p1: "column_2",
+      });
+    });
+
+    test("comma separates three keys and preserves declaration order", () => {
+      const statement: Statement = sortStatementFor({
+        column_2: SortOrder.Ascending,
+        column_1: SortOrder.Descending,
+        column_ObjectID: SortOrder.Ascending,
+      });
+
+      expect(statement.query).toBe(
+        "{p0:Identifier} ASC, {p1:Identifier} DESC, {p2:Identifier} ASC",
+      );
+      expect(statement.query_params).toStrictEqual({
+        p0: "column_2",
+        p1: "column_1",
+        p2: "column_ObjectID",
+      });
+    });
+
+    /*
+     * Regression guard for the exact malformed shape. Asserting on the
+     * rendered text rather than the param map is deliberate: the old bug
+     * produced correct PARAMS and broken SQL, so a params-only assertion
+     * would have passed straight through it.
+     */
+    test("never concatenates a direction into the next identifier", () => {
+      const statement: Statement = sortStatementFor({
+        column_1: SortOrder.Descending,
+        column_2: SortOrder.Ascending,
+      });
+
+      expect(statement.query).not.toContain("DESC{");
+      expect(statement.query).not.toContain("ASC{");
+      expect(statement.query).toContain(", ");
+    });
+
+    test("separator count is always one fewer than the key count", () => {
+      const statement: Statement = sortStatementFor({
+        column_ObjectID: SortOrder.Ascending,
+        column_1: SortOrder.Ascending,
+        column_2: SortOrder.Ascending,
+      });
+
+      expect(statement.query.split(", ")).toHaveLength(3);
+    });
+
+    test("an empty sort produces an empty statement", () => {
+      const statement: Statement = sortStatementFor({});
+
+      expect(statement.query).toBe("");
+      expect(statement.query_params).toStrictEqual({});
+    });
+
+    test("still rejects an unknown column", () => {
+      expect(() => {
+        return sortStatementFor({ not_a_column: SortOrder.Ascending });
+      }).toThrow(BadDataException);
+    });
+
+    test("rejects an unknown column that appears after a valid one", () => {
+      expect(() => {
+        return sortStatementFor({
+          column_1: SortOrder.Ascending,
+          not_a_column: SortOrder.Descending,
+        });
+      }).toThrow(BadDataException);
+    });
+  });
+
   describe("toColumnsCreateStatement", () => {
     test("should return the columns of a CREATE TABLE statement", () => {
       const statement: Statement = generator.toColumnsCreateStatement([