@oneuptime/common 11.5.4 → 11.5.6

package/Tests/Server/Services/TelemetryExceptionCodeFixRun.test.ts

@@ -0,0 +1,322 @@
+import TelemetryExceptionService from "../../../Server/Services/TelemetryExceptionService";
+import FixRunBudget from "../../../Server/Utils/AI/CodeFix/FixRunBudget";
+import LlmProviderService from "../../../Server/Services/LlmProviderService";
+import ServiceService from "../../../Server/Services/ServiceService";
+import CodeRepositoryService from "../../../Server/Services/CodeRepositoryService";
+import AIAgentService from "../../../Server/Services/AIAgentService";
+import AIRunService from "../../../Server/Services/AIRunService";
+import { RepoResolution } from "../../../Server/Utils/CodeRepository/StackTraceRepoResolver";
+import FindOneBy from "../../../Server/Types/Database/FindOneBy";
+import TelemetryException from "../../../Models/DatabaseModels/TelemetryException";
+import TelemetryService from "../../../Models/DatabaseModels/Service";
+import LlmProvider from "../../../Models/DatabaseModels/LlmProvider";
+import AIRun from "../../../Models/DatabaseModels/AIRun";
+import AIAgent, {
+  AIAgentConnectionStatus,
+} from "../../../Models/DatabaseModels/AIAgent";
+import AIRunType from "../../../Types/AI/AIRunType";
+import AIRunStatus from "../../../Types/AI/AIRunStatus";
+import CodeFixTaskType from "../../../Types/AI/CodeFixTaskType";
+import BadDataException from "../../../Types/Exception/BadDataException";
+import ObjectID from "../../../Types/ObjectID";
+import { describe, expect, test, afterEach, beforeEach } from "@jest/globals";
+
+/*
+ * "Fix with AI Agent" on the AIRun substrate: creation records a Queued
+ * CodeFix run carrying triggeredByTelemetryExceptionId (the link table is
+ * gone) and the task recipe (codeFixTaskType), and the duplicate guard
+ * blocks a second run of the SAME recipe while one is still in a
+ * non-terminal status — per (exception, taskType), so a live FixException
+ * run does not block queuing a WriteRegressionTest run and vice versa.
+ * Recipes that are declared but not yet user-triggerable are rejected
+ * before any run is created.
+ */
+
+const projectId: ObjectID = ObjectID.generate();
+const exceptionId: ObjectID = ObjectID.generate();
+const serviceId: ObjectID = ObjectID.generate();
+
+function fakeException(): TelemetryException {
+  return {
+    id: exceptionId,
+    projectId: projectId,
+    primaryEntityId: serviceId,
+    stackTrace: "at charge (/app/src/billing/charge.ts:12:5)",
+  } as unknown as TelemetryException;
+}
+
+function fakeResolution(): RepoResolution {
+  return {
+    codeRepositoryId: ObjectID.generate().toString(),
+    organizationName: "acme",
+    repositoryName: "checkout",
+    servicePathInRepository: null,
+    method: "stack-trace",
+    evidence: "Matched src/billing/charge.ts in acme/checkout",
+  };
+}
+
+// All three readiness checks pass; run creation is gated only by the guard.
+function mockReadinessOk(): void {
+  jest
+    .spyOn(TelemetryExceptionService, "findOneById")
+    .mockResolvedValue(fakeException());
+  jest
+    .spyOn(LlmProviderService, "getLlmProviderForMeteredAgentPath")
+    .mockResolvedValue({
+      id: ObjectID.generate(),
+      name: "BYO",
+    } as unknown as LlmProvider);
+  jest.spyOn(ServiceService, "findOneById").mockResolvedValue({
+    id: serviceId,
+    name: "checkout",
+  } as unknown as TelemetryService);
+  jest
+    .spyOn(CodeRepositoryService, "resolveRepositoryForException")
+    .mockResolvedValue(fakeResolution());
+  jest.spyOn(AIAgentService, "getAIAgentForProject").mockResolvedValue({
+    id: ObjectID.generate(),
+    name: "agent",
+    connectionStatus: AIAgentConnectionStatus.Connected,
+  } as unknown as AIAgent);
+}
+
+describe("TelemetryExceptionService.createCodeFixRunForException", () => {
+  beforeEach(() => {
+    // The daily fix-run budget has its own suite (FixRunBudget.test.ts).
+    jest.spyOn(FixRunBudget, "assertWithinBudget").mockResolvedValue();
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("records a Queued CodeFix AIRun carrying the triggering exception", async () => {
+    mockReadinessOk();
+
+    // Duplicate guard finds nothing.
+    jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(null);
+    const createdId: ObjectID = ObjectID.generate();
+    const create: jest.SpyInstance = jest
+      .spyOn(AIRunService, "create")
+      .mockResolvedValue({ id: createdId } as unknown as AIRun);
+
+    const run: AIRun =
+      await TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true },
+      });
+
+    expect(run.id).toBe(createdId);
+    expect(create).toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: expect.objectContaining({
+          projectId: projectId,
+          runType: AIRunType.CodeFix,
+          status: AIRunStatus.Queued,
+          triggeredByTelemetryExceptionId: exceptionId,
+          // New rows are stamped explicitly; only legacy rows are null.
+          codeFixTaskType: CodeFixTaskType.FixException,
+        }),
+        props: expect.objectContaining({ isRoot: true }),
+      }),
+    );
+  });
+
+  test("over the daily fix-run budget: rejects BEFORE the readiness probes, nothing created", async () => {
+    mockReadinessOk();
+
+    jest
+      .spyOn(FixRunBudget, "assertWithinBudget")
+      .mockRejectedValue(
+        new BadDataException(
+          "The project's daily AI fix task limit has been reached",
+        ),
+      );
+    const readinessProbe: jest.SpyInstance = jest.spyOn(
+      LlmProviderService,
+      "getLlmProviderForMeteredAgentPath",
+    );
+    const create: jest.SpyInstance = jest.spyOn(AIRunService, "create");
+
+    await expect(
+      TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true },
+      }),
+    ).rejects.toThrow(/daily AI fix task limit/);
+
+    // Budget is the cheaper gate — the readiness checks never ran.
+    expect(readinessProbe).not.toHaveBeenCalled();
+    expect(create).not.toHaveBeenCalled();
+  });
+
+  test("duplicate guard: an active (non-terminal) run for the exception blocks creation", async () => {
+    mockReadinessOk();
+
+    const findActiveRun: jest.SpyInstance = jest
+      .spyOn(AIRunService, "findOneBy")
+      .mockResolvedValue({ id: ObjectID.generate() } as unknown as AIRun);
+    const create: jest.SpyInstance = jest.spyOn(AIRunService, "create");
+
+    await expect(
+      TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true },
+      }),
+    ).rejects.toThrow(/already in progress/);
+
+    expect(create).not.toHaveBeenCalled();
+    // The guard queries by run type + triggering exception, not a link table.
+    expect(findActiveRun).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.objectContaining({
+          runType: AIRunType.CodeFix,
+          triggeredByTelemetryExceptionId: exceptionId,
+        }),
+      }),
+    );
+  });
+
+  test("duplicate guard is per task type: an active FixException run blocks another FixException but not a WriteRegressionTest", async () => {
+    mockReadinessOk();
+
+    /*
+     * The guard queries codeFixTaskType per recipe: WriteRegressionTest as
+     * the plain enum value, FixException as a null-inclusive operator
+     * (legacy rows predate the column). Simulate a live FixException run
+     * and nothing else.
+     */
+    jest
+      .spyOn(AIRunService, "findOneBy")
+      .mockImplementation(
+        async (findOneBy: FindOneBy<AIRun>): Promise<AIRun | null> => {
+          if (
+            findOneBy.query.codeFixTaskType ===
+            CodeFixTaskType.WriteRegressionTest
+          ) {
+            return null;
+          }
+
+          return { id: ObjectID.generate() } as unknown as AIRun;
+        },
+      );
+
+    const create: jest.SpyInstance = jest
+      .spyOn(AIRunService, "create")
+      .mockResolvedValue({ id: ObjectID.generate() } as unknown as AIRun);
+
+    // The same recipe is blocked while its run is live...
+    await expect(
+      TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true },
+        taskType: CodeFixTaskType.FixException,
+      }),
+    ).rejects.toThrow(/already in progress/);
+    expect(create).not.toHaveBeenCalled();
+
+    // ...but a different recipe for the same exception is not.
+    await expect(
+      TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true },
+        taskType: CodeFixTaskType.WriteRegressionTest,
+      }),
+    ).resolves.toBeDefined();
+    expect(create).toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: expect.objectContaining({
+          codeFixTaskType: CodeFixTaskType.WriteRegressionTest,
+          triggeredByTelemetryExceptionId: exceptionId,
+        }),
+      }),
+    );
+  });
+
+  test("rejects task recipes that are not user-triggerable yet, before touching anything", async () => {
+    const findOneBy: jest.SpyInstance = jest.spyOn(AIRunService, "findOneBy");
+    const create: jest.SpyInstance = jest.spyOn(AIRunService, "create");
+
+    await expect(
+      TelemetryExceptionService.createCodeFixRunForException({
+        telemetryExceptionId: exceptionId,
+        props: { isRoot: true },
+        taskType: CodeFixTaskType.ImproveInstrumentation,
+      }),
+    ).rejects.toThrow(/not user-triggerable yet/);
+
+    expect(findOneBy).not.toHaveBeenCalled();
+    expect(create).not.toHaveBeenCalled();
+  });
+
+  test("attributes the run to the requesting user when one is present", async () => {
+    mockReadinessOk();
+
+    jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(null);
+    const create: jest.SpyInstance = jest
+      .spyOn(AIRunService, "create")
+      .mockResolvedValue({ id: ObjectID.generate() } as unknown as AIRun);
+
+    const userId: ObjectID = ObjectID.generate();
+
+    await TelemetryExceptionService.createCodeFixRunForException({
+      telemetryExceptionId: exceptionId,
+      props: { isRoot: true, userId },
+    });
+
+    expect(create).toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: expect.objectContaining({ userId }),
+      }),
+    );
+  });
+});
+
+/*
+ * The worker dispatches its task handler on the claimed run's taskType
+ * (POST /ai-agent-task/get-pending-task), so the claim must normalize the
+ * legacy null column to FixException before the run leaves the service.
+ */
+describe("AIRunService.claimNextQueuedCodeFixRun task recipe", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  function fakeQueuedRun(codeFixTaskType?: CodeFixTaskType): AIRun {
+    return {
+      id: ObjectID.generate(),
+      projectId: projectId,
+      triggeredByTelemetryExceptionId: exceptionId,
+      attemptCount: 0,
+      codeFixTaskType: codeFixTaskType,
+    } as unknown as AIRun;
+  }
+
+  test("a claimed legacy run (null codeFixTaskType) carries FixException", async () => {
+    jest
+      .spyOn(AIRunService, "findOneBy")
+      .mockResolvedValue(fakeQueuedRun(undefined));
+    jest.spyOn(AIRunService, "attemptStatusTransition").mockResolvedValue(1);
+
+    const claimed: AIRun | null = await AIRunService.claimNextQueuedCodeFixRun({
+      aiAgentId: ObjectID.generate(),
+    });
+
+    expect(claimed).not.toBeNull();
+    expect(claimed?.codeFixTaskType).toBe(CodeFixTaskType.FixException);
+  });
+
+  test("a claimed run keeps its explicit task recipe", async () => {
+    jest
+      .spyOn(AIRunService, "findOneBy")
+      .mockResolvedValue(fakeQueuedRun(CodeFixTaskType.WriteRegressionTest));
+    jest.spyOn(AIRunService, "attemptStatusTransition").mockResolvedValue(1);
+
+    const claimed: AIRun | null = await AIRunService.claimNextQueuedCodeFixRun({
+      aiAgentId: ObjectID.generate(),
+    });
+
+    expect(claimed?.codeFixTaskType).toBe(CodeFixTaskType.WriteRegressionTest);
+  });
+});

package/Tests/Server/Services/TraceServiceLatencyProfile.test.ts

@@ -0,0 +1,278 @@
+import TraceAggregationService, {
+  ServiceLatencyProfile,
+} from "../../../Server/Services/TraceAggregationService";
+import SpanService from "../../../Server/Services/SpanService";
+import { Results } from "../../../Server/Services/AnalyticsDatabaseService";
+import { Statement } from "../../../Server/Utils/AnalyticsDatabase/Statement";
+import { JSONObject } from "../../../Types/JSON";
+import ObjectID from "../../../Types/ObjectID";
+import { afterEach, describe, expect, test } from "@jest/globals";
+
+/*
+ * getServiceLatencyProfile is the ONLY query the unattended insight scanner
+ * runs against the full Span table every 15 minutes for every opted-in
+ * project. It is affordable exactly and only while ClickHouse can serve it
+ * from the proj_agg_by_service aggregate projection
+ * (projectId, primaryEntityId, toStartOfMinute(startTime) -> count() /
+ * avg(durationUnixNano) / quantile(0.99)(durationUnixNano)).
+ *
+ * The projection match is a BYTE-MATCH on the aggregate expressions and the
+ * key expression: fold a `/ 1000000` into the quantile, compare raw
+ * startTime instead of toStartOfMinute(startTime), add the retentionDate
+ * filter or group by anything outside the projection's keys, and ClickHouse
+ * silently reverts to a 25-hour raw-span scan — which then trips the
+ * execution cap and returns partial aggregates. Nothing in the query plan is
+ * observable from TypeScript, so these SQL-shape assertions are the guard.
+ */
+
+const projectId: ObjectID = ObjectID.generate();
+const startTime: Date = new Date("2026-07-14T11:00:00.000Z");
+const endTime: Date = new Date("2026-07-14T12:00:00.000Z");
+
+const stubQuery: (response: {
+  rows?: Array<JSONObject>;
+  elapsedSeconds?: number;
+  throwOnParse?: boolean;
+}) => jest.SpyInstance = (response: {
+  rows?: Array<JSONObject>;
+  elapsedSeconds?: number;
+  throwOnParse?: boolean;
+}): jest.SpyInstance => {
+  const fakeResult: Results = {
+    json: (): Promise<unknown> => {
+      if (response.throwOnParse) {
+        return Promise.reject(new Error("Unexpected end of JSON input"));
+      }
+      return Promise.resolve({
+        data: response.rows || [],
+        statistics: {
+          elapsed: response.elapsedSeconds ?? 0.004,
+          rows_read: 10,
+          bytes_read: 100,
+        },
+      });
+    },
+  } as unknown as Results;
+
+  return jest
+    .spyOn(SpanService, "executeQuery")
+    .mockResolvedValue(fakeResult as never);
+};
+
+const capturedQuery: (spy: jest.SpyInstance) => string = (
+  spy: jest.SpyInstance,
+): string => {
+  const statement: Statement = spy.mock.calls[0]![0] as Statement;
+  // The query getter dedents — normalize whitespace before matching SQL.
+  return statement.query.replace(/\s+/g, " ");
+};
+
+describe("TraceAggregationService.getServiceLatencyProfile (projection-served SQL shape)", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("selects ONLY the aggregates proj_agg_by_service stores, byte-for-byte", async () => {
+    const spy: jest.SpyInstance = stubQuery({ rows: [] });
+
+    await TraceAggregationService.getServiceLatencyProfile({
+      projectId,
+      startTime,
+      endTime,
+      limit: 25,
+    });
+
+    const query: string = capturedQuery(spy);
+
+    expect(query).toContain("count() AS cnt");
+    expect(query).toContain("quantile(0.99)(durationUnixNano) AS p99_ns");
+
+    /*
+     * The ns->ms division MUST stay in TypeScript: `quantile(0.99)(x) /
+     * 1000000` is not the stored expression, so it disqualifies the
+     * projection.
+     */
+    expect(query).not.toContain("1000000");
+
+    // Every other stat in getAnalyticsTable's set has no stored state.
+    expect(query).not.toContain("countIf");
+    expect(query).not.toContain("quantile(0.95)");
+    expect(query).not.toContain("quantile(0.9)(");
+    expect(query).not.toContain("quantile(0.5)");
+    expect(query).not.toContain("min(durationUnixNano)");
+    expect(query).not.toContain("max(durationUnixNano)");
+  });
+
+  test("filters on the projection key expression, never raw startTime, and never retentionDate", async () => {
+    const spy: jest.SpyInstance = stubQuery({ rows: [] });
+
+    await TraceAggregationService.getServiceLatencyProfile({
+      projectId,
+      startTime,
+      endTime,
+      limit: 25,
+    });
+
+    const query: string = capturedQuery(spy);
+
+    expect(query).toContain("toStartOfMinute(startTime) >= toStartOfMinute(");
+    // Half-open: the boundary minute must not be counted in both windows.
+    expect(query).toContain("toStartOfMinute(startTime) < toStartOfMinute(");
+
+    // A bare `startTime <op>` comparison would reject the projection.
+    expect(query).not.toMatch(/startTime\s*(>=|<=|>|<)/);
+
+    // retentionDate is not stored by the aggregate projection.
+    expect(query).not.toContain("retentionDate");
+  });
+
+  test("groups by a subset of the projection keys and caps the row count", async () => {
+    const spy: jest.SpyInstance = stubQuery({ rows: [] });
+
+    await TraceAggregationService.getServiceLatencyProfile({
+      projectId,
+      startTime,
+      endTime,
+      limit: 25,
+    });
+
+    const query: string = capturedQuery(spy);
+
+    /*
+     * primaryEntityId only — dropping the `minute` key merges the stored
+     * per-minute aggregate states into a whole-window value. Any dimension
+     * outside (projectId, primaryEntityId, minute) forces a full scan.
+     */
+    expect(query).toContain("GROUP BY primaryEntityId ORDER BY cnt DESC LIMIT");
+    expect(query).not.toContain("statusCode");
+    expect(query).not.toContain("attributes[");
+
+    const statement: Statement = spy.mock.calls[0]![0] as Statement;
+    expect(Object.values(statement.query_params)).toContain(25);
+  });
+
+  test("caps execution at 45s and explicitly enables projections", async () => {
+    const spy: jest.SpyInstance = stubQuery({ rows: [] });
+
+    await TraceAggregationService.getServiceLatencyProfile({
+      projectId,
+      startTime,
+      endTime,
+      limit: 25,
+    });
+
+    const query: string = capturedQuery(spy);
+
+    expect(query).toContain("max_execution_time = 45");
+    expect(query).toContain("timeout_overflow_mode = 'break'");
+    expect(query).toContain("optimize_use_projections = 1");
+  });
+
+  test("converts ns to ms in TS and drops rows without an entity id", async () => {
+    stubQuery({
+      rows: [
+        { primaryEntityId: "svc-1", cnt: "500", p99_ns: "2300000000" },
+        { primaryEntityId: "", cnt: "9", p99_ns: "1000000" },
+      ],
+    });
+
+    const result: ServiceLatencyProfile =
+      await TraceAggregationService.getServiceLatencyProfile({
+        projectId,
+        startTime,
+        endTime,
+        limit: 25,
+      });
+
+    expect(result.rows).toEqual([
+      { primaryEntityId: "svc-1", count: 500, p99DurationMs: 2300 },
+    ]);
+    expect(result.isPartial).toBe(false);
+  });
+});
+
+/*
+ * With timeout_overflow_mode='break' a cut-off query returns a well-formed
+ * body holding PARTIAL aggregates — no exception, no marker. The insight
+ * scanner compares two windows, so partial data is not "less data", it is a
+ * wrong p99. `statistics.elapsed` at the cap is the only available signal
+ * that this happened, and the caller is contractually required to discard
+ * the tick when it is set.
+ */
+describe("TraceAggregationService.getServiceLatencyProfile (partial-result detection)", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("a query that burned its whole execution budget is reported as partial", async () => {
+    stubQuery({
+      rows: [{ primaryEntityId: "svc-1", cnt: "500", p99_ns: "2300000000" }],
+      elapsedSeconds: 45,
+    });
+
+    const result: ServiceLatencyProfile =
+      await TraceAggregationService.getServiceLatencyProfile({
+        projectId,
+        startTime,
+        endTime,
+        limit: 25,
+      });
+
+    expect(result.isPartial).toBe(true);
+  });
+
+  test("break fires slightly before the cap — 90% of the budget already counts as partial", async () => {
+    stubQuery({
+      rows: [{ primaryEntityId: "svc-1", cnt: "500", p99_ns: "2300000000" }],
+      elapsedSeconds: 40.5,
+    });
+
+    const result: ServiceLatencyProfile =
+      await TraceAggregationService.getServiceLatencyProfile({
+        projectId,
+        startTime,
+        endTime,
+        limit: 25,
+      });
+
+    expect(result.isPartial).toBe(true);
+  });
+
+  test("a fast projection-served query is complete", async () => {
+    stubQuery({
+      rows: [{ primaryEntityId: "svc-1", cnt: "500", p99_ns: "2300000000" }],
+      elapsedSeconds: 0.006,
+    });
+
+    const result: ServiceLatencyProfile =
+      await TraceAggregationService.getServiceLatencyProfile({
+        projectId,
+        startTime,
+        endTime,
+        limit: 25,
+      });
+
+    expect(result.isPartial).toBe(false);
+    expect(result.rows).toHaveLength(1);
+  });
+
+  /*
+   * 'break' can also truncate the JSON body outright. An unparseable body is
+   * partial — NOT an empty profile, which the caller would read as "this
+   * project has no traffic".
+   */
+  test("an unparseable (truncated) response is partial, not empty-and-trusted", async () => {
+    stubQuery({ throwOnParse: true });
+
+    const result: ServiceLatencyProfile =
+      await TraceAggregationService.getServiceLatencyProfile({
+        projectId,
+        startTime,
+        endTime,
+        limit: 25,
+      });
+
+    expect(result.rows).toEqual([]);
+    expect(result.isPartial).toBe(true);
+  });
+});