@oneuptime/common 11.5.4 → 11.5.6

package/Tests/Server/Utils/AI/InvestigationGrader.test.ts

@@ -0,0 +1,343 @@
+import InvestigationGrader from "../../../../Server/Utils/AI/SRE/InvestigationGrader";
+import AIRunService from "../../../../Server/Services/AIRunService";
+import IncidentService from "../../../../Server/Services/IncidentService";
+import IncidentFeedService from "../../../../Server/Services/IncidentFeedService";
+import AIService, {
+  AILogResponse,
+} from "../../../../Server/Services/AIService";
+import AIRun from "../../../../Models/DatabaseModels/AIRun";
+import Incident from "../../../../Models/DatabaseModels/Incident";
+import IncidentFeed from "../../../../Models/DatabaseModels/IncidentFeed";
+import AIRunAutoGrade from "../../../../Types/AI/AIRunAutoGrade";
+import ObjectID from "../../../../Types/ObjectID";
+import { describe, expect, test, afterEach } from "@jest/globals";
+
+/*
+ * On-resolve investigation grading (Phase 2 measurement layer): when an
+ * incident resolves with a human-recorded root cause and a completed
+ * investigation exists, ONE constrained LLM call grades the posted analysis
+ * MATCH / PARTIAL / MISMATCH. Under test: the pure gate
+ * (shouldGradeInvestigation), the defensive one-token parse
+ * (parseAutoGradeToken — including the MATCH-inside-MISMATCH trap), and the
+ * end-to-end skip/store/swallow behavior of gradeInvestigationOnResolve.
+ */
+
+const incidentId: ObjectID = ObjectID.generate();
+const projectId: ObjectID = ObjectID.generate();
+
+function fakeRun(autoGrade?: AIRunAutoGrade): AIRun {
+  return { id: ObjectID.generate(), autoGrade } as unknown as AIRun;
+}
+
+function fakeIncident(rootCause?: string | undefined): Incident {
+  return { id: incidentId, rootCause } as unknown as Incident;
+}
+
+function fakeFeedItem(markdown: string): IncidentFeed {
+  return { feedInfoInMarkdown: markdown } as unknown as IncidentFeed;
+}
+
+function fakeLlmResponse(content: string): AILogResponse {
+  return { content } as unknown as AILogResponse;
+}
+
+describe("InvestigationGrader.shouldGradeInvestigation", () => {
+  test("no run → false", () => {
+    expect(
+      InvestigationGrader.shouldGradeInvestigation({
+        run: null,
+        rootCause: "db down",
+      }),
+    ).toBe(false);
+  });
+
+  test("run without an id → false", () => {
+    expect(
+      InvestigationGrader.shouldGradeInvestigation({
+        run: {},
+        rootCause: "db down",
+      }),
+    ).toBe(false);
+  });
+
+  test("already graded → false (dedupe: a run is graded once)", () => {
+    expect(
+      InvestigationGrader.shouldGradeInvestigation({
+        run: { id: ObjectID.generate(), autoGrade: AIRunAutoGrade.Match },
+        rootCause: "db down",
+      }),
+    ).toBe(false);
+  });
+
+  test("missing / empty / whitespace root cause → false", () => {
+    const run: { id: ObjectID } = { id: ObjectID.generate() };
+
+    expect(
+      InvestigationGrader.shouldGradeInvestigation({ run, rootCause: null }),
+    ).toBe(false);
+    expect(
+      InvestigationGrader.shouldGradeInvestigation({
+        run,
+        rootCause: undefined,
+      }),
+    ).toBe(false);
+    expect(
+      InvestigationGrader.shouldGradeInvestigation({ run, rootCause: "" }),
+    ).toBe(false);
+    expect(
+      InvestigationGrader.shouldGradeInvestigation({ run, rootCause: "   " }),
+    ).toBe(false);
+  });
+
+  test("completed ungraded run + non-empty root cause → true", () => {
+    expect(
+      InvestigationGrader.shouldGradeInvestigation({
+        run: { id: ObjectID.generate() },
+        rootCause: "Connection pool exhausted",
+      }),
+    ).toBe(true);
+  });
+});
+
+describe("InvestigationGrader.parseAutoGradeToken", () => {
+  test("exact single tokens parse (case-insensitively)", () => {
+    expect(InvestigationGrader.parseAutoGradeToken("MATCH")).toBe(
+      AIRunAutoGrade.Match,
+    );
+    expect(InvestigationGrader.parseAutoGradeToken(" partial\n")).toBe(
+      AIRunAutoGrade.Partial,
+    );
+    expect(InvestigationGrader.parseAutoGradeToken("mismatch")).toBe(
+      AIRunAutoGrade.Mismatch,
+    );
+  });
+
+  test("MISMATCH does not double-count as MATCH (word boundaries)", () => {
+    expect(InvestigationGrader.parseAutoGradeToken("MISMATCH")).toBe(
+      AIRunAutoGrade.Mismatch,
+    );
+  });
+
+  test("a token embedded in editorializing prose still parses", () => {
+    expect(
+      InvestigationGrader.parseAutoGradeToken("The verdict is: MATCH."),
+    ).toBe(AIRunAutoGrade.Match);
+    expect(
+      InvestigationGrader.parseAutoGradeToken(
+        "I would grade this as PARTIAL, since only one of the two causes was found.",
+      ),
+    ).toBe(AIRunAutoGrade.Partial);
+  });
+
+  test("a token embedded inside a larger word does not parse", () => {
+    expect(InvestigationGrader.parseAutoGradeToken("rematch")).toBeNull();
+    expect(InvestigationGrader.parseAutoGradeToken("partially")).toBeNull();
+  });
+
+  test("ambiguous (several distinct tokens) → null", () => {
+    expect(
+      InvestigationGrader.parseAutoGradeToken(
+        "Either MATCH or MISMATCH, hard to say.",
+      ),
+    ).toBeNull();
+  });
+
+  test("no token / empty / null → null", () => {
+    expect(
+      InvestigationGrader.parseAutoGradeToken("I cannot grade this."),
+    ).toBeNull();
+    expect(InvestigationGrader.parseAutoGradeToken("")).toBeNull();
+    expect(InvestigationGrader.parseAutoGradeToken(null)).toBeNull();
+    expect(InvestigationGrader.parseAutoGradeToken(undefined)).toBeNull();
+  });
+});
+
+describe("InvestigationGrader.gradeInvestigationOnResolve", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("no completed investigation → skips before loading the incident, no LLM call", async () => {
+    jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(null);
+    const findOneById: jest.SpyInstance = jest.spyOn(
+      IncidentService,
+      "findOneById",
+    );
+    const executeWithLogging: jest.SpyInstance = jest.spyOn(
+      AIService,
+      "executeWithLogging",
+    );
+
+    await InvestigationGrader.gradeInvestigationOnResolve({
+      incidentId,
+      projectId,
+    });
+
+    expect(findOneById).not.toHaveBeenCalled();
+    expect(executeWithLogging).not.toHaveBeenCalled();
+  });
+
+  test("already graded run → no LLM call, nothing written", async () => {
+    jest
+      .spyOn(AIRunService, "findOneBy")
+      .mockResolvedValue(fakeRun(AIRunAutoGrade.Partial));
+    jest
+      .spyOn(IncidentService, "findOneById")
+      .mockResolvedValue(fakeIncident("db down"));
+    const executeWithLogging: jest.SpyInstance = jest.spyOn(
+      AIService,
+      "executeWithLogging",
+    );
+    const updateOneById: jest.SpyInstance = jest.spyOn(
+      AIRunService,
+      "updateOneById",
+    );
+
+    await InvestigationGrader.gradeInvestigationOnResolve({
+      incidentId,
+      projectId,
+    });
+
+    expect(executeWithLogging).not.toHaveBeenCalled();
+    expect(updateOneById).not.toHaveBeenCalled();
+  });
+
+  test("incident without a root cause → no LLM call", async () => {
+    jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(fakeRun());
+    jest
+      .spyOn(IncidentService, "findOneById")
+      .mockResolvedValue(fakeIncident(undefined));
+    const executeWithLogging: jest.SpyInstance = jest.spyOn(
+      AIService,
+      "executeWithLogging",
+    );
+
+    await InvestigationGrader.gradeInvestigationOnResolve({
+      incidentId,
+      projectId,
+    });
+
+    expect(executeWithLogging).not.toHaveBeenCalled();
+  });
+
+  test("no posted analysis (no RootCause feed item) → no LLM call", async () => {
+    jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(fakeRun());
+    jest
+      .spyOn(IncidentService, "findOneById")
+      .mockResolvedValue(fakeIncident("db down"));
+    jest.spyOn(IncidentFeedService, "findOneBy").mockResolvedValue(null);
+    const executeWithLogging: jest.SpyInstance = jest.spyOn(
+      AIService,
+      "executeWithLogging",
+    );
+
+    await InvestigationGrader.gradeInvestigationOnResolve({
+      incidentId,
+      projectId,
+    });
+
+    expect(executeWithLogging).not.toHaveBeenCalled();
+  });
+
+  test("happy path: one budgeted, preview-less LLM call; grade + timestamp stored on the run", async () => {
+    const run: AIRun = fakeRun();
+    jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(run);
+    jest
+      .spyOn(IncidentService, "findOneById")
+      .mockResolvedValue(fakeIncident("The connection pool was exhausted."));
+    jest
+      .spyOn(IncidentFeedService, "findOneBy")
+      .mockResolvedValue(
+        fakeFeedItem("## Analysis\nThe db connection pool ran dry."),
+      );
+    const executeWithLogging: jest.SpyInstance = jest
+      .spyOn(AIService, "executeWithLogging")
+      .mockResolvedValue(fakeLlmResponse("MATCH"));
+    const updateOneById: jest.SpyInstance = jest
+      .spyOn(AIRunService, "updateOneById")
+      .mockResolvedValue(undefined as never);
+
+    await InvestigationGrader.gradeInvestigationOnResolve({
+      incidentId,
+      projectId,
+    });
+
+    expect(executeWithLogging).toHaveBeenCalledTimes(1);
+    expect(executeWithLogging).toHaveBeenCalledWith(
+      expect.objectContaining({
+        projectId,
+        incidentId,
+        aiRunId: run.id,
+        /*
+         * Budget coverage: must be an AUTONOMOUS_AI_FEATURES member. Pinned as
+         * a literal on purpose — this string is persisted into LlmLog.feature
+         * and the G4 budget sums by matching it, so a rename must break here.
+         */
+        feature: "AI Investigation Grading",
+        // Grades live on the run — no prompt previews in LlmLog.
+        storeContentPreviews: false,
+      }),
+    );
+
+    expect(updateOneById).toHaveBeenCalledWith(
+      expect.objectContaining({
+        id: run.id,
+        data: expect.objectContaining({
+          autoGrade: AIRunAutoGrade.Match,
+          autoGradeAt: expect.any(Date),
+        }),
+        props: expect.objectContaining({ isRoot: true }),
+      }),
+    );
+  });
+
+  test("unparseable grade response → warn, no grade stored", async () => {
+    jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(fakeRun());
+    jest
+      .spyOn(IncidentService, "findOneById")
+      .mockResolvedValue(fakeIncident("db down"));
+    jest
+      .spyOn(IncidentFeedService, "findOneBy")
+      .mockResolvedValue(fakeFeedItem("Analysis text."));
+    jest
+      .spyOn(AIService, "executeWithLogging")
+      .mockResolvedValue(fakeLlmResponse("Either MATCH or MISMATCH."));
+    const updateOneById: jest.SpyInstance = jest.spyOn(
+      AIRunService,
+      "updateOneById",
+    );
+
+    await InvestigationGrader.gradeInvestigationOnResolve({
+      incidentId,
+      projectId,
+    });
+
+    expect(updateOneById).not.toHaveBeenCalled();
+  });
+
+  test("an LLM failure is swallowed — grading can never throw into the resolve path", async () => {
+    jest.spyOn(AIRunService, "findOneBy").mockResolvedValue(fakeRun());
+    jest
+      .spyOn(IncidentService, "findOneById")
+      .mockResolvedValue(fakeIncident("db down"));
+    jest
+      .spyOn(IncidentFeedService, "findOneBy")
+      .mockResolvedValue(fakeFeedItem("Analysis text."));
+    jest
+      .spyOn(AIService, "executeWithLogging")
+      .mockRejectedValue(new Error("provider down"));
+    const updateOneById: jest.SpyInstance = jest.spyOn(
+      AIRunService,
+      "updateOneById",
+    );
+
+    await expect(
+      InvestigationGrader.gradeInvestigationOnResolve({
+        incidentId,
+        projectId,
+      }),
+    ).resolves.toBeUndefined();
+
+    expect(updateOneById).not.toHaveBeenCalled();
+  });
+});

package/Tests/Server/Utils/AI/LLMServiceRequestPolicy.test.ts

@@ -0,0 +1,307 @@
+import API from "../../../../Utils/API";
+import logger from "../../../../Server/Utils/Logger";
+import HTTPErrorResponse from "../../../../Types/API/HTTPErrorResponse";
+import HTTPResponse from "../../../../Types/API/HTTPResponse";
+import { JSONObject } from "../../../../Types/JSON";
+import LlmType from "../../../../Types/LLM/LlmType";
+import LLMService, {
+  LLMProviderConfig,
+} from "../../../../Server/Utils/LLM/LLMService";
+import { afterEach, describe, expect, test } from "@jest/globals";
+
+type PostSpy = ReturnType<typeof jest.spyOn>;
+
+interface ProviderCase {
+  name: string;
+  config: LLMProviderConfig;
+  response: JSONObject;
+  defaultTimeoutInMs: number;
+}
+
+const providerCases: Array<ProviderCase> = [
+  {
+    name: "OpenAI-compatible",
+    config: {
+      llmType: LlmType.OpenAI,
+      apiKey: "test-key",
+      modelName: "gpt-test",
+    },
+    response: {
+      choices: [{ message: { content: "ok" } }],
+      usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+    },
+    defaultTimeoutInMs: 120_000,
+  },
+  {
+    name: "Azure OpenAI",
+    config: {
+      llmType: LlmType.AzureOpenAI,
+      apiKey: "test-key",
+      baseUrl:
+        "https://example.openai.azure.com/openai/deployments/test-deployment",
+      modelName: "gpt-test",
+    },
+    response: {
+      choices: [{ message: { content: "ok" } }],
+      usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 },
+    },
+    defaultTimeoutInMs: 120_000,
+  },
+  {
+    name: "Anthropic",
+    config: {
+      llmType: LlmType.Anthropic,
+      apiKey: "test-key",
+      modelName: "claude-test",
+    },
+    response: {
+      content: [{ type: "text", text: "ok" }],
+      stop_reason: "end_turn",
+      usage: { input_tokens: 1, output_tokens: 1 },
+    },
+    defaultTimeoutInMs: 120_000,
+  },
+  {
+    name: "Ollama",
+    config: {
+      llmType: LlmType.Ollama,
+      baseUrl: "http://localhost:11434",
+      modelName: "llama-test",
+    },
+    response: {
+      message: { role: "assistant", content: "ok" },
+      prompt_eval_count: 1,
+      eval_count: 1,
+    },
+    defaultTimeoutInMs: 300_000,
+  },
+];
+
+function mockPost(response: JSONObject): PostSpy {
+  return jest.spyOn(API, "post").mockResolvedValue({
+    jsonData: response,
+  } as unknown as HTTPResponse<JSONObject>) as PostSpy;
+}
+
+afterEach(() => {
+  jest.restoreAllMocks();
+});
+
+describe("LLMService request timeout and retry policy", () => {
+  test.each(providerCases)(
+    "$name forwards an explicit per-request policy",
+    async ({ config, response }: ProviderCase) => {
+      const postSpy: PostSpy = mockPost(response);
+
+      await LLMService.getCompletion({
+        llmProviderConfig: config,
+        messages: [{ role: "user", content: "hello" }],
+        requestRetries: 0,
+        requestTimeoutInMs: 12_345,
+      });
+
+      expect(postSpy).toHaveBeenCalledTimes(1);
+      expect(postSpy.mock.calls[0]![0]).toEqual(
+        expect.objectContaining({
+          options: {
+            retries: 0,
+            exponentialBackoff: true,
+            timeout: 12_345,
+          },
+        }),
+      );
+    },
+  );
+
+  test.each(providerCases)(
+    "$name retains its existing default policy when no override is supplied",
+    async ({ config, response, defaultTimeoutInMs }: ProviderCase) => {
+      const postSpy: PostSpy = mockPost(response);
+
+      await LLMService.getCompletion({
+        llmProviderConfig: config,
+        messages: [{ role: "user", content: "hello" }],
+      });
+
+      expect(postSpy.mock.calls[0]![0]).toEqual(
+        expect.objectContaining({
+          options: {
+            retries: 2,
+            exponentialBackoff: true,
+            timeout: defaultTimeoutInMs,
+          },
+        }),
+      );
+    },
+  );
+
+  test("protected OpenAI-compatible requests cannot have workflow-owned fields overridden", async () => {
+    const postSpy: PostSpy = mockPost(providerCases[0]!.response);
+
+    await LLMService.getCompletion({
+      llmProviderConfig: providerCases[0]!.config,
+      messages: [{ role: "user", content: "private workflow prompt" }],
+      temperature: 0.2,
+      maxTokens: 256,
+      protectRequestParameters: true,
+      additionalParams: {
+        model: "overridden-model",
+        messages: [{ role: "user", content: "overridden prompt" }],
+        temperature: 1,
+        max_tokens: 99_999,
+        n: 50,
+        stream: true,
+        tools: [{ type: "function", function: { name: "unsafe_tool" } }],
+        tool_choice: "required",
+        best_of: 10,
+        web_search_options: { search_context_size: "high" },
+        data_sources: [{ type: "azure_search" }],
+        modalities: ["text", "audio"],
+        audio: { format: "wav", voice: "alloy" },
+        store: true,
+        metadata: { private: "provider-retained" },
+        future_agent_capability: { enabled: true },
+        top_p: 0.9,
+      },
+    });
+
+    const requestBody: JSONObject = (
+      postSpy.mock.calls[0]![0] as { data: JSONObject }
+    ).data;
+
+    expect(requestBody).toMatchObject({
+      model: "gpt-test",
+      messages: [{ role: "user", content: "private workflow prompt" }],
+      temperature: 0.2,
+      max_tokens: 256,
+      n: 1,
+      stream: false,
+      top_p: 0.9,
+    });
+    expect(requestBody["tools"]).toBeUndefined();
+    expect(requestBody["tool_choice"]).toBeUndefined();
+    expect(requestBody["best_of"]).toBeUndefined();
+    expect(requestBody["web_search_options"]).toBeUndefined();
+    expect(requestBody["data_sources"]).toBeUndefined();
+    expect(requestBody["modalities"]).toBeUndefined();
+    expect(requestBody["audio"]).toBeUndefined();
+    expect(requestBody["store"]).toBeUndefined();
+    expect(requestBody["metadata"]).toBeUndefined();
+    expect(requestBody["future_agent_capability"]).toBeUndefined();
+  });
+
+  test("protected requests retain allowlisted generation-only tuning", async () => {
+    const postSpy: PostSpy = mockPost(providerCases[0]!.response);
+
+    await LLMService.getCompletion({
+      llmProviderConfig: providerCases[0]!.config,
+      messages: [{ role: "user", content: "hello" }],
+      protectRequestParameters: true,
+      additionalParams: {
+        top_p: 0.8,
+        frequency_penalty: 0.1,
+        presence_penalty: 0.2,
+        seed: 42,
+        stop: ["END"],
+        response_format: { type: "json_object" },
+      },
+    });
+
+    const requestBody: JSONObject = (
+      postSpy.mock.calls[0]![0] as { data: JSONObject }
+    ).data;
+
+    expect(requestBody).toMatchObject({
+      top_p: 0.8,
+      frequency_penalty: 0.1,
+      presence_penalty: 0.2,
+      seed: 42,
+      stop: ["END"],
+      response_format: { type: "json_object" },
+    });
+  });
+
+  test("protected requests retain max_completion_tokens compatibility without losing the cap", async () => {
+    const postSpy: PostSpy = mockPost(providerCases[0]!.response);
+
+    await LLMService.getCompletion({
+      llmProviderConfig: providerCases[0]!.config,
+      messages: [{ role: "user", content: "hello" }],
+      maxTokens: 512,
+      protectRequestParameters: true,
+      additionalParams: { max_completion_tokens: 40_000 },
+    });
+
+    const requestBody: JSONObject = (
+      postSpy.mock.calls[0]![0] as { data: JSONObject }
+    ).data;
+    expect(requestBody["max_completion_tokens"]).toBe(512);
+    expect(requestBody["max_tokens"]).toBeUndefined();
+  });
+
+  test.each(providerCases)(
+    "$name excludes echoed private content from logs and captured exceptions when provider details are disabled",
+    async ({ config }: ProviderCase) => {
+      const privateProviderDetail: string =
+        "provider echoed private workflow prompt SECRET-123";
+      jest
+        .spyOn(API, "post")
+        .mockResolvedValue(
+          new HTTPErrorResponse(
+            400,
+            { error: { message: privateProviderDetail } },
+            {},
+          ),
+        );
+      const errorLog: jest.SpyInstance = jest
+        .spyOn(logger, "error")
+        .mockImplementation((): void => {});
+
+      let thrownMessage: string = "";
+
+      try {
+        await LLMService.getCompletion({
+          llmProviderConfig: config,
+          messages: [{ role: "user", content: "SECRET-123" }],
+          includeProviderErrorDetails: false,
+        });
+      } catch (error) {
+        thrownMessage = error instanceof Error ? error.message : String(error);
+      }
+
+      const loggedContent: string = errorLog.mock.calls
+        .flat()
+        .map((value: unknown) => {
+          if (value instanceof Error) {
+            return value.message;
+          }
+
+          return typeof value === "string" ? value : JSON.stringify(value);
+        })
+        .join("\n");
+
+      expect(thrownMessage).toMatch(/provider|API|request failed/i);
+      expect(thrownMessage).not.toContain(privateProviderDetail);
+      expect(thrownMessage).not.toContain("SECRET-123");
+      expect(loggedContent).not.toContain(privateProviderDetail);
+      expect(loggedContent).not.toContain("SECRET-123");
+    },
+  );
+
+  test("preserves detailed provider HTTP errors by default for existing callers", async () => {
+    const privateProviderDetail: string = "existing detailed provider error";
+    jest
+      .spyOn(API, "post")
+      .mockResolvedValue(
+        new HTTPErrorResponse(400, { error: privateProviderDetail }, {}),
+      );
+    jest.spyOn(logger, "error").mockImplementation((): void => {});
+
+    await expect(
+      LLMService.getCompletion({
+        llmProviderConfig: providerCases[0]!.config,
+        messages: [{ role: "user", content: "hello" }],
+      }),
+    ).rejects.toThrow(privateProviderDetail);
+  });
+});