@oneuptime/common 12.0.31 → 12.0.33

package/Tests/Server/Utils/AI/Insights/InsightTriageRunner.test.ts

@@ -495,3 +495,211 @@ describe("InsightTriageRunner.executeTriage", () => {
     );
   });
 });
+
+/*
+ * ERROR-CLASS PRECEDENCE: declared > manual > ai > default.
+ *
+ * The triage runner writes TWO fields through TWO different service methods,
+ * and the difference is the whole point:
+ *
+ *   aiClassification  -> updateOneById, UNCONDITIONAL. The audit record of
+ *                        what the model said, readable even when it lost.
+ *   errorClass        -> updateOneBy,   GUARDED on errorClassSource='default'.
+ *                        The acted-on field: it scopes the Issues list.
+ *
+ * A test that spies only one of them sees half the behaviour and passes
+ * vacuously, which is why both are asserted together here.
+ */
+describe("InsightTriageRunner error-class precedence", () => {
+  /*
+   * Own lifecycle: the sibling describe's afterEach does not reach here, and
+   * jest.spyOn on the same static returns the SAME spy — so without this,
+   * mock.calls accumulate across tests and a "was never called" assertion sees
+   * the previous test's writes.
+   */
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  function stubTriage(telemetryExceptionId: ObjectID): void {
+    jest
+      .spyOn(AIInsightService, "findOneById")
+      .mockResolvedValue(makeInsight({ telemetryExceptionId }));
+    jest.spyOn(AIInsightService, "updateOneById").mockResolvedValue(1);
+    jest
+      .spyOn(AIInsightService, "updateOneBy")
+      .mockResolvedValue(undefined as never);
+    jest.spyOn(ProjectService, "findOneById").mockResolvedValue({
+      id: projectId,
+      enableAi: true,
+      enableInsightFixTasks: false,
+      autoArchiveNonActionableExceptions: false,
+    } as unknown as Project);
+  }
+
+  test("promotes errorClass when nobody else has classified the group", async () => {
+    const telemetryExceptionId: ObjectID = ObjectID.generate();
+    stubTriage(telemetryExceptionId);
+
+    const auditWrite: jest.SpyInstance = jest
+      .spyOn(TelemetryExceptionService, "updateOneById")
+      .mockResolvedValue(undefined as never);
+    // 1 matched row: the group's source was still 'default'.
+    const promotion: jest.SpyInstance = jest
+      .spyOn(TelemetryExceptionService, "updateOneBy")
+      .mockResolvedValue(1 as never);
+
+    driveTriageWithAnalysis(
+      "The caller sent a malformed id.\n\nClassification: user-error",
+    );
+
+    await InsightTriageRunner.executeTriage({
+      aiRunId,
+      projectId,
+      sentinelInsightId,
+      attemptCount: 1,
+    });
+
+    expect(auditWrite).toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: expect.objectContaining({
+          aiClassification: ExceptionAIClassification.UserError,
+        }),
+      }),
+    );
+
+    /*
+     * The source predicate in the WHERE clause is the regression this guards,
+     * and it is invisible to any test whose fixture already has source
+     * 'default' — dropping it would let a stale LLM verdict overwrite a class
+     * the throwing code declared for itself.
+     */
+    expect(promotion).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.objectContaining({
+          _id: telemetryExceptionId.toString(),
+          errorClassSource: "default",
+        }),
+        data: expect.objectContaining({
+          errorClass: ExceptionAIClassification.UserError,
+          errorClassSource: "ai",
+        }),
+        props: expect.objectContaining({ isRoot: true }),
+      }),
+    );
+  });
+
+  /*
+   * A stronger source already owns the field: the conditional UPDATE matches
+   * no row. The AUDIT write must still happen — asserting only "errorClass
+   * unchanged" would pass even if the audit write were wrongly removed.
+   */
+  test("a declared or manual class outranks the AI verdict, but the audit record still lands", async () => {
+    const telemetryExceptionId: ObjectID = ObjectID.generate();
+    stubTriage(telemetryExceptionId);
+
+    const auditWrite: jest.SpyInstance = jest
+      .spyOn(TelemetryExceptionService, "updateOneById")
+      .mockResolvedValue(undefined as never);
+    // 0 matched rows: source was 'declared' or 'manual'.
+    jest
+      .spyOn(TelemetryExceptionService, "updateOneBy")
+      .mockResolvedValue(0 as never);
+
+    driveTriageWithAnalysis(
+      "Looks like a real defect.\n\nClassification: code-fault",
+    );
+
+    await InsightTriageRunner.executeTriage({
+      aiRunId,
+      projectId,
+      sentinelInsightId,
+      attemptCount: 1,
+    });
+
+    expect(auditWrite).toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: expect.objectContaining({
+          aiClassification: ExceptionAIClassification.CodeFault,
+        }),
+      }),
+    );
+  });
+
+  /*
+   * The promotion is a secondary, self-healing write — live traffic re-stamps
+   * errorClass on the group's next occurrence. The auto-archive below it is
+   * user-visible and comparatively irreversible, so a failure here must not
+   * cost us that. Without the guard the two are coupled only by statement
+   * order.
+   */
+  test("a failed promotion does not abort the rest of triage", async () => {
+    const telemetryExceptionId: ObjectID = ObjectID.generate();
+    stubTriage(telemetryExceptionId);
+    jest.spyOn(ProjectService, "findOneById").mockResolvedValue({
+      id: projectId,
+      enableAi: true,
+      enableInsightFixTasks: false,
+      autoArchiveNonActionableExceptions: true,
+    } as unknown as Project);
+
+    const exceptionWrites: jest.SpyInstance = jest
+      .spyOn(TelemetryExceptionService, "updateOneById")
+      .mockResolvedValue(undefined as never);
+    jest
+      .spyOn(TelemetryExceptionService, "updateOneBy")
+      .mockRejectedValue(new Error("postgres said no") as never);
+
+    driveTriageWithAnalysis(
+      "The paywall doing its job.\n\nClassification: expected-denial",
+    );
+
+    await expect(
+      InsightTriageRunner.executeTriage({
+        aiRunId,
+        projectId,
+        sentinelInsightId,
+        attemptCount: 1,
+      }),
+    ).resolves.not.toThrow();
+
+    expect(exceptionWrites).toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: expect.objectContaining({ isArchived: true }),
+      }),
+    );
+  });
+
+  test("no telemetryExceptionId means neither write is attempted", async () => {
+    jest
+      .spyOn(AIInsightService, "findOneById")
+      // makeInsight sets no telemetryExceptionId unless one is passed in.
+      .mockResolvedValue(makeInsight());
+    jest.spyOn(AIInsightService, "updateOneById").mockResolvedValue(1);
+    jest.spyOn(ProjectService, "findOneById").mockResolvedValue({
+      id: projectId,
+      enableAi: true,
+      enableInsightFixTasks: false,
+      autoArchiveNonActionableExceptions: false,
+    } as unknown as Project);
+
+    const auditWrite: jest.SpyInstance = jest
+      .spyOn(TelemetryExceptionService, "updateOneById")
+      .mockResolvedValue(undefined as never);
+    const promotion: jest.SpyInstance = jest
+      .spyOn(TelemetryExceptionService, "updateOneBy")
+      .mockResolvedValue(1 as never);
+
+    driveTriageWithAnalysis("Nothing to attach to.\n\nClassification: unknown");
+
+    await InsightTriageRunner.executeTriage({
+      aiRunId,
+      projectId,
+      sentinelInsightId,
+      attemptCount: 1,
+    });
+
+    expect(auditWrite).not.toHaveBeenCalled();
+    expect(promotion).not.toHaveBeenCalled();
+  });
+});

package/Tests/Server/Utils/AI/LLMServiceToolCalling.test.ts

@@ -248,6 +248,241 @@ describe("LLMService tool calling — Ollama", () => {
     expect(requestBody["tools"]).toHaveLength(1);
     expect(requestBody["stream"]).toBe(false);
   });
+
+  /*
+   * num_ctx is the reason additionalParams had to reach the Ollama branch at
+   * all. Ollama silently truncates a request that exceeds the server's
+   * default context (2048/4096 on common builds) — no error, no warning — and
+   * the chat agent's tool belt alone is several thousand tokens. The
+   * definitions fall off the end of the prompt and the model then answers
+   * that it has no tool for the question, which is exactly the bug users
+   * reported. Raising num_ctx is the operator's only lever, and before this
+   * change the Ollama branch ignored additionalParams entirely.
+   */
+  test("merges additionalParams.options into the Ollama options block, keeping the defaults", async () => {
+    const spy: PostSpy = mockPostResponse({
+      message: { content: "OK" },
+    });
+
+    await LLMService.getCompletion({
+      llmProviderConfig: {
+        llmType: LlmType.Ollama,
+        baseUrl: "http://localhost:11434",
+      },
+      messages: [{ role: "user", content: "which incidents are active?" }],
+      maxTokens: 512,
+      additionalParams: { options: { num_ctx: 32768 } },
+    });
+
+    const requestBody: JSONObject = (
+      spy.mock.calls[0]![0] as { data: JSONObject }
+    ).data;
+    const options: JSONObject = requestBody["options"] as JSONObject;
+
+    expect(options["num_ctx"]).toBe(32768);
+
+    /*
+     * Merged INTO the defaults, not over them: an operator raising num_ctx
+     * must not silently lose the sampling temperature or the output cap the
+     * caller asked for.
+     */
+    expect(options["temperature"]).toBe(0.7);
+    expect(options["num_predict"]).toBe(512);
+  });
+
+  test("applies a non-options additionalParams key at the top level of the body", async () => {
+    const spy: PostSpy = mockPostResponse({
+      message: { content: "OK" },
+    });
+
+    await LLMService.getCompletion({
+      llmProviderConfig: {
+        llmType: LlmType.Ollama,
+        baseUrl: "http://localhost:11434",
+      },
+      messages: [{ role: "user", content: "hi" }],
+      // keep_alive is a top-level /api/chat field, not a generation option.
+      additionalParams: { keep_alive: "30m" },
+    });
+
+    const requestBody: JSONObject = (
+      spy.mock.calls[0]![0] as { data: JSONObject }
+    ).data;
+
+    expect(requestBody["keep_alive"]).toBe("30m");
+    // A top-level key must not leak into the generation options.
+    expect(
+      (requestBody["options"] as JSONObject)["keep_alive"],
+    ).toBeUndefined();
+  });
+
+  /*
+   * additionalParams is operator-supplied tuning stored on the provider row,
+   * so it must never be able to replace the conversation, silence the tool
+   * belt, swap the model, or flip the request to streaming (which this branch
+   * cannot parse). Fail closed on the structural fields regardless of whether
+   * the caller protected its request.
+   */
+  test("additionalParams cannot overwrite model, messages, tools or stream", async () => {
+    const spy: PostSpy = mockPostResponse({
+      message: { content: "OK" },
+    });
+
+    await LLMService.getCompletion({
+      llmProviderConfig: {
+        llmType: LlmType.Ollama,
+        baseUrl: "http://localhost:11434",
+        modelName: "llama3.1",
+      },
+      messages: [{ role: "user", content: "which incidents are active?" }],
+      tools: [
+        {
+          name: "query_incidents",
+          description: "query incidents",
+          inputSchema: { type: "object", properties: {} },
+        },
+      ],
+      additionalParams: {
+        model: "attacker-model",
+        messages: [],
+        tools: [],
+        stream: true,
+      },
+    });
+
+    const requestBody: JSONObject = (
+      spy.mock.calls[0]![0] as { data: JSONObject }
+    ).data;
+
+    expect(requestBody["model"]).toBe("llama3.1");
+
+    const messages: Array<JSONObject> = requestBody[
+      "messages"
+    ] as Array<JSONObject>;
+    expect(messages).toHaveLength(1);
+    expect(messages[0]!["content"]).toBe("which incidents are active?");
+
+    const tools: Array<JSONObject> = requestBody["tools"] as Array<JSONObject>;
+    expect(tools).toHaveLength(1);
+    expect((tools[0]!["function"] as JSONObject)["name"]).toBe(
+      "query_incidents",
+    );
+
+    // Non-streaming: this branch reads a single JSON body, not an SSE stream.
+    expect(requestBody["stream"]).toBe(false);
+  });
+
+  test("protectRequestParameters limits top-level additionalParams to the allowlist but still merges options", async () => {
+    const spy: PostSpy = mockPostResponse({
+      message: { content: "OK" },
+    });
+
+    await LLMService.getCompletion({
+      llmProviderConfig: {
+        llmType: LlmType.Ollama,
+        baseUrl: "http://localhost:11434",
+      },
+      messages: [{ role: "user", content: "hi" }],
+      protectRequestParameters: true,
+      additionalParams: {
+        // On the allowlist of generation-safe tuning fields.
+        top_p: 0.5,
+        // Not on it — a protected caller drops anything it does not know.
+        keep_alive: "30m",
+        options: { num_ctx: 16384 },
+      },
+    });
+
+    const requestBody: JSONObject = (
+      spy.mock.calls[0]![0] as { data: JSONObject }
+    ).data;
+
+    expect(requestBody["top_p"]).toBe(0.5);
+    expect(requestBody["keep_alive"]).toBeUndefined();
+
+    /*
+     * The options merge still applies for a protected caller, and unlike the
+     * top-level keys it is not filtered by the allowlist. That is what keeps
+     * num_ctx reachable for the chat agent — the caller that protects its
+     * request and is also the one whose tool definitions have to survive the
+     * context window. Pinned here so the merge is not later "tidied" behind
+     * the protection check, which would put the truncation bug back.
+     */
+    expect((requestBody["options"] as JSONObject)["num_ctx"]).toBe(16384);
+  });
+
+  test("a protected caller keeps its own temperature and output cap through an options merge", async () => {
+    /*
+     * Every Ollama generation knob lives inside `options`, so the merge that
+     * makes num_ctx reachable is also the one that would let provider config
+     * overwrite what the CALLER owns. Unattended callers protect their
+     * request precisely so their temperature and output cap survive provider
+     * tuning — the OpenAI wire re-asserts both after applying
+     * additionalParams, and the Ollama wire has to match it. Without this,
+     * a provider configured with a creative temperature would silently apply
+     * to graders and summarisers that asked for a deterministic one.
+     */
+    const spy: PostSpy = mockPostResponse({
+      message: { content: "OK" },
+    });
+
+    await LLMService.getCompletion({
+      llmProviderConfig: {
+        llmType: LlmType.Ollama,
+        baseUrl: "http://localhost:11434",
+      },
+      messages: [{ role: "user", content: "hi" }],
+      temperature: 0,
+      maxTokens: 64,
+      protectRequestParameters: true,
+      additionalParams: {
+        options: {
+          num_ctx: 32768,
+          temperature: 1.5,
+          num_predict: 4096,
+        },
+      },
+    });
+
+    const options: JSONObject = (spy.mock.calls[0]![0] as { data: JSONObject })
+      .data["options"] as JSONObject | undefined as JSONObject;
+
+    // The operator's context size still applies — that is the whole point.
+    expect(options["num_ctx"]).toBe(32768);
+
+    // The caller's own settings win over the provider's.
+    expect(options["temperature"]).toBe(0);
+    expect(options["num_predict"]).toBe(64);
+  });
+
+  test("an unprotected caller lets provider options tuning through", async () => {
+    /*
+     * The counterpart to the test above: the chat agent does not protect its
+     * request, and an operator who sets a temperature on their provider row
+     * expects it to take effect. Protection is what narrows this, not the
+     * default.
+     */
+    const spy: PostSpy = mockPostResponse({
+      message: { content: "OK" },
+    });
+
+    await LLMService.getCompletion({
+      llmProviderConfig: {
+        llmType: LlmType.Ollama,
+        baseUrl: "http://localhost:11434",
+      },
+      messages: [{ role: "user", content: "hi" }],
+      temperature: 0,
+      additionalParams: {
+        options: { temperature: 1.5 },
+      },
+    });
+
+    const options: JSONObject = (spy.mock.calls[0]![0] as { data: JSONObject })
+      .data["options"] as JSONObject;
+
+    expect(options["temperature"]).toBe(1.5);
+  });
 });
 
 describe("LLMService — OpenAI-compatible (generic, e.g. vLLM)", () => {