@oneuptime/common 11.5.4 → 11.5.5

package/Tests/Server/Utils/AI/Insights/InsightTriageRunner.test.ts

@@ -0,0 +1,221 @@
+import InsightTriageRunner from "../../../../../Server/Utils/AI/SRE/Insights/InsightTriageRunner";
+import AIInvestigationEngine, {
+  InvestigationRequest,
+} from "../../../../../Server/Utils/AI/SRE/AIInvestigationEngine";
+import AIInvestigationQueue from "../../../../../Server/Utils/AI/SRE/InvestigationQueue";
+import InstrumentationTaskTrigger from "../../../../../Server/Utils/AI/SRE/InstrumentationTaskTrigger";
+import { ConfidenceSignal } from "../../../../../Server/Utils/AI/SRE/ConfidenceSignal";
+import { ObservabilityAssistantResult } from "../../../../../Server/Utils/AI/Chat/ObservabilityAssistant";
+import AIInsightService from "../../../../../Server/Services/AIInsightService";
+import IncidentFeedService from "../../../../../Server/Services/IncidentFeedService";
+import AlertFeedService from "../../../../../Server/Services/AlertFeedService";
+import { AI_INSIGHT_TRIAGE_FEATURE } from "../../../../../Server/Services/AIService";
+import AIInsight from "../../../../../Models/DatabaseModels/AIInsight";
+import AIInsightType from "../../../../../Types/AI/AIInsightType";
+import AIInsightSeverity from "../../../../../Types/AI/AIInsightSeverity";
+import ObjectID from "../../../../../Types/ObjectID";
+import { describe, expect, test, afterEach } from "@jest/globals";
+
+/*
+ * The insight triage runner — the Preventive lane's quiet-inbox executor.
+ * The invariants these tests lock in:
+ *   (a) a context-build failure (insight gone, db error) hands the CLAIMED
+ *       run to the queue's retry policy as NON-permanent, and the engine
+ *       never runs;
+ *   (b) the engine request carries the dedicated budgeted feature label
+ *       (AI_INSIGHT_TRIAGE_FEATURE — G4) and a context that recasts
+ *       the run as preventive triage of the detector's finding;
+ *   (c) postAnalysis writes triageSummaryMarkdown + triageCompletedAt onto
+ *       the insight row and NOTHING else — no feed items, no workspace
+ *       notification, no instrumentation-task trigger. Insights never page.
+ */
+
+const aiRunId: ObjectID = ObjectID.generate();
+const projectId: ObjectID = ObjectID.generate();
+const sentinelInsightId: ObjectID = ObjectID.generate();
+
+function makeInsight(): AIInsight {
+  return {
+    id: sentinelInsightId,
+    insightType: AIInsightType.ExceptionSpike,
+    severity: AIInsightSeverity.High,
+    title: "Spike: NullPointerException in checkout",
+    detailMarkdown: "Recent hour: 240 occurrences vs a baseline of 4/hour.",
+    serviceName: "checkout",
+    evidence: {
+      exception: {
+        exceptionMessage: "NullPointerException",
+        recentOccurrenceCount: 240,
+        baselineHourlyAverage: 4,
+        spikeMultiplier: 60,
+      },
+    },
+  } as unknown as AIInsight;
+}
+
+function makeConfidence(): ConfidenceSignal {
+  return { confident: true, source: "classification" };
+}
+
+describe("InsightTriageRunner.executeTriage", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("a context-build failure hands the claimed run to the retry policy (non-permanent)", async () => {
+    jest
+      .spyOn(AIInsightService, "findOneById")
+      .mockRejectedValue(new Error("db down"));
+    const failOrRequeue: jest.SpyInstance = jest
+      .spyOn(AIInvestigationQueue, "failOrRequeue")
+      .mockResolvedValue(undefined);
+    const executeRun: jest.SpyInstance = jest.spyOn(
+      AIInvestigationEngine,
+      "executeRun",
+    );
+
+    await InsightTriageRunner.executeTriage({
+      aiRunId,
+      projectId,
+      sentinelInsightId,
+      attemptCount: 1,
+    });
+
+    expect(failOrRequeue).toHaveBeenCalledWith(
+      expect.objectContaining({
+        aiRunId,
+        attemptCount: 1,
+        isPermanent: false,
+      }),
+    );
+    expect(executeRun).not.toHaveBeenCalled();
+  });
+
+  test("a deleted insight is also a non-permanent context failure", async () => {
+    jest.spyOn(AIInsightService, "findOneById").mockResolvedValue(null);
+    const failOrRequeue: jest.SpyInstance = jest
+      .spyOn(AIInvestigationQueue, "failOrRequeue")
+      .mockResolvedValue(undefined);
+
+    await InsightTriageRunner.executeTriage({
+      aiRunId,
+      projectId,
+      sentinelInsightId,
+      attemptCount: 1,
+    });
+
+    expect(failOrRequeue).toHaveBeenCalledWith(
+      expect.objectContaining({ isPermanent: false }),
+    );
+  });
+
+  test("the engine request carries the triage feature and a preventive-triage context built from the insight", async () => {
+    jest
+      .spyOn(AIInsightService, "findOneById")
+      .mockResolvedValue(makeInsight());
+    const executeRun: jest.SpyInstance = jest
+      .spyOn(AIInvestigationEngine, "executeRun")
+      .mockResolvedValue(undefined);
+
+    await InsightTriageRunner.executeTriage({
+      aiRunId,
+      projectId,
+      sentinelInsightId,
+      attemptCount: 1,
+    });
+
+    expect(executeRun).toHaveBeenCalledWith(
+      expect.objectContaining({
+        aiRunId,
+        projectId,
+        attemptCount: 1,
+        request: expect.objectContaining({
+          feature: AI_INSIGHT_TRIAGE_FEATURE,
+        }),
+      }),
+    );
+
+    const request: InvestigationRequest = (
+      executeRun.mock.calls[0]![0] as {
+        request: InvestigationRequest;
+      }
+    ).request;
+
+    // Recast as preventive triage — no incident exists, a sensor filed this.
+    expect(request.contextSummary).toContain("PREVENTIVE TRIAGE");
+    expect(request.contextSummary).toContain("ONE next action");
+    // The subject description: type, title and the detector's evidence.
+    expect(request.contextSummary).toContain(AIInsightType.ExceptionSpike);
+    expect(request.contextSummary).toContain(
+      "Spike: NullPointerException in checkout",
+    );
+    expect(request.contextSummary).toContain(
+      "Recent hour: 240 occurrences vs a baseline of 4/hour.",
+    );
+    // The raw evidence numbers rendered compactly.
+    expect(request.contextSummary).toContain('"spikeMultiplier":60');
+  });
+
+  test("postAnalysis writes the summary onto the insight row — and notifies NOTHING (quiet inbox)", async () => {
+    jest
+      .spyOn(AIInsightService, "findOneById")
+      .mockResolvedValue(makeInsight());
+    const persist: jest.SpyInstance = jest
+      .spyOn(AIInsightService, "updateOneById")
+      .mockResolvedValue(undefined);
+    const incidentFeed: jest.SpyInstance = jest
+      .spyOn(IncidentFeedService, "createIncidentFeedItem")
+      .mockResolvedValue(undefined);
+    const alertFeed: jest.SpyInstance = jest
+      .spyOn(AlertFeedService, "createAlertFeedItem")
+      .mockResolvedValue(undefined);
+    const instrumentationTrigger: jest.SpyInstance = jest
+      .spyOn(InstrumentationTaskTrigger, "enqueueForInconclusiveInvestigation")
+      .mockResolvedValue(undefined);
+
+    // Drive the engine's postAnalysis callback the way the real engine does.
+    jest
+      .spyOn(AIInvestigationEngine, "executeRun")
+      .mockImplementation(
+        async (data: {
+          aiRunId: ObjectID;
+          projectId: ObjectID;
+          attemptCount: number;
+          request: InvestigationRequest;
+        }): Promise<void> => {
+          await data.request.postAnalysis({
+            analysisMarkdown: "## 🧠 AI SRE — branded triage analysis",
+            confidence: makeConfidence(),
+            result: {} as unknown as ObservabilityAssistantResult,
+          });
+        },
+      );
+
+    await InsightTriageRunner.executeTriage({
+      aiRunId,
+      projectId,
+      sentinelInsightId,
+      attemptCount: 1,
+    });
+
+    expect(persist).toHaveBeenCalledWith(
+      expect.objectContaining({
+        id: sentinelInsightId,
+        data: expect.objectContaining({
+          triageSummaryMarkdown: "## 🧠 AI SRE — branded triage analysis",
+          triageCompletedAt: expect.any(Date),
+        }),
+        props: expect.objectContaining({ isRoot: true }),
+      }),
+    );
+
+    /*
+     * The quiet-inbox rule: insights never page and never enter the
+     * notification pipeline — zero feed items, zero workspace pings, zero
+     * autonomous follow-up tasks from triage (v1).
+     */
+    expect(incidentFeed).not.toHaveBeenCalled();
+    expect(alertFeed).not.toHaveBeenCalled();
+    expect(instrumentationTrigger).not.toHaveBeenCalled();
+  });
+});

package/Tests/Server/Utils/AI/Insights/InvestigationQueueInsightSubject.test.ts

@@ -0,0 +1,328 @@
+import AIInvestigationQueue, {
+  INSIGHT_TRIAGE_RESERVED_SLOTS,
+} from "../../../../../Server/Utils/AI/SRE/InvestigationQueue";
+import InsightTriageRunner from "../../../../../Server/Utils/AI/SRE/Insights/InsightTriageRunner";
+import AIIncidentInvestigationRunner from "../../../../../Server/Utils/AI/SRE/IncidentInvestigationRunner";
+import AIRunService from "../../../../../Server/Services/AIRunService";
+import AIService from "../../../../../Server/Services/AIService";
+import ProjectService from "../../../../../Server/Services/ProjectService";
+import Project from "../../../../../Models/DatabaseModels/Project";
+import AIRun from "../../../../../Models/DatabaseModels/AIRun";
+import AIRunStatus from "../../../../../Types/AI/AIRunStatus";
+import ObjectID from "../../../../../Types/ObjectID";
+import PositiveNumber from "../../../../../Types/PositiveNumber";
+import { describe, expect, test, afterEach, beforeEach } from "@jest/globals";
+
+/*
+ * The investigation queue's AI-insight subject support. The
+ * invariants these tests lock in:
+ *   (a) enqueue stamps subjectAIInsightId onto the run as
+ *       triggeredByAiInsightId (provenance + the dispatch key) and
+ *       returns the created run's id so the caller can link the insight;
+ *   (b) the budget quiet-skip returns null and creates nothing — the
+ *       existing enqueue-time gate applies to triage runs too;
+ *   (c) dispatch recognizes insight runs as subject-BEARING and routes
+ *       them to InsightTriageRunner.executeTriage with the claimed attempt;
+ *   (d) a run with no subject at all is still failed at dispatch;
+ *   (e) LANE PRIORITY: triage (the preventive lane) may hold at most
+ *       (cap - INSIGHT_TRIAGE_RESERVED_SLOTS) of the project's concurrency
+ *       slots, so a scan's triage backlog can never starve the interactive
+ *       lane (incident/alert RCA, where a human is waiting) — while the
+ *       interactive lane's own gating is left byte-for-byte unchanged.
+ */
+
+function mockBudget(exhausted: boolean): void {
+  jest.spyOn(AIService, "getAutonomousDailyBudgetStatus").mockResolvedValue({
+    exhausted,
+    limitInTokens: exhausted ? 1000 : null,
+    usedTokensToday: exhausted ? 1000 : 0,
+  });
+}
+
+describe("AIInvestigationQueue — insight subject", () => {
+  beforeEach(() => {
+    // No per-project cap override; no investigations currently running.
+    jest
+      .spyOn(ProjectService, "findOneById")
+      .mockResolvedValue({ id: ObjectID.generate() } as unknown as Project);
+    jest
+      .spyOn(AIRunService, "countBy")
+      .mockResolvedValue(new PositiveNumber(0));
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("enqueue stamps triggeredByAiInsightId and returns the created run id", async () => {
+    mockBudget(false);
+    const insightId: ObjectID = ObjectID.generate();
+    const createdId: ObjectID = ObjectID.generate();
+
+    const create: jest.SpyInstance = jest
+      .spyOn(AIRunService, "create")
+      .mockResolvedValue({ id: createdId } as unknown as AIRun);
+    jest.spyOn(AIInvestigationQueue, "processRun").mockResolvedValue(undefined);
+
+    const returnedId: ObjectID | null = await AIInvestigationQueue.enqueue({
+      projectId: ObjectID.generate(),
+      subjectAIInsightId: insightId,
+    });
+
+    expect(returnedId).toEqual(createdId);
+    expect(create).toHaveBeenCalledWith(
+      expect.objectContaining({
+        data: expect.objectContaining({
+          status: AIRunStatus.Queued,
+          triggeredByAiInsightId: insightId,
+        }),
+        props: expect.objectContaining({ isRoot: true }),
+      }),
+    );
+  });
+
+  test("the budget quiet-skip returns null and creates no run", async () => {
+    mockBudget(true);
+    const create: jest.SpyInstance = jest.spyOn(AIRunService, "create");
+
+    const returnedId: ObjectID | null = await AIInvestigationQueue.enqueue({
+      projectId: ObjectID.generate(),
+      subjectAIInsightId: ObjectID.generate(),
+    });
+
+    expect(returnedId).toBeNull();
+    expect(create).not.toHaveBeenCalled();
+  });
+
+  test("a won claim on an insight run dispatches to the triage runner with the claimed attempt", async () => {
+    mockBudget(false);
+    const runId: ObjectID = ObjectID.generate();
+    const projectId: ObjectID = ObjectID.generate();
+    const insightId: ObjectID = ObjectID.generate();
+
+    jest.spyOn(AIRunService, "attemptStatusTransition").mockResolvedValue(1);
+    const executeTriage: jest.SpyInstance = jest
+      .spyOn(InsightTriageRunner, "executeTriage")
+      .mockResolvedValue(undefined);
+
+    await AIInvestigationQueue.processRun({
+      id: runId,
+      projectId,
+      attemptCount: 0,
+      triggeredByAiInsightId: insightId,
+    });
+
+    expect(executeTriage).toHaveBeenCalledWith(
+      expect.objectContaining({
+        aiRunId: runId,
+        projectId,
+        sentinelInsightId: insightId,
+        attemptCount: 1,
+      }),
+    );
+  });
+
+  test("a run with no subject at all is still failed at dispatch", async () => {
+    mockBudget(false);
+    const runId: ObjectID = ObjectID.generate();
+    const transition: jest.SpyInstance = jest
+      .spyOn(AIRunService, "attemptStatusTransition")
+      .mockResolvedValue(1);
+    const executeTriage: jest.SpyInstance = jest.spyOn(
+      InsightTriageRunner,
+      "executeTriage",
+    );
+
+    await AIInvestigationQueue.processRun({
+      id: runId,
+      projectId: ObjectID.generate(),
+      attemptCount: 0,
+    });
+
+    expect(executeTriage).not.toHaveBeenCalled();
+    expect(transition).toHaveBeenCalledWith(
+      expect.objectContaining({
+        aiRunId: runId,
+        fromStatus: AIRunStatus.Running,
+        set: expect.objectContaining({
+          status: AIRunStatus.Error,
+        }),
+      }),
+    );
+  });
+});
+
+/*
+ * The starvation scenario this locks down: a scan files 10 insights, so 10
+ * triage runs queue. Without the lane sub-cap they would fill every slot of
+ * the default cap of 3 and the incident that fires a minute later would wait
+ * behind them. With it, triage tops out at (cap - 1) and the incident's
+ * inline kick at enqueue always finds the reserved slot free.
+ */
+describe("AIInvestigationQueue — lane priority (triage never starves RCA)", () => {
+  const projectId: ObjectID = ObjectID.generate();
+
+  // Per-project concurrency cap. The default is 3; make it explicit here.
+  function mockCap(cap: number): void {
+    jest.spyOn(ProjectService, "findOneById").mockResolvedValue({
+      id: projectId,
+      aiMaxConcurrentInvestigations: cap,
+    } as unknown as Project);
+  }
+
+  /*
+   * The claim gates count Running investigations twice: once globally, and
+   * once filtered to the triage lane (the query carrying
+   * triggeredByAiInsightId). Answer each query independently so the
+   * two caps can be driven apart.
+   */
+  function mockRunningCounts(counts: {
+    total: number;
+    triage: number;
+  }): jest.SpyInstance {
+    return jest
+      .spyOn(AIRunService, "countBy")
+      .mockImplementation((data: unknown): Promise<PositiveNumber> => {
+        const query: Record<string, unknown> =
+          ((data as { query?: Record<string, unknown> }).query as Record<
+            string,
+            unknown
+          >) || {};
+
+        const isTriageLaneQuery: boolean =
+          query["triggeredByAiInsightId"] !== undefined;
+
+        return Promise.resolve(
+          new PositiveNumber(isTriageLaneQuery ? counts.triage : counts.total),
+        );
+      });
+  }
+
+  beforeEach(() => {
+    mockBudget(false);
+    jest.spyOn(AIRunService, "attemptStatusTransition").mockResolvedValue(1);
+  });
+
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("the reserved slot is real: with the triage lane full (2 of cap 3), an incident investigation still claims and dispatches", async () => {
+    mockCap(3);
+    // Both running investigations are triage runs — the lane's sub-cap.
+    mockRunningCounts({ total: 2, triage: 2 });
+    const claim: jest.SpyInstance = jest.spyOn(
+      AIRunService,
+      "attemptStatusTransition",
+    );
+    const executeInvestigation: jest.SpyInstance = jest
+      .spyOn(AIIncidentInvestigationRunner, "executeInvestigation")
+      .mockResolvedValue(undefined);
+
+    const incidentId: ObjectID = ObjectID.generate();
+
+    await AIInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId,
+      attemptCount: 0,
+      triggeredByIncidentId: incidentId,
+    });
+
+    expect(claim).toHaveBeenCalledWith(
+      expect.objectContaining({ fromStatus: AIRunStatus.Queued }),
+    );
+    expect(executeInvestigation).toHaveBeenCalledWith(
+      expect.objectContaining({ incidentId, attemptCount: 1 }),
+    );
+  });
+
+  test("a triage run at the lane sub-cap is left Queued — the claim is never even attempted", async () => {
+    mockCap(3);
+    // 2 triage running = cap(3) - RESERVED(1): the lane is full.
+    mockRunningCounts({ total: 2, triage: 3 - INSIGHT_TRIAGE_RESERVED_SLOTS });
+    const claim: jest.SpyInstance = jest.spyOn(
+      AIRunService,
+      "attemptStatusTransition",
+    );
+    const executeTriage: jest.SpyInstance = jest.spyOn(
+      InsightTriageRunner,
+      "executeTriage",
+    );
+
+    await AIInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId,
+      attemptCount: 0,
+      triggeredByAiInsightId: ObjectID.generate(),
+    });
+
+    // Left Queued for the poller/TTL — never claimed, never executed.
+    expect(claim).not.toHaveBeenCalled();
+    expect(executeTriage).not.toHaveBeenCalled();
+  });
+
+  test("triage still runs while under the lane sub-cap — the sub-cap throttles, it does not disable", async () => {
+    mockCap(3);
+    mockRunningCounts({ total: 1, triage: 1 });
+    const executeTriage: jest.SpyInstance = jest
+      .spyOn(InsightTriageRunner, "executeTriage")
+      .mockResolvedValue(undefined);
+
+    await AIInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId,
+      attemptCount: 0,
+      triggeredByAiInsightId: ObjectID.generate(),
+    });
+
+    expect(executeTriage).toHaveBeenCalledWith(
+      expect.objectContaining({ attemptCount: 1 }),
+    );
+  });
+
+  test("a project pinned to the minimum cap of 1 still triages — the lane floor is 1, not 0 (no deadlock)", async () => {
+    mockCap(1);
+    mockRunningCounts({ total: 0, triage: 0 });
+    const executeTriage: jest.SpyInstance = jest
+      .spyOn(InsightTriageRunner, "executeTriage")
+      .mockResolvedValue(undefined);
+
+    await AIInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId,
+      attemptCount: 0,
+      triggeredByAiInsightId: ObjectID.generate(),
+    });
+
+    expect(executeTriage).toHaveBeenCalled();
+  });
+
+  test("interactive gating is unchanged when no insight runs exist: one global count query, no lane query, and the run claims", async () => {
+    mockCap(3);
+    const countBy: jest.SpyInstance = mockRunningCounts({
+      total: 0,
+      triage: 0,
+    });
+    const executeInvestigation: jest.SpyInstance = jest
+      .spyOn(AIIncidentInvestigationRunner, "executeInvestigation")
+      .mockResolvedValue(undefined);
+
+    await AIInvestigationQueue.processRun({
+      id: ObjectID.generate(),
+      projectId,
+      attemptCount: 0,
+      triggeredByIncidentId: ObjectID.generate(),
+    });
+
+    expect(executeInvestigation).toHaveBeenCalled();
+    // The lane sub-cap query is only paid for by the lane it caps.
+    expect(countBy).toHaveBeenCalledTimes(1);
+    expect(countBy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        query: expect.not.objectContaining({
+          triggeredByAiInsightId: expect.anything(),
+        }),
+      }),
+    );
+  });
+});