@oneuptime/common 11.5.3 → 11.5.5

package/Server/Utils/AI/SRE/Insights/Types.ts

@@ -0,0 +1,49 @@
+import ObjectID from "../../../../../Types/ObjectID";
+import AIInsightType from "../../../../../Types/AI/AIInsightType";
+import AIInsightSeverity from "../../../../../Types/AI/AIInsightSeverity";
+import AIInsightEvidence from "../../../../../Types/AI/AIInsightEvidence";
+
+/*
+ * AI Insights — detector contracts.
+ *
+ * THE RULE: no LLM in the watch loop. Detectors are deterministic
+ * statistical sensors — they compare counts, percentiles and means against
+ * the project's own baselines and emit candidates with the evidence already
+ * computed. The LLM only engages per-finding AFTERWARDS (budgeted, read-only
+ * triage), and the decision to open a fix task is deterministic too
+ * (readiness/evidence-based, never LLM-decided).
+ */
+
+// What the scanner hands every detector for one project's scan tick.
+export interface InsightScanContext {
+  projectId: ObjectID;
+  // The scan tick's single clock — all windows are computed from this.
+  now: Date;
+}
+
+/*
+ * One deterministic finding a detector emits. The scanner turns candidates
+ * into AIInsight rows (or refreshes the existing non-terminal insight
+ * with the same fingerprint).
+ */
+export interface InsightCandidate {
+  insightType: AIInsightType;
+  // Stable dedupe key, e.g. "new-exception:<telemetryExceptionId>".
+  fingerprint: string;
+  title: string;
+  // The evidence rendered as markdown — real numbers, no LLM output.
+  detailMarkdown: string;
+  severity: AIInsightSeverity;
+  serviceName?: string | undefined;
+  telemetryServiceId?: ObjectID | undefined;
+  telemetryExceptionId?: ObjectID | undefined;
+  traceId?: string | undefined;
+  metricName?: string | undefined;
+  evidence: AIInsightEvidence;
+}
+
+// One deterministic sensor. Implementations live in ./Detectors/.
+export interface InsightDetector {
+  insightType: AIInsightType;
+  detect(context: InsightScanContext): Promise<Array<InsightCandidate>>;
+}

package/Server/Utils/AI/SRE/InstrumentationTaskTrigger.ts

@@ -0,0 +1,212 @@
+import ObjectID from "../../../../Types/ObjectID";
+import CodeFixTaskType from "../../../../Types/AI/CodeFixTaskType";
+import AIRun from "../../../../Models/DatabaseModels/AIRun";
+import Project from "../../../../Models/DatabaseModels/Project";
+import ProjectService from "../../../Services/ProjectService";
+import SubjectCodeFixRun from "./SubjectCodeFixRun";
+import FixRunBudget, { FixRunBudgetDecision } from "../CodeFix/FixRunBudget";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+
+/*
+ * AI SRE — the ImproveInstrumentation trigger.
+ *
+ * When an investigation finishes INCONCLUSIVE — per the structured G6
+ * confidence signal (ConfidenceSignal.ts), never the analysis prose — the
+ * telemetry itself is the bug: the code paths involved were not observable
+ * enough to diagnose. For projects that opted in
+ * (Project.enableInstrumentationFixTasks, default FALSE — G11 posture:
+ * autonomous PR creation is opt-in only), the inconclusive analysis becomes
+ * the input to a CodeFix AIRun (codeFixTaskType: ImproveInstrumentation)
+ * that the agent worker turns into a pull request adding the missing
+ * observability — so the NEXT investigation of a similar signal can reach a
+ * conclusion.
+ *
+ * Called from the investigation runners' postAnalysis, strictly AFTER the
+ * analysis is posted; this trigger never throws, so a failed enqueue can
+ * never fail (or duplicate) the investigation itself.
+ */
+
+export interface InstrumentationTaskGateInput {
+  // The project row with enableAi + enableInstrumentationFixTasks selected.
+  project: Project | null;
+  // Whether the project has at least one GitHub-App-connected repository.
+  hasConnectedRepository: boolean;
+  // A non-terminal ImproveInstrumentation run for the same subject, if any.
+  existingRun: AIRun | null;
+}
+
+export interface InstrumentationTaskGateDecision {
+  enqueue: boolean;
+  // Human-readable reason recorded in the debug log when skipping.
+  reason: string;
+}
+
+export default class InstrumentationTaskTrigger {
+  /*
+   * The pure trigger decision, separated from IO so it can be tested
+   * directly: opt-in (default FALSE), a repository the agent can actually
+   * open a PR against, and the per-subject dedupe guard.
+   */
+  public static shouldEnqueueInstrumentationTask(
+    input: InstrumentationTaskGateInput,
+  ): InstrumentationTaskGateDecision {
+    if (!input.project) {
+      return { enqueue: false, reason: "project not found" };
+    }
+
+    if (input.project.enableAi === false) {
+      return { enqueue: false, reason: "AI is disabled for the project" };
+    }
+
+    /*
+     * Strict opt-in — the column defaults to false, so unset/legacy rows
+     * never enqueue. Autonomous PR creation must never be default-on.
+     */
+    if (input.project.enableInstrumentationFixTasks !== true) {
+      return {
+        enqueue: false,
+        reason: "project has not opted in to instrumentation fix tasks",
+      };
+    }
+
+    if (!input.hasConnectedRepository) {
+      return {
+        enqueue: false,
+        reason: "project has no GitHub-App-connected repository",
+      };
+    }
+
+    /*
+     * Dedupe: at most one non-terminal ImproveInstrumentation run per
+     * incident/alert — repeated inconclusive attempts on the same subject
+     * must not fan out into duplicate PRs.
+     */
+    if (input.existingRun) {
+      return {
+        enqueue: false,
+        reason:
+          "a non-terminal ImproveInstrumentation run already exists for this subject",
+      };
+    }
+
+    return {
+      enqueue: true,
+      reason: "passed opt-in, repository and dedupe gates",
+    };
+  }
+
+  /*
+   * Record the durable intent as a Queued CodeFix AIRun that the external
+   * agent worker claims via /ai-agent-task/get-pending-task. Exactly one of
+   * incidentId/alertId is expected (the investigation's subject). NEVER
+   * throws — every failure is logged and swallowed, because this runs
+   * inside the investigation's postAnalysis and must not fail it.
+   */
+  @CaptureSpan()
+  public static async enqueueForInconclusiveInvestigation(data: {
+    projectId: ObjectID;
+    incidentId?: ObjectID | undefined;
+    alertId?: ObjectID | undefined;
+  }): Promise<void> {
+    const { projectId } = data;
+
+    try {
+      if (!data.incidentId && !data.alertId) {
+        return;
+      }
+
+      const project: Project | null = await ProjectService.findOneById({
+        id: projectId,
+        select: {
+          enableAi: true,
+          enableInstrumentationFixTasks: true,
+        },
+        props: { isRoot: true },
+      });
+
+      /*
+       * Cheapest gate first: skip the repository count for the (default)
+       * not-opted-in case.
+       */
+      const optInDecision: InstrumentationTaskGateDecision =
+        this.shouldEnqueueInstrumentationTask({
+          project,
+          hasConnectedRepository: true,
+          existingRun: null,
+        });
+
+      if (!optInDecision.enqueue) {
+        logger.debug(
+          `AI: not enqueueing instrumentation task for project ${projectId.toString()} — ${optInDecision.reason}.`,
+        );
+        return;
+      }
+
+      /*
+       * G11 guardrail: the per-project daily fix-run budget. For this
+       * AUTOMATIC trigger, over-budget is a logged skip — never an error
+       * thrown into the investigation (enqueueSubjectCodeFixRun would
+       * throw the same rejection, but that lands in the catch below as an
+       * error log; a budget skip is expected behavior, not a failure).
+       */
+      const budget: FixRunBudgetDecision =
+        await FixRunBudget.getBudgetStatus(projectId);
+
+      if (!budget.allowed) {
+        logger.debug(
+          `AI: not enqueueing instrumentation task for project ${projectId.toString()} — ${FixRunBudget.describeRejection(budget)}`,
+        );
+        return;
+      }
+
+      /*
+       * Repository + per-subject dedupe checks are the enqueue idiom shared
+       * with the FixFromIncident sibling — see SubjectCodeFixRun.
+       */
+      const hasConnectedRepository: boolean =
+        await SubjectCodeFixRun.hasGitHubAppConnectedRepository(projectId);
+
+      const existingRun: AIRun | null =
+        await SubjectCodeFixRun.findNonTerminalRunForSubject({
+          taskType: CodeFixTaskType.ImproveInstrumentation,
+          incidentId: data.incidentId,
+          alertId: data.alertId,
+        });
+
+      const decision: InstrumentationTaskGateDecision =
+        this.shouldEnqueueInstrumentationTask({
+          project,
+          hasConnectedRepository,
+          existingRun,
+        });
+
+      if (!decision.enqueue) {
+        logger.debug(
+          `AI: not enqueueing instrumentation task for project ${projectId.toString()} — ${decision.reason}.`,
+        );
+        return;
+      }
+
+      const createdRun: AIRun =
+        await SubjectCodeFixRun.enqueueSubjectCodeFixRun({
+          projectId,
+          taskType: CodeFixTaskType.ImproveInstrumentation,
+          incidentId: data.incidentId,
+          alertId: data.alertId,
+        });
+
+      logger.debug(
+        `AI: enqueued ImproveInstrumentation run ${createdRun.id?.toString()} for ${
+          data.incidentId
+            ? `incident ${data.incidentId.toString()}`
+            : `alert ${data.alertId?.toString()}`
+        } after an inconclusive investigation.`,
+      );
+    } catch (error) {
+      logger.error(
+        `AI: failed to enqueue instrumentation task for project ${projectId.toString()}: ${error}`,
+      );
+    }
+  }
+}

package/Server/Utils/AI/SRE/InvestigationGrader.ts

@@ -0,0 +1,282 @@
+import ObjectID from "../../../../Types/ObjectID";
+import OneUptimeDate from "../../../../Types/Date";
+import AIRunType from "../../../../Types/AI/AIRunType";
+import AIRunStatus from "../../../../Types/AI/AIRunStatus";
+import AIRunAutoGrade from "../../../../Types/AI/AIRunAutoGrade";
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
+import AIRun from "../../../../Models/DatabaseModels/AIRun";
+import Incident from "../../../../Models/DatabaseModels/Incident";
+import IncidentFeed, {
+  IncidentFeedEventType,
+} from "../../../../Models/DatabaseModels/IncidentFeed";
+import AIRunService from "../../../Services/AIRunService";
+import IncidentService from "../../../Services/IncidentService";
+import IncidentFeedService from "../../../Services/IncidentFeedService";
+import AIService, {
+  AILogResponse,
+  AI_INVESTIGATION_GRADING_FEATURE,
+} from "../../../Services/AIService";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+
+/*
+ * AI SRE — on-resolve investigation grading (Phase 2 measurement layer).
+ *
+ * When an incident resolves with a human-recorded root cause AND a completed
+ * AI investigation exists for it, ONE constrained LLM call compares the
+ * investigation's posted analysis (the RootCause feed item — the only
+ * persisted copy, the same source the fix recipes read) with
+ * Incident.rootCause and stores MATCH / PARTIAL / MISMATCH on the run
+ * (AIRun.autoGrade + autoGradeAt). This feeds the vision §8 accuracy metrics
+ * and the G3 eval work.
+ *
+ * Trigger idiom mirrors IncidentPostmortemRunner: fired (fire-and-forget)
+ * from IncidentStateTimelineService when the incident enters a resolved
+ * state; never throws, never blocks the resolve. Deduped via autoGrade — a
+ * run is graded at most once. Budgeted: the feature is in
+ * AUTONOMOUS_AI_FEATURES, so the daily autonomous token budget covers it.
+ *
+ * Incidents only for now: alerts do have a rootCause column, but no
+ * human-authored resolution flow populates it the way incident resolution
+ * does — grade quality there is unproven, so the alert side is deliberately
+ * skipped (noted in the roadmap).
+ */
+
+/*
+ * Persisted LlmLog label. Owned by AIService so this writer and the G4 budget
+ * match-list (AUTONOMOUS_AI_FEATURES) can never drift — a drift here would
+ * silently drop grading out of the daily budget forever.
+ */
+const FEATURE_NAME: string = AI_INVESTIGATION_GRADING_FEATURE;
+
+// Truncation caps keep the comparison call cheap and inside context limits.
+const MAX_ANALYSIS_CHARS: number = 8000;
+const MAX_ROOT_CAUSE_CHARS: number = 2000;
+
+// Word-bounded token matchers (hoisted: wrap-regex and prettier disagree inline).
+const MISMATCH_RE: RegExp = /\bMISMATCH\b/;
+const PARTIAL_RE: RegExp = /\bPARTIAL\b/;
+const MATCH_RE: RegExp = /\bMATCH\b/;
+
+export interface GradeGateInput {
+  run: {
+    id?: ObjectID | null | undefined;
+    autoGrade?: AIRunAutoGrade | undefined;
+  } | null;
+  rootCause: string | null | undefined;
+}
+
+export default class InvestigationGrader {
+  /*
+   * Pure gating decision (exported for tests): grade only when a completed
+   * investigation run exists, it has not been graded yet (dedupe), and the
+   * incident carries a non-empty human-recorded root cause to grade against.
+   */
+  public static shouldGradeInvestigation(data: GradeGateInput): boolean {
+    if (!data.run || !data.run.id) {
+      return false;
+    }
+
+    if (data.run.autoGrade) {
+      // Dedupe: a run is graded at most once.
+      return false;
+    }
+
+    if (!data.rootCause || data.rootCause.trim().length === 0) {
+      return false;
+    }
+
+    return true;
+  }
+
+  /*
+   * Defensive one-token parse (pure, exported for tests). The prompt demands
+   * exactly one token, but models editorialize — accept the verdict when
+   * exactly ONE of the three tokens appears in the response, word-bounded so
+   * the MATCH inside MISMATCH does not double-count. None found, or more
+   * than one distinct token → unparseable → null (no grade stored).
+   */
+  public static parseAutoGradeToken(
+    response: string | null | undefined,
+  ): AIRunAutoGrade | null {
+    if (!response) {
+      return null;
+    }
+
+    const text: string = response.toUpperCase();
+
+    const found: Array<AIRunAutoGrade> = [];
+
+    if (MISMATCH_RE.test(text)) {
+      found.push(AIRunAutoGrade.Mismatch);
+    }
+
+    if (PARTIAL_RE.test(text)) {
+      found.push(AIRunAutoGrade.Partial);
+    }
+
+    if (MATCH_RE.test(text)) {
+      found.push(AIRunAutoGrade.Match);
+    }
+
+    if (found.length !== 1) {
+      return null;
+    }
+
+    return found[0]!;
+  }
+
+  /*
+   * Entry point called (fire-and-forget) from IncidentStateTimelineService
+   * when an incident enters a resolved state. Never throws — all failures
+   * are logged and swallowed so the resolve flow can never be affected.
+   */
+  @CaptureSpan()
+  public static async gradeInvestigationOnResolve(data: {
+    incidentId: ObjectID;
+    projectId: ObjectID;
+  }): Promise<void> {
+    const { incidentId, projectId } = data;
+
+    try {
+      // The latest completed investigation for this incident (if any).
+      const run: AIRun | null = await AIRunService.findOneBy({
+        query: {
+          runType: AIRunType.Investigation,
+          status: AIRunStatus.Completed,
+          triggeredByIncidentId: incidentId,
+        },
+        select: {
+          _id: true,
+          autoGrade: true,
+        },
+        sort: { createdAt: SortOrder.Descending },
+        props: { isRoot: true },
+      });
+
+      if (!run) {
+        // Nothing to grade — skip before loading anything else.
+        return;
+      }
+
+      const incident: Incident | null = await IncidentService.findOneById({
+        id: incidentId,
+        select: {
+          _id: true,
+          rootCause: true,
+        },
+        props: { isRoot: true },
+      });
+
+      if (
+        !this.shouldGradeInvestigation({
+          run,
+          rootCause: incident?.rootCause,
+        })
+      ) {
+        return;
+      }
+
+      /*
+       * The posted analysis: the latest RootCause feed item — the AI's
+       * postAnalysis is its only writer, and it is the only persisted copy
+       * of the analysis (the same source the fix recipes read).
+       */
+      const feedItem: IncidentFeed | null = await IncidentFeedService.findOneBy(
+        {
+          query: {
+            incidentId: incidentId,
+            incidentFeedEventType: IncidentFeedEventType.RootCause,
+          },
+          select: {
+            feedInfoInMarkdown: true,
+          },
+          sort: {
+            createdAt: SortOrder.Descending,
+          },
+          props: { isRoot: true },
+        },
+      );
+
+      const analysisMarkdown: string = (
+        feedItem?.feedInfoInMarkdown || ""
+      ).trim();
+
+      if (!analysisMarkdown) {
+        logger.debug(
+          `AI grading: no posted analysis found for incident ${incidentId.toString()} — skipping.`,
+        );
+        return;
+      }
+
+      const rootCause: string = incident!.rootCause!.trim();
+
+      const response: AILogResponse = await AIService.executeWithLogging({
+        projectId,
+        feature: FEATURE_NAME,
+        incidentId,
+        aiRunId: run.id!,
+        // Grades are stored on the run; no need to persist prompt previews.
+        storeContentPreviews: false,
+        temperature: 0,
+        maxTokens: 20,
+        messages: [
+          {
+            role: "system",
+            content: [
+              "You grade an AI incident investigation against the root cause humans recorded when they resolved the incident.",
+              "Respond with EXACTLY one token and nothing else:",
+              "MATCH — the investigation identified the same root cause.",
+              "PARTIAL — the investigation identified part of the cause, or a closely related cause.",
+              "MISMATCH — the investigation pointed somewhere else.",
+            ].join("\n"),
+          },
+          {
+            role: "user",
+            content: [
+              "Human-recorded root cause:",
+              '"""',
+              rootCause.substring(0, MAX_ROOT_CAUSE_CHARS),
+              '"""',
+              "",
+              "AI investigation analysis:",
+              '"""',
+              analysisMarkdown.substring(0, MAX_ANALYSIS_CHARS),
+              '"""',
+              "",
+              "One token only: MATCH, PARTIAL, or MISMATCH.",
+            ].join("\n"),
+          },
+        ],
+      });
+
+      const grade: AIRunAutoGrade | null = this.parseAutoGradeToken(
+        response.content,
+      );
+
+      if (!grade) {
+        logger.warn(
+          `AI grading: unparseable grade response for incident ${incidentId.toString()} (run ${run.id!.toString()}) — no grade stored.`,
+        );
+        return;
+      }
+
+      await AIRunService.updateOneById({
+        id: run.id!,
+        data: {
+          autoGrade: grade,
+          autoGradeAt: OneUptimeDate.getCurrentDate(),
+        },
+        props: { isRoot: true },
+      });
+
+      logger.debug(
+        `AI grading: incident ${incidentId.toString()} investigation graded ${grade}.`,
+      );
+    } catch (error) {
+      logger.error(
+        `AI grading: failed for incident ${incidentId.toString()}: ${error}`,
+      );
+    }
+  }
+}