@oneuptime/common 11.5.3 → 11.5.5

package/Server/Utils/AI/SRE/Insights/FixRouting.ts

@@ -0,0 +1,242 @@
+import ObjectID from "../../../../../Types/ObjectID";
+import AIInsightType from "../../../../../Types/AI/AIInsightType";
+import CodeFixTaskType from "../../../../../Types/AI/CodeFixTaskType";
+import {
+  PerformanceCodeLocation,
+  PerformanceFinding,
+} from "../../../../../Types/AI/CodeFixTaskContext";
+import AIInsight from "../../../../../Models/DatabaseModels/AIInsight";
+import Project from "../../../../../Models/DatabaseModels/Project";
+import AIRun from "../../../../../Models/DatabaseModels/AIRun";
+import AIRunService from "../../../../Services/AIRunService";
+import TelemetryExceptionService, {
+  AIFixReadiness,
+} from "../../../../Services/TelemetryExceptionService";
+import FixPerformanceTaskTrigger from "../FixPerformanceTaskTrigger";
+import FixRunBudget, { FixRunBudgetDecision } from "../../CodeFix/FixRunBudget";
+import logger from "../../../Logger";
+import CaptureSpan from "../../../Telemetry/CaptureSpan";
+
+/*
+ * Deterministic fix routing for newly created insights — gated on the
+ * Project.enableAi master switch AND on Project.enableInsightFixTasks
+ * (default FALSE): insights whose evidence points at code (new/spiking
+ * exceptions with a resolvable repository, latency regressions with
+ * span-tree findings) queue a CodeFix AIRun that opens a draft pull
+ * request. The decision is readiness/evidence-based, never LLM-decided.
+ *
+ * Everything downstream reuses the existing budgeted creation paths, so
+ * every guardrail comes free: the daily fix-run budget (G11, pre-checked
+ * here as a quiet skip AND enforced at creation), readiness/repository
+ * gates, per-(exception, recipe) and per-trace dedupe, and the open-PR cap
+ * at the repository-token gate.
+ */
+
+export interface InsightFixRoutingResult {
+  // Set when a fix task was queued for the insight.
+  fixAiRunId?: ObjectID | undefined;
+}
+
+export default class InsightFixRouting {
+  /*
+   * NEVER throws — this runs inside the scanner's per-insight routing and a
+   * failed fix enqueue must not fail (or duplicate) the scan itself. Every
+   * failure is logged and swallowed; an empty result means "no fix opened"
+   * (the insight stays ActionRequired, where the manual fix paths remain
+   * available).
+   */
+  @CaptureSpan()
+  public static async routeInsightFix(data: {
+    insight: AIInsight;
+    project: Project;
+  }): Promise<InsightFixRoutingResult> {
+    const { insight, project } = data;
+
+    try {
+      if (!insight.id || !insight.projectId) {
+        return {};
+      }
+
+      /*
+       * The master AI kill switch, checked BEFORE the per-feature opt-in —
+       * the same order InstrumentationTaskTrigger and InsightTriage use. A
+       * project that turned AI off gets no autonomous agent work at all,
+       * whatever a legacy per-feature flag still says. `=== false` (not
+       * `!== true`) because the column defaults to TRUE: an unselected or
+       * undefined value must never be read as "AI disabled". The scanner
+       * selects enableAi onto the project row it hands us.
+       */
+      if (project.enableAi === false) {
+        logger.debug(
+          `AI insights: not routing a fix for insight ${insight.id.toString()} — AI is disabled for the project.`,
+        );
+        return {};
+      }
+
+      /*
+       * Strict opt-in — the column defaults to false, so unset/legacy rows
+       * never enqueue. Autonomous PR creation must never be default-on.
+       */
+      if (project.enableInsightFixTasks !== true) {
+        logger.debug(
+          `AI insights: not routing a fix for insight ${insight.id.toString()} — project has not opted in to insight fix tasks.`,
+        );
+        return {};
+      }
+
+      const isExceptionInsight: boolean =
+        insight.insightType === AIInsightType.NewException ||
+        insight.insightType === AIInsightType.ExceptionSpike;
+
+      const isLatencyInsight: boolean =
+        insight.insightType === AIInsightType.TraceLatencyRegression;
+
+      /*
+       * ErrorLogSpike and MetricDrift are NEVER auto-fixed: neither carries
+       * evidence that points at a specific code path, so there is no
+       * grounded recipe to run — they route to triage + the human inbox.
+       */
+      if (!isExceptionInsight && !isLatencyInsight) {
+        return {};
+      }
+
+      // Exception insights are only fixable with the exception row to fix.
+      if (isExceptionInsight && !insight.telemetryExceptionId) {
+        logger.debug(
+          `AI insights: not routing a fix for insight ${insight.id.toString()} — no telemetryExceptionId on the insight.`,
+        );
+        return {};
+      }
+
+      /*
+       * Latency insights are only fixable with the deterministic span-tree
+       * findings (and the trace they came from) captured at detect time.
+       */
+      const latencyFindings: Array<PerformanceFinding> =
+        insight.evidence?.latency?.performanceFindings || [];
+      const latencyCodeLocations: Array<PerformanceCodeLocation> =
+        insight.evidence?.latency?.codeLocations || [];
+      const latencyTraceId: string | undefined =
+        insight.traceId || insight.evidence?.latency?.sampleTraceId;
+
+      if (
+        isLatencyInsight &&
+        (latencyFindings.length === 0 || !latencyTraceId)
+      ) {
+        logger.debug(
+          `AI insights: not routing a fix for insight ${insight.id.toString()} — the latency evidence has no span-tree findings or no trace id.`,
+        );
+        return {};
+      }
+
+      /*
+       * G11 guardrail: the per-project daily fix-run budget. For this
+       * AUTOMATIC trigger, over-budget is a logged skip — never an error
+       * (the creation paths below would throw the same rejection, but that
+       * lands in the catch as an unexpected failure; a budget skip is
+       * expected behavior, not a failure).
+       */
+      const budget: FixRunBudgetDecision = await FixRunBudget.getBudgetStatus(
+        insight.projectId,
+      );
+
+      if (!budget.allowed) {
+        logger.debug(
+          `AI insights: not routing a fix for insight ${insight.id.toString()} — ${FixRunBudget.describeRejection(budget)}`,
+        );
+        return {};
+      }
+
+      let createdRun: AIRun;
+
+      if (isExceptionInsight) {
+        /*
+         * Readiness precheck (provider payable, repository resolvable,
+         * agent alive) so an unready project gets a quiet skip instead of
+         * a thrown gate: the insight stays ActionRequired and the exception
+         * page's manual "Fix with AI Agent" button remains available once
+         * the project fixes its setup.
+         */
+        const readiness: AIFixReadiness =
+          await TelemetryExceptionService.getAIFixReadiness({
+            telemetryExceptionId: insight.telemetryExceptionId!,
+            props: { isRoot: true },
+          });
+
+        if (!readiness.ready) {
+          logger.debug(
+            `AI insights: not routing a fix for insight ${insight.id.toString()} — AI fix readiness checks failed.`,
+          );
+          return {};
+        }
+
+        /*
+         * The exception-page creation path, system-triggered (no userId in
+         * props — the run stays system-authored): budget, readiness
+         * re-check and per-(exception, recipe) dedupe are enforced inside.
+         */
+        createdRun =
+          await TelemetryExceptionService.createCodeFixRunForException({
+            telemetryExceptionId: insight.telemetryExceptionId!,
+            props: { isRoot: true },
+            taskType: CodeFixTaskType.FixException,
+          });
+      } else {
+        /*
+         * The findings-based FixPerformance path: the deterministic
+         * evidence gate already ran at detect time (the stored findings ARE
+         * its output); repo gate, per-trace dedupe and budget are enforced
+         * inside.
+         */
+        createdRun =
+          await FixPerformanceTaskTrigger.createPerformanceFixTaskFromFindings({
+            projectId: insight.projectId,
+            traceId: latencyTraceId!,
+            serviceName: insight.serviceName,
+            findings: latencyFindings,
+            codeLocations: latencyCodeLocations,
+          });
+      }
+
+      if (!createdRun.id) {
+        return {};
+      }
+
+      /*
+       * Provenance: stamp the created run with the insight that opened it,
+       * so the run's origin is queryable from either side. Best-effort —
+       * the fix run exists either way, and the scanner links insight ->
+       * run via the returned fixAiRunId.
+       */
+      try {
+        await AIRunService.updateOneById({
+          id: createdRun.id,
+          data: {
+            triggeredByAiInsightId: insight.id,
+          },
+          props: { isRoot: true },
+        });
+      } catch (error) {
+        logger.error(
+          `AI insights: failed to stamp fix run ${createdRun.id.toString()} with insight ${insight.id.toString()}: ${error}`,
+        );
+      }
+
+      logger.debug(
+        `AI insights: queued ${isExceptionInsight ? "FixException" : "FixPerformance"} run ${createdRun.id.toString()} for insight ${insight.id.toString()}.`,
+      );
+
+      return { fixAiRunId: createdRun.id };
+    } catch (error) {
+      /*
+       * Creation-path throws are mostly EXPECTED quiet skips for this
+       * automatic trigger (dedupe hit, readiness re-check raced, budget
+       * raced) — log at debug and leave the insight on the human path.
+       */
+      logger.debug(
+        `AI insights: fix routing skipped for insight ${insight.id?.toString()}: ${error}`,
+      );
+      return {};
+    }
+  }
+}

package/Server/Utils/AI/SRE/Insights/InsightScanner.ts

@@ -0,0 +1,290 @@
+import ObjectID from "../../../../../Types/ObjectID";
+import OneUptimeDate from "../../../../../Types/Date";
+import LIMIT_MAX from "../../../../../Types/Database/LimitMax";
+import Project from "../../../../../Models/DatabaseModels/Project";
+import AIInsight from "../../../../../Models/DatabaseModels/AIInsight";
+import AIInsightStatus from "../../../../../Types/AI/AIInsightStatus";
+import ProjectService from "../../../../Services/ProjectService";
+import AIInsightService from "../../../../Services/AIInsightService";
+import InsightDetectors from "./Detectors/Index";
+import InsightStore, { UpsertCandidatesResult } from "./InsightStore";
+import InsightFixRouting, { InsightFixRoutingResult } from "./FixRouting";
+import InsightTriage, { InsightTriageResult } from "./Triage";
+import { InsightCandidate, InsightDetector, InsightScanContext } from "./Types";
+import logger from "../../../Logger";
+import CaptureSpan from "../../../Telemetry/CaptureSpan";
+
+/*
+ * AI Insights — the scan orchestration (the "watch loop").
+ *
+ * THE RULE: no LLM here. Detectors are deterministic statistical sensors;
+ * the LLM only engages per-finding AFTERWARDS through InsightTriage
+ * (budgeted, read-only), and the fix decision inside InsightFixRouting is
+ * deterministic and reuses the existing budgeted CodeFix creation paths.
+ *
+ * Everything is opt-in and quiet: only projects with the default-false
+ * enableAiInsights flag are scanned, insights never page and never
+ * open incidents, and every layer (project, detector, insight) fails in
+ * isolation — one broken tenant or sensor must not stop the sweep.
+ */
+export default class InsightScanner {
+  /*
+   * One scan tick across all opted-in projects. The flag query IS the gate:
+   * enableAiInsights defaults to false, so nothing runs for a project
+   * that never opted in. Runs as root with explicit projectId scoping
+   * (monitor-worker precedent), not per-user ACL. Never throws — the cron
+   * caller gets a clean resolve either way.
+   */
+  @CaptureSpan()
+  public static async scanAllProjects(): Promise<void> {
+    let projects: Array<Project> = [];
+
+    try {
+      projects = await ProjectService.findBy({
+        query: { enableAiInsights: true },
+        select: {
+          _id: true,
+          enableInsightFixTasks: true,
+          enableAi: true,
+        },
+        limit: LIMIT_MAX,
+        skip: 0,
+        props: { isRoot: true },
+      });
+    } catch (error) {
+      logger.error(
+        `AI Insights: failed to list opted-in projects for the scan tick: ${error}`,
+      );
+      return;
+    }
+
+    for (const project of projects) {
+      try {
+        await this.scanProjectForInsights(project);
+      } catch (error) {
+        logger.error(
+          `AI Insights: scan failed for project ${project.id?.toString()}: ${error}`,
+        );
+      }
+    }
+  }
+
+  /*
+   * One project's scan: self-heal any insight an earlier tick left unrouted,
+   * run every registered detector (isolated — one failing sensor must not
+   * blind the others), dedupe/upsert the candidates through InsightStore,
+   * then route each NEWLY created insight to FixOpened or ActionRequired and
+   * enqueue its triage.
+   */
+  @CaptureSpan()
+  public static async scanProjectForInsights(project: Project): Promise<void> {
+    const projectId: ObjectID | undefined = project.id || undefined;
+
+    if (!projectId) {
+      return;
+    }
+
+    /*
+     * The self-heal sweep runs FIRST — before the detectors — so it can only
+     * ever see rows from EARLIER ticks: this tick's own new rows do not
+     * exist yet and therefore cannot be routed twice.
+     */
+    await this.rerouteStrandedInsights({
+      projectId: projectId,
+      project: project,
+    });
+
+    // The tick's single clock — detectors compute all windows from it.
+    const now: Date = OneUptimeDate.getCurrentDate();
+
+    const context: InsightScanContext = {
+      projectId: projectId,
+      now: now,
+    };
+
+    const candidates: Array<InsightCandidate> = [];
+    const detectors: Array<InsightDetector> =
+      InsightDetectors.getAllDetectors();
+
+    for (const detector of detectors) {
+      try {
+        const detected: Array<InsightCandidate> =
+          await detector.detect(context);
+        candidates.push(...detected);
+      } catch (error) {
+        logger.error(
+          `AI Insights: ${detector.insightType} detector failed for project ${projectId.toString()} — continuing with the remaining detectors: ${error}`,
+        );
+      }
+    }
+
+    if (candidates.length === 0) {
+      return;
+    }
+
+    const upsertResult: UpsertCandidatesResult =
+      await InsightStore.upsertCandidates({
+        projectId: projectId,
+        candidates: candidates,
+        now: now,
+      });
+
+    /*
+     * Only NEWLY created insights are routed — refreshed rows were routed
+     * on the tick that created them and their status is owned by humans and
+     * the fix flow from then on.
+     */
+    for (const insight of upsertResult.created) {
+      try {
+        await this.routeNewInsight({ insight: insight, project: project });
+      } catch (error) {
+        logger.error(
+          `AI Insights: routing failed for insight ${insight.id?.toString()} (project ${projectId.toString()}) — continuing with the remaining insights: ${error}`,
+        );
+      }
+    }
+  }
+
+  /*
+   * Self-heal for insights stranded in the defensive Detected state.
+   *
+   * InsightStore CREATES rows as Detected and the routing status write is a
+   * SEPARATE, later write. If the pod dies in between (eviction, OOM,
+   * deploy) — or that write simply fails — the row is stranded: later ticks
+   * take the REFRESH path, which by contract never touches status and never
+   * re-routes, so the insight would stay Detected FOREVER: never triaged,
+   * never fix-routed, and its fingerprint permanently pinned (the dedupe
+   * lookup would keep refreshing the dead row instead of letting a fresh,
+   * routable insight be created for that finding).
+   *
+   * RE-ROUTING IS SAFE BY CONSTRUCTION — it cannot double-create work:
+   *   - fix creation dedupes inside the creation paths (per (exception,
+   *     recipe) for FixException, per trace for FixPerformance), so a row
+   *     that DID get its fix run created before the pod died gets a rejected
+   *     create here, which InsightFixRouting swallows into "no fix opened";
+   *   - triage dedupes to one non-terminal run per insight, so a re-routed
+   *     insight with a live triage run enqueues nothing;
+   *   - both status writes are idempotent overwrites of the same row.
+   * NEVER throws: a sweep failure must not cost the project its scan.
+   */
+  private static async rerouteStrandedInsights(data: {
+    projectId: ObjectID;
+    project: Project;
+  }): Promise<void> {
+    const { projectId, project } = data;
+
+    let stranded: Array<AIInsight> = [];
+
+    try {
+      stranded = await AIInsightService.findBy({
+        query: {
+          projectId: projectId,
+          status: AIInsightStatus.Detected,
+        },
+        // Exactly the fields routeNewInsight's collaborators read.
+        select: {
+          _id: true,
+          projectId: true,
+          insightType: true,
+          serviceName: true,
+          traceId: true,
+          telemetryExceptionId: true,
+          evidence: true,
+        },
+        limit: LIMIT_MAX,
+        skip: 0,
+        props: { isRoot: true },
+      });
+    } catch (error) {
+      logger.error(
+        `AI Insights: failed to sweep unrouted (Detected) insights for project ${projectId.toString()} — continuing with the scan: ${error}`,
+      );
+      return;
+    }
+
+    for (const insight of stranded) {
+      try {
+        await this.routeNewInsight({ insight: insight, project: project });
+      } catch (error) {
+        logger.error(
+          `AI Insights: re-routing failed for stranded insight ${insight.id?.toString()} (project ${projectId.toString()}) — continuing with the remaining stranded insights: ${error}`,
+        );
+      }
+    }
+  }
+
+  /*
+   * Route one insight — newly created this tick, or swept up as stranded
+   * from an earlier one — out of the defensive Detected state:
+   *   1. deterministic fix routing (gated on enableAi + enableInsightFixTasks
+   *      inside InsightFixRouting) → FixOpened + fixAiRunId, else
+   *      ActionRequired;
+   *   2. triage enqueue for EVERY new insight — triage enriches fix and
+   *      non-fix insights alike → triageAiRunId when a run was enqueued.
+   * Both collaborators are never-throws by contract, but they are wrapped
+   * anyway: a contract breach must degrade (no fix / no triage), not leave
+   * the insight stuck in Detected or fail the scan.
+   */
+  private static async routeNewInsight(data: {
+    insight: AIInsight;
+    project: Project;
+  }): Promise<void> {
+    const { insight, project } = data;
+
+    let fixAiRunId: ObjectID | undefined = undefined;
+
+    try {
+      const fixResult: InsightFixRoutingResult =
+        await InsightFixRouting.routeInsightFix({
+          insight: insight,
+          project: project,
+        });
+      fixAiRunId = fixResult.fixAiRunId;
+    } catch (error) {
+      logger.error(
+        `AI Insights: fix routing threw for insight ${insight.id?.toString()} (treating as no fix opened): ${error}`,
+      );
+    }
+
+    if (fixAiRunId) {
+      await AIInsightService.updateOneById({
+        id: insight.id!,
+        data: {
+          status: AIInsightStatus.FixOpened,
+          fixAiRunId: fixAiRunId,
+        },
+        props: { isRoot: true },
+      });
+    } else {
+      await AIInsightService.updateOneById({
+        id: insight.id!,
+        data: {
+          status: AIInsightStatus.ActionRequired,
+        },
+        props: { isRoot: true },
+      });
+    }
+
+    let triageAiRunId: ObjectID | undefined = undefined;
+
+    try {
+      const triageResult: InsightTriageResult =
+        await InsightTriage.enqueueInsightTriage({ insight: insight });
+      triageAiRunId = triageResult.triageAiRunId;
+    } catch (error) {
+      logger.error(
+        `AI Insights: triage enqueue threw for insight ${insight.id?.toString()} (continuing without triage): ${error}`,
+      );
+    }
+
+    if (triageAiRunId) {
+      await AIInsightService.updateOneById({
+        id: insight.id!,
+        data: {
+          triageAiRunId: triageAiRunId,
+        },
+        props: { isRoot: true },
+      });
+    }
+  }
+}