@oneuptime/common 12.0.4 → 12.0.6

package/Server/Utils/AI/SRE/AIInvestigationEngine.ts

@@ -6,6 +6,7 @@ import {
   AIRunEventResultSummary,
 } from "../../../../Types/AI/AIChatTypes";
 import AIRunStatus from "../../../../Types/AI/AIRunStatus";
+import AIRunCodeFixRecommendation from "../../../../Types/AI/AIRunCodeFixRecommendation";
 import AIRunEventType from "../../../../Types/AI/AIRunEventType";
 import AIRunEvent from "../../../../Models/DatabaseModels/AIRunEvent";
 import Project from "../../../../Models/DatabaseModels/Project";
@@ -16,6 +17,7 @@ import ProjectService from "../../../Services/ProjectService";
 import LlmProviderService from "../../../Services/LlmProviderService";
 import AIInvestigationQueue from "./InvestigationQueue";
 import AIConfidenceSignal, { ConfidenceSignal } from "./ConfidenceSignal";
+import InvestigationTldr from "./InvestigationTldr";
 import ObservabilityAssistant, {
   ObservabilityAssistantExtraTool,
   ObservabilityAssistantResult,
@@ -49,6 +51,12 @@ const MAX_TOOL_CALLS: number = 12;
 const MAX_WALL_CLOCK_MS: number = 150 * 1000;
 const MAX_OUTPUT_TOKENS: number = 2000;
 
+/*
+ * Conditional recommendation persistence is idempotent, so transient
+ * database failures can be retried without risking a second decision.
+ */
+const CODE_FIX_RECOMMENDATION_PERSIST_ATTEMPTS: number = 3;
+
 /*
  * Failures a retry cannot fix within the run's usefulness window: missing/
  * broken provider configuration and budget exhaustion (both messages minted
@@ -89,6 +97,12 @@ export type AISubjectType = "Incident" | "Alert";
 export interface InvestigationRequest {
   // Label recorded on LlmLog, e.g. "AI Incident Investigation".
   feature: string;
+  /*
+   * The subject lane used for per-incident / per-alert autonomous budgets.
+   * Insight triage is intentionally subjectless and leaves both unset.
+   */
+  incidentId?: ObjectID | undefined;
+  alertId?: ObjectID | undefined;
   // A compact markdown summary of the subject that seeds the investigation.
   contextSummary: string;
   /*
@@ -105,6 +119,28 @@ export interface InvestigationRequest {
     confidence: ConfidenceSignal;
     result: ObservabilityAssistantResult;
   }) => Promise<void>;
+  /*
+   * Incident and alert RCA runs opt into a persisted code-fix decision.
+   * Other users of this engine (insight triage and remediation) omit it and
+   * remain NotRecommended throughout their lifecycle.
+   */
+  persistCodeFixRecommendation?: boolean | undefined;
+  /*
+   * Incident and alert RCA runs opt into the AI-written TL;DR that the
+   * dashboard panel shows above the report. Display-only and best-effort:
+   * a failed generation leaves the run without one and changes nothing else.
+   * Engine users with no panel (insight triage, remediation) omit it.
+   */
+  persistAnalysisTldr?: boolean | undefined;
+  /*
+   * Runs only after a Recommended decision and its exact analysis snapshot
+   * have been durably committed. Incident/alert runners use this for the
+   * optional automatic FixFromIncident lane; persistence failure means the
+   * callback is never invoked.
+   */
+  onCodeFixRecommended?:
+    | ((data: { analysisMarkdown: string }) => Promise<void>)
+    | undefined;
   /*
    * Optional: called exactly once when THIS attempt settles the run into a
    * terminal state — Completed (with or without an analysis, and even when
@@ -264,7 +300,7 @@ export default class AIInvestigationEngine {
        * EVERY step — a slow self-hosted LLM call can approach the sweeper's
        * timeout, so the heartbeat must be as frequent as we can make it.
        */
-      await this.touchHeartbeat(aiRunId);
+      await this.touchHeartbeat(aiRunId, data.attemptCount);
     };
 
     /*
@@ -279,6 +315,9 @@ export default class AIInvestigationEngine {
       const result: ObservabilityAssistantResult =
         await ObservabilityAssistant.answerQuestion({
           projectId,
+          incidentId: request.incidentId,
+          alertId: request.alertId,
+          aiRunId,
           // System run — full read access to the project's telemetry.
           props: { isRoot: true },
           feature: request.feature,
@@ -307,8 +346,14 @@ export default class AIInvestigationEngine {
         {
           aiRunId,
           fromStatus: AIRunStatus.Running,
+          expectedAttemptCount: data.attemptCount,
           set: {
             status: AIRunStatus.Completed,
+            ...(request.persistCodeFixRecommendation === true
+              ? {
+                  codeFixRecommendation: AIRunCodeFixRecommendation.Pending,
+                }
+              : {}),
             completedAt: OneUptimeDate.getCurrentDate(),
             lastHeartbeatAt: OneUptimeDate.getCurrentDate(),
             llmCallCount: result.llmCallCount,
@@ -327,13 +372,6 @@ export default class AIInvestigationEngine {
 
       settledAsCompleted = true;
 
-      await this.emitEvent({
-        projectId,
-        aiRunId,
-        sequence: sequence++,
-        eventType: AIRunEventType.RunCompleted,
-      });
-
       const analysis: string = (result.contentInMarkdown || "").trim();
 
       if (!analysis) {
@@ -341,10 +379,24 @@ export default class AIInvestigationEngine {
           `AI: investigation (run ${aiRunId.toString()}) produced no analysis; nothing posted.`,
         );
 
+        if (request.persistCodeFixRecommendation === true) {
+          await this.finalizeCodeFixRecommendation({
+            aiRunId,
+            recommendation: AIRunCodeFixRecommendation.NotRecommended,
+          });
+        }
+
         /*
          * No analysis is still a settled run — the subject must not lose
          * its deferred remediation because the model had nothing to say.
          */
+        await this.emitEvent({
+          projectId,
+          aiRunId,
+          sequence: sequence++,
+          eventType: AIRunEventType.RunCompleted,
+        });
+
         await this.invokeOnSettled(request, aiRunId);
         return;
       }
@@ -363,22 +415,127 @@ export default class AIInvestigationEngine {
        * the WON Completed transition also means a falsely-requeued duplicate
        * attempt can never double-spend it.
        */
-      const confidence: ConfidenceSignal =
-        await AIConfidenceSignal.computeConfidenceSignal({
-          projectId,
-          aiRunId,
-          analysisMarkdown: analysis,
-          evidence: AIConfidenceSignal.evidenceFromCitations(
-            result.citations || [],
-          ),
-        });
+      /*
+       * The display-only TL;DR (see InvestigationTldr) runs CONCURRENTLY with
+       * the classification: both are post-loop calls over the same analysis,
+       * so serializing them would add their latencies to the delay before the
+       * RCA reaches the on-call engineer. Neither call throws — the TL;DR
+       * degrades to null and the report is published regardless.
+       */
+      const [confidence, analysisTldr]: [ConfidenceSignal, string | null] =
+        await Promise.all([
+          AIConfidenceSignal.computeConfidenceSignal({
+            projectId,
+            aiRunId,
+            incidentId: request.incidentId,
+            alertId: request.alertId,
+            analysisMarkdown: analysis,
+            evidence: AIConfidenceSignal.evidenceFromCitations(
+              result.citations || [],
+            ),
+          }),
+          request.persistAnalysisTldr === true
+            ? InvestigationTldr.generateTldr({
+                projectId,
+                aiRunId,
+                incidentId: request.incidentId,
+                alertId: request.alertId,
+                /*
+                 * Summarize the model's own analysis, not the branded form —
+                 * the branding wrapper is boilerplate plus an evidence list,
+                 * and a summary of it would waste the TL;DR on both.
+                 */
+                analysisMarkdown: analysis,
+              }).catch((error: unknown): null => {
+                /*
+                 * generateTldr is total by construction, but it sits in a
+                 * Promise.all inside the main try: a rejection would land in
+                 * the catch below AFTER the Completed transition was won, so
+                 * the run would be failed and the report never posted at all
+                 * — a cosmetic line of text destroying the whole RCA. Belt
+                 * and braces, because the blast radius is that asymmetric.
+                 */
+                logger.error(
+                  `AI: TL;DR generation threw for run ${aiRunId.toString()}; publishing the report without one: ${error}`,
+                );
+                return null;
+              })
+            : Promise.resolve(null),
+        ]);
+
+      /*
+       * Persist the TL;DR BEFORE the report is posted, so the panel can never
+       * render the report for a beat without its summary. Best-effort: a
+       * write failure is logged and the report still posts.
+       */
+      if (analysisTldr) {
+        await this.persistAnalysisTldr({ aiRunId, analysisTldr });
+      }
+
+      /*
+       * Build the posted form exactly once. The same immutable string is
+       * handed to the subject feed, persisted with the recommendation, and
+       * supplied to the automatic fix trigger by the subject runner.
+       */
+      const investigationAnalysisMarkdown: string = this.buildBrandedMarkdown(
+        result,
+        analysis,
+      );
 
       await request.postAnalysis({
-        analysisMarkdown: this.buildBrandedMarkdown(result, analysis),
+        analysisMarkdown: investigationAnalysisMarkdown,
         confidence,
         result,
       });
 
+      /*
+       * Only expose the manual Fix PR action after the cited analysis has
+       * actually been posted. The same structured, fail-closed decision also
+       * gates the automatic FixFromIncident lane in the subject runner.
+       */
+      if (request.persistCodeFixRecommendation === true) {
+        if (AIConfidenceSignal.isCodeFixRecommended(confidence)) {
+          const recommendationPersisted: boolean =
+            await this.finalizeCodeFixRecommendation({
+              aiRunId,
+              recommendation: AIRunCodeFixRecommendation.Recommended,
+              investigationAnalysisMarkdown,
+            });
+
+          if (recommendationPersisted && request.onCodeFixRecommended) {
+            try {
+              await request.onCodeFixRecommended({
+                analysisMarkdown: investigationAnalysisMarkdown,
+              });
+            } catch (error) {
+              logger.error(
+                `AI: post-recommendation follow-up failed for run ${aiRunId.toString()}: ${error}`,
+              );
+            }
+          }
+        } else {
+          await this.finalizeCodeFixRecommendation({
+            aiRunId,
+            recommendation: AIRunCodeFixRecommendation.NotRecommended,
+          });
+        }
+      }
+
+      /*
+       * RunCompleted is the durable publication-settlement signal, not just
+       * the earlier status CAS. The dashboard keeps polling a Completed run
+       * until either the matching report exists or this final event proves
+       * that finalization ended without one. Recommendation settlement and
+       * its post-persistence callback have finished before this event. If
+       * confidence/postAnalysis throws, the catch path emits RunFailed.
+       */
+      await this.emitEvent({
+        projectId,
+        aiRunId,
+        sequence: sequence++,
+        eventType: AIRunEventType.RunCompleted,
+      });
+
       logger.debug(
         `AI: investigation complete (run ${aiRunId.toString()}, confident=${confidence.confident} via ${confidence.source}, ${result.llmCallCount} LLM calls, ${result.toolCallCount} tools, ${result.totalTokens} tokens).`,
       );
@@ -389,12 +546,17 @@ export default class AIInvestigationEngine {
       const message: string =
         error instanceof Error ? error.message : String(error);
 
-      await this.emitEvent({
-        projectId,
-        aiRunId,
-        sequence: sequence++,
-        eventType: AIRunEventType.RunFailed,
-      });
+      /*
+       * A failure after the Running -> Completed transition (classification
+       * or posting the RCA) must not strand the panel in Pending forever or
+       * offer a PR without a posted analysis. Fail closed.
+       */
+      if (settledAsCompleted && request.persistCodeFixRecommendation === true) {
+        await this.finalizeCodeFixRecommendation({
+          aiRunId,
+          recommendation: AIRunCodeFixRecommendation.NotRecommended,
+        });
+      }
 
       /*
        * Hand the failure to the queue's retry policy: transient errors
@@ -423,7 +585,22 @@ export default class AIInvestigationEngine {
        * settled (the retry owns it), and "noop" without a won Completed
        * means another actor moved the run and owns settlement.
        */
-      if (settledAsCompleted || outcome === "finalized") {
+      const ownsTerminalOutcome: boolean =
+        settledAsCompleted || outcome === "finalized";
+
+      if (ownsTerminalOutcome) {
+        /*
+         * Emit terminal failure only after ownership is established. A stale
+         * or requeued attempt can finish after the retry's RunStarted; letting
+         * it append RunFailed would falsely look like the retry settled.
+         */
+        await this.emitEvent({
+          projectId,
+          aiRunId,
+          sequence: sequence++,
+          eventType: AIRunEventType.RunFailed,
+        });
+
         await this.invokeOnSettled(request, aiRunId);
       }
 
@@ -455,6 +632,104 @@ export default class AIInvestigationEngine {
     }
   }
 
+  /*
+   * Store the display-only TL;DR on the run. Unlike the code-fix decision
+   * this authorizes nothing, so it is not retried: a failed write (or a run
+   * that is no longer Completed) simply leaves the panel showing the report
+   * with no summary above it.
+   */
+  private static async persistAnalysisTldr(data: {
+    aiRunId: ObjectID;
+    analysisTldr: string;
+  }): Promise<void> {
+    try {
+      const updatedCount: number =
+        await AIRunService.setInvestigationAnalysisTldr(data);
+
+      if (updatedCount === 0) {
+        logger.warn(
+          `AI: could not store the TL;DR for run ${data.aiRunId.toString()} — it was no longer Completed.`,
+        );
+      }
+    } catch (error) {
+      logger.error(
+        `AI: failed to store the TL;DR for run ${data.aiRunId.toString()}: ${error}`,
+      );
+    }
+  }
+
+  /*
+   * Resolve the recommendation exactly once. Pending is written by the
+   * winning Completed CAS, so this conditional update cannot race a stale
+   * attempt or overwrite a decision another actor already settled. A
+   * Recommended decision carries the exact analysis snapshot it classified;
+   * manual and automatic fix tasks never have to re-read mutable feed state.
+   *
+   * Best-effort like the event trail: transient database errors are retried,
+   * but recommendation persistence never turns a completed investigation
+   * into a failed run. A zero-row update is safe (the row was already settled
+   * or removed) and is logged rather than retried.
+   */
+  private static async finalizeCodeFixRecommendation(
+    data:
+      | {
+          aiRunId: ObjectID;
+          recommendation: AIRunCodeFixRecommendation.Recommended;
+          investigationAnalysisMarkdown: string;
+        }
+      | {
+          aiRunId: ObjectID;
+          recommendation: AIRunCodeFixRecommendation.NotRecommended;
+        },
+  ): Promise<boolean> {
+    for (
+      let attempt: number = 1;
+      attempt <= CODE_FIX_RECOMMENDATION_PERSIST_ATTEMPTS;
+      attempt++
+    ) {
+      try {
+        const updatedCount: number =
+          await AIRunService.finalizeInvestigationCodeFixRecommendation(
+            data.recommendation === AIRunCodeFixRecommendation.Recommended
+              ? {
+                  aiRunId: data.aiRunId,
+                  recommendation: data.recommendation,
+                  taskContext: {
+                    sourceInvestigationRunId: data.aiRunId.toString(),
+                    sourceInvestigationAnalysisMarkdown:
+                      data.investigationAnalysisMarkdown,
+                  },
+                }
+              : {
+                  aiRunId: data.aiRunId,
+                  recommendation: data.recommendation,
+                },
+          );
+
+        if (updatedCount === 0) {
+          logger.warn(
+            `AI: code-fix recommendation for run ${data.aiRunId.toString()} was no longer Pending; leaving the existing decision unchanged.`,
+          );
+          return false;
+        }
+        return true;
+      } catch (error) {
+        if (attempt === CODE_FIX_RECOMMENDATION_PERSIST_ATTEMPTS) {
+          logger.error(
+            `AI: failed to persist code-fix recommendation for run ${data.aiRunId.toString()} after ${attempt} attempts: ${error}`,
+          );
+          return false;
+        }
+
+        logger.warn(
+          `AI: failed to persist code-fix recommendation for run ${data.aiRunId.toString()} (attempt ${attempt}/${CODE_FIX_RECOMMENDATION_PERSIST_ATTEMPTS}); retrying: ${error}`,
+        );
+      }
+    }
+
+    return false;
+  }
+
   // Wrap the agent's analysis with AI branding + an evidence list.
   private static buildBrandedMarkdown(
     result: ObservabilityAssistantResult,
@@ -481,14 +756,19 @@ export default class AIInvestigationEngine {
   }
 
   // Refresh lastHeartbeatAt so a long-running investigation isn't swept as stale.
-  private static async touchHeartbeat(aiRunId: ObjectID): Promise<void> {
+  private static async touchHeartbeat(
+    aiRunId: ObjectID,
+    attemptCount: number,
+  ): Promise<void> {
     try {
-      await AIRunService.updateOneBy({
-        query: { _id: aiRunId.toString(), status: AIRunStatus.Running },
-        data: {
+      await AIRunService.attemptStatusTransition({
+        aiRunId,
+        fromStatus: AIRunStatus.Running,
+        expectedAttemptCount: attemptCount,
+        set: {
+          status: AIRunStatus.Running,
           lastHeartbeatAt: OneUptimeDate.getCurrentDate(),
-        } as never,
-        props: { isRoot: true },
+        },
       });
     } catch (error) {
       logger.error(`AI: heartbeat update failed: ${error}`);

package/Server/Utils/AI/SRE/AlertInvestigationRunner.ts

@@ -169,7 +169,10 @@ export default class AIAlertInvestigationRunner {
       request: {
         // Persisted LlmLog label — owned by AIService (G4 budget list).
         feature: AI_ALERT_INVESTIGATION_FEATURE,
+        alertId,
         contextSummary,
+        persistCodeFixRecommendation: true,
+        persistAnalysisTldr: true,
         postAnalysis: async (postData: {
           analysisMarkdown: string;
           confidence: ConfidenceSignal;
@@ -178,6 +181,7 @@ export default class AIAlertInvestigationRunner {
           await AlertFeedService.createAlertFeedItem({
             alertId,
             projectId,
+            aiRunId,
             alertFeedEventType: AlertFeedEventType.RootCause,
             displayColor: Blue500,
             feedInfoInMarkdown: postData.analysisMarkdown,
@@ -197,7 +201,7 @@ export default class AIAlertInvestigationRunner {
 
           /*
            * Inconclusive means the telemetry was insufficient — for
-           * opted-in projects (Project.enableInstrumentationFixTasks,
+           * opted-in projects (the alert instrumentation-fix setting,
            * default false), queue an ImproveInstrumentation fix task that
            * opens a PR adding the missing observability. Runs strictly
            * AFTER the analysis is posted, and the trigger never throws, so
@@ -218,29 +222,25 @@ export default class AIAlertInvestigationRunner {
               },
             );
           }
-
-          /*
-           * The confident twin: a POSITIVE confident classification means
-           * the posted analysis asserts an evidenced root cause — for
-           * opted-in projects (Project.enableAutomaticCodeFixes, default
-           * false), automatically queue the FixFromIncident task that
-           * opens a draft fix PR from this analysis (the automatic form
-           * of the "Open Fix PR from this analysis" button). Runs strictly
-           * AFTER the analysis is posted because the posted RootCause feed
-           * item IS the fix task's entire context, and the trigger never
-           * throws. Fail direction (G6): the deterministic floor and a
-           * failed classification never auto-open a PR.
-           */
-          if (
-            AIConfidenceSignal.shouldAutoEnqueueCodeFixTask(postData.confidence)
-          ) {
-            await FixFromIncidentTaskTrigger.autoEnqueueFromConfidentInvestigation(
-              {
-                projectId,
-                alertId,
-              },
-            );
-          }
+        },
+        /*
+         * The engine calls this only after it atomically persists Recommended
+         * with the exact posted analysis snapshot. That durable decision is
+         * the prerequisite for the optional automatic FixFromIncident task;
+         * the trigger then applies the alert lane's independent opt-in and
+         * remaining repository, budget and dedupe gates.
+         */
+        onCodeFixRecommended: async (data: {
+          analysisMarkdown: string;
+        }): Promise<void> => {
+          await FixFromIncidentTaskTrigger.autoEnqueueFromRecommendedInvestigation(
+            {
+              projectId,
+              investigationRunId: aiRunId,
+              analysisMarkdown: data.analysisMarkdown,
+              alertId,
+            },
+          );
         },
         /*
          * RCA-first ordering: auto-remediation for this alert was deferred
@@ -338,6 +338,8 @@ export default class AIAlertInvestigationRunner {
             projectId,
             monitorId: alert.monitorId,
             runType: AIRunType.Investigation,
+            triggeredByIncidentId: QueryHelper.isNull(),
+            triggeredByAlertId: QueryHelper.notNull(),
             createdAt: QueryHelper.greaterThan(windowStart),
           },
           props: { isRoot: true },