@oneuptime/common 12.0.4 → 12.0.6

package/Server/Utils/AI/SRE/InstrumentationTaskTrigger.ts

@@ -15,8 +15,8 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  * confidence signal (ConfidenceSignal.ts), never the analysis prose — the
  * telemetry itself is the bug: the code paths involved were not observable
  * enough to diagnose. For projects that opted in
- * (Project.enableInstrumentationFixTasks, default FALSE — G11 posture:
- * autonomous PR creation is opt-in only), the inconclusive analysis becomes
+ * (the subject lane's incident/alert instrumentation setting, default FALSE
+ * — G11 posture: autonomous PR creation is opt-in only), the inconclusive analysis becomes
  * the input to a CodeFix AIRun (codeFixTaskType: ImproveInstrumentation)
  * that the agent worker turns into a pull request adding the missing
  * observability — so the NEXT investigation of a similar signal can reach a
@@ -28,8 +28,11 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  */
 
 export interface InstrumentationTaskGateInput {
-  // The project row with enableAi + enableInstrumentationFixTasks selected.
+  // The project row with enableAi + the subject lane's opt-in selected.
   project: Project | null;
+  // Exactly one subject selects the incident or alert opt-in.
+  incidentId?: ObjectID | undefined;
+  alertId?: ObjectID | undefined;
   // Whether the project has at least one GitHub-App-connected repository.
   hasConnectedRepository: boolean;
   // A non-terminal ImproveInstrumentation run for the same subject, if any.
@@ -63,10 +66,21 @@ export default class InstrumentationTaskTrigger {
      * Strict opt-in — the column defaults to false, so unset/legacy rows
      * never enqueue. Autonomous PR creation must never be default-on.
      */
-    if (input.project.enableInstrumentationFixTasks !== true) {
+    if (Boolean(input.incidentId) === Boolean(input.alertId)) {
       return {
         enqueue: false,
-        reason: "project has not opted in to instrumentation fix tasks",
+        reason: "exactly one incident or alert subject is required",
+      };
+    }
+
+    const instrumentationFixTasksEnabled: boolean = input.incidentId
+      ? input.project.enableIncidentInstrumentationFixTasks === true
+      : input.project.enableAlertInstrumentationFixTasks === true;
+
+    if (!instrumentationFixTasksEnabled) {
+      return {
+        enqueue: false,
+        reason: `project has not opted in to ${input.incidentId ? "incident" : "alert"} instrumentation fix tasks`,
       };
     }
 
@@ -112,16 +126,21 @@ export default class InstrumentationTaskTrigger {
     const { projectId } = data;
 
     try {
-      if (!data.incidentId && !data.alertId) {
+      if (Boolean(data.incidentId) === Boolean(data.alertId)) {
         return;
       }
 
       const project: Project | null = await ProjectService.findOneById({
         id: projectId,
-        select: {
-          enableAi: true,
-          enableInstrumentationFixTasks: true,
-        },
+        select: data.incidentId
+          ? {
+              enableAi: true,
+              enableIncidentInstrumentationFixTasks: true,
+            }
+          : {
+              enableAi: true,
+              enableAlertInstrumentationFixTasks: true,
+            },
         props: { isRoot: true },
       });
 
@@ -132,6 +151,8 @@ export default class InstrumentationTaskTrigger {
       const optInDecision: InstrumentationTaskGateDecision =
         this.shouldEnqueueInstrumentationTask({
           project,
+          incidentId: data.incidentId,
+          alertId: data.alertId,
           hasConnectedRepository: true,
           existingRun: null,
         });
@@ -150,12 +171,23 @@ export default class InstrumentationTaskTrigger {
        * throw the same rejection, but that lands in the catch below as an
        * error log; a budget skip is expected behavior, not a failure).
        */
-      const budget: FixRunBudgetDecision =
-        await FixRunBudget.getBudgetStatus(projectId);
+      const budget: FixRunBudgetDecision = await FixRunBudget.getBudgetStatus(
+        projectId,
+        {
+          incidentId: data.incidentId,
+          alertId: data.alertId,
+        },
+      );
 
       if (!budget.allowed) {
         logger.debug(
-          `AI: not enqueueing instrumentation task for project ${projectId.toString()} — ${FixRunBudget.describeRejection(budget)}`,
+          `AI: not enqueueing instrumentation task for project ${projectId.toString()} — ${FixRunBudget.describeRejection(
+            budget,
+            {
+              incidentId: data.incidentId,
+              alertId: data.alertId,
+            },
+          )}`,
         );
         return;
       }
@@ -177,6 +209,8 @@ export default class InstrumentationTaskTrigger {
       const decision: InstrumentationTaskGateDecision =
         this.shouldEnqueueInstrumentationTask({
           project,
+          incidentId: data.incidentId,
+          alertId: data.alertId,
           hasConnectedRepository,
           existingRun,
         });

package/Server/Utils/AI/SRE/InvestigationGrader.ts

@@ -146,6 +146,7 @@ export default class InvestigationGrader {
         select: {
           _id: true,
           autoGrade: true,
+          completedAt: true,
         },
         sort: { createdAt: SortOrder.Descending },
         props: { isRoot: true },
@@ -175,13 +176,16 @@ export default class InvestigationGrader {
       }
 
       /*
-       * The posted analysis: the latest RootCause feed item — the AI's
-       * postAnalysis is its only writer, and it is the only persisted copy
-       * of the analysis (the same source the fix recipes and the remediation
-       * planner read, through this one helper).
+       * The posted analysis owned by this exact run. A newer investigation or
+       * a human RootCause item may exist by resolution time; neither may be
+       * graded as though it came from the selected investigation.
        */
       const analysisMarkdown: string | null =
-        await PostedRootCause.getForIncident(incidentId);
+        await PostedRootCause.getForInvestigation({
+          incidentId,
+          aiRunId: run.id!,
+          runCompletedAt: run.completedAt,
+        });
 
       if (!analysisMarkdown) {
         logger.debug(

package/Server/Utils/AI/SRE/InvestigationQueue.ts

@@ -1,7 +1,9 @@
 import ObjectID from "../../../../Types/ObjectID";
 import OneUptimeDate from "../../../../Types/Date";
+import Query from "../../../../Types/BaseDatabase/Query";
 import AIRunType from "../../../../Types/AI/AIRunType";
 import AIRunStatus from "../../../../Types/AI/AIRunStatus";
+import AIRunCodeFixRecommendation from "../../../../Types/AI/AIRunCodeFixRecommendation";
 import AIRun from "../../../../Models/DatabaseModels/AIRun";
 import Project from "../../../../Models/DatabaseModels/Project";
 import AIRunService from "../../../Services/AIRunService";
@@ -11,6 +13,7 @@ import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
 import QueryHelper from "../../../Types/Database/QueryHelper";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
+import InvestigationSubjectLock from "./InvestigationSubjectLock";
 
 /*
  * AI SRE — the durable investigation queue (Phase 2's first item; Q1
@@ -47,11 +50,11 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
 export const MAX_INVESTIGATION_ATTEMPTS: number = 2;
 
 /*
- * G4 cost guardrail: at most this many investigations may be Running per
- * project at once (per-project override in
- * Project.aiMaxConcurrentInvestigations). Enforced at CLAIM time, so a storm
- * queues (bounded by dedupe + severity gates + this TTL) and drains at cap
- * rate instead of being dropped.
+ * G4 cost guardrail: at most this many investigations may be Running in an
+ * incident, alert or subjectless lane at once. Each lane has its own project
+ * override; subjectless work keeps Project.aiMaxConcurrentInvestigations.
+ * Enforced at CLAIM time, so a storm queues (bounded by dedupe + severity
+ * gates + this TTL) and drains at cap rate instead of being dropped.
  */
 export const DEFAULT_MAX_CONCURRENT_INVESTIGATIONS: number = 3;
 /*
@@ -62,7 +65,7 @@ const MIN_CONCURRENT_INVESTIGATIONS: number = 1;
 const MAX_CONCURRENT_INVESTIGATIONS: number = 25;
 
 /*
- * Lane priority. Two kinds of run share this queue and the per-project
+ * Lane priority. Two kinds of run share this queue and each subject lane's
  * concurrency cap above:
  *   - the INTERACTIVE lane (incident/alert RCA) — a human is waiting;
  *   - the PREVENTIVE lane (AI-insight triage, identified by
@@ -143,6 +146,13 @@ export default class AIInvestigationQueue {
   }): Promise<ObjectID | null> {
     const { projectId } = data;
 
+    if (data.subjectIncidentId && data.subjectAlertId) {
+      logger.error(
+        `AI: not enqueueing investigation for project ${projectId.toString()} — a run cannot belong to both an incident and an alert.`,
+      );
+      return null;
+    }
+
     /*
      * Budget quiet-skip at enqueue: when the daily budget is already
      * exhausted there is no point recording intent that the TTL would
@@ -150,7 +160,10 @@ export default class AIInvestigationQueue {
      */
     try {
       const budget: AutonomousBudgetStatus =
-        await AIService.getAutonomousDailyBudgetStatus(projectId);
+        await AIService.getAutonomousDailyBudgetStatus(projectId, {
+          incidentId: data.subjectIncidentId,
+          alertId: data.subjectAlertId,
+        });
 
       if (budget.exhausted) {
         logger.debug(
@@ -173,6 +186,13 @@ export default class AIInvestigationQueue {
     run.projectId = projectId;
     run.runType = runType;
     run.status = AIRunStatus.Queued;
+    /*
+     * Queued and Running rows must never advertise an unresolved code-fix
+     * decision. The engine atomically moves eligible incident/alert runs to
+     * Pending only when it wins the Running -> Completed transition; until
+     * then the fail-closed default is NotRecommended.
+     */
+    run.codeFixRecommendation = AIRunCodeFixRecommendation.NotRecommended;
 
     if (data.subjectIncidentId) {
       run.triggeredByIncidentId = data.subjectIncidentId;
@@ -193,10 +213,32 @@ export default class AIInvestigationQueue {
 
     let createdRun: AIRun;
     try {
-      createdRun = await AIRunService.create({
-        data: run,
-        props: { isRoot: true },
-      });
+      const createRun: () => Promise<AIRun> = async (): Promise<AIRun> => {
+        return AIRunService.create({
+          data: run,
+          props: { isRoot: true },
+        });
+      };
+
+      /*
+       * Linearize subject Investigation creation with FixFromIncident's
+       * final latest-source validation and INSERT. Whichever writer gets the
+       * shared lock first becomes authoritative: a fix inserted first is
+       * valid at that instant; an Investigation inserted first is observed by
+       * the fix gate and makes the stale request fail closed.
+       */
+      createdRun =
+        runType === AIRunType.Investigation &&
+        (data.subjectIncidentId || data.subjectAlertId)
+          ? await InvestigationSubjectLock.runExclusive(
+              {
+                projectId,
+                incidentId: data.subjectIncidentId,
+                alertId: data.subjectAlertId,
+              },
+              createRun,
+            )
+          : await createRun();
     } catch (error) {
       logger.error(`AI: failed to enqueue investigation run: ${error}`);
       return null;
@@ -291,6 +333,7 @@ export default class AIInvestigationQueue {
         fromStatus: AIRunStatus.Queued,
         set: {
           status: AIRunStatus.Cancelled,
+          codeFixRecommendation: AIRunCodeFixRecommendation.NotRecommended,
           completedAt: OneUptimeDate.getCurrentDate(),
           errorMessage: `Expired in the investigation queue after ${QUEUE_TTL_MINUTES} minutes — a first-pass analysis this late would no longer be useful. The project may have been at its concurrency cap or daily token budget.`,
         },
@@ -404,8 +447,10 @@ export default class AIInvestigationQueue {
       const requeued: number = await AIRunService.attemptStatusTransition({
         aiRunId: data.aiRunId,
         fromStatus: AIRunStatus.Running,
+        expectedAttemptCount: data.attemptCount,
         set: {
           status: AIRunStatus.Queued,
+          codeFixRecommendation: AIRunCodeFixRecommendation.NotRecommended,
           errorMessage: `Attempt ${data.attemptCount} failed and the run was requeued: ${truncatedMessage}`,
         },
       });
@@ -422,8 +467,10 @@ export default class AIInvestigationQueue {
     const finalized: number = await AIRunService.attemptStatusTransition({
       aiRunId: data.aiRunId,
       fromStatus: AIRunStatus.Running,
+      expectedAttemptCount: data.attemptCount,
       set: {
         status: AIRunStatus.Error,
+        codeFixRecommendation: AIRunCodeFixRecommendation.NotRecommended,
         completedAt: OneUptimeDate.getCurrentDate(),
         errorMessage: truncatedMessage,
       },
@@ -448,17 +495,21 @@ export default class AIInvestigationQueue {
   public static async requeueOrMarkStale(run: {
     id: ObjectID;
     attemptCount: number;
+    lastHeartbeatAt?: Date | undefined;
     runType?: AIRunType | undefined;
     projectId?: ObjectID | undefined;
     triggeredByIncidentId?: ObjectID | undefined;
     triggeredByAlertId?: ObjectID | undefined;
-  }): Promise<"requeued" | "stale"> {
+  }): Promise<"requeued" | "stale" | "noop"> {
     if ((run.attemptCount || 0) < MAX_INVESTIGATION_ATTEMPTS) {
       const requeued: number = await AIRunService.attemptStatusTransition({
         aiRunId: run.id,
         fromStatus: AIRunStatus.Running,
+        expectedAttemptCount: run.attemptCount,
+        expectedLastHeartbeatAt: run.lastHeartbeatAt,
         set: {
           status: AIRunStatus.Queued,
+          codeFixRecommendation: AIRunCodeFixRecommendation.NotRecommended,
           errorMessage: `Attempt ${run.attemptCount} stopped reporting progress (the server processing it may have restarted) and the run was requeued.`,
         },
       });
@@ -471,8 +522,11 @@ export default class AIInvestigationQueue {
     const staled: number = await AIRunService.attemptStatusTransition({
       aiRunId: run.id,
       fromStatus: AIRunStatus.Running,
+      expectedAttemptCount: run.attemptCount,
+      expectedLastHeartbeatAt: run.lastHeartbeatAt,
       set: {
         status: AIRunStatus.Stale,
+        codeFixRecommendation: AIRunCodeFixRecommendation.NotRecommended,
         completedAt: OneUptimeDate.getCurrentDate(),
         errorMessage:
           "The run stopped reporting progress and was marked as stale after exhausting its retry attempts. The server processing it may have restarted.",
@@ -485,9 +539,10 @@ export default class AIInvestigationQueue {
      */
     if (staled > 0) {
       await this.handOffSettledInvestigationToRemediation(run);
+      return "stale";
     }
 
-    return "stale";
+    return "noop";
   }
 
   /*
@@ -544,26 +599,50 @@ export default class AIInvestigationQueue {
    * errors.
    */
   private static async passesClaimGates(run: QueuedRunRef): Promise<boolean> {
-    // Per-project cap override, defaulting to 3 and clamped to [1, 25].
+    if (run.triggeredByIncidentId && run.triggeredByAlertId) {
+      logger.error(
+        `AI: leaving run ${run.id.toString()} queued — a run cannot belong to both an incident and an alert.`,
+      );
+      return false;
+    }
+
+    /*
+     * Each subject lane owns its own concurrency pool. Insight triage and any
+     * future subjectless queue work retain the legacy project-wide fallback.
+     */
     const project: Project | null = await ProjectService.findOneById({
       id: run.projectId,
-      select: { aiMaxConcurrentInvestigations: true },
+      select: {
+        aiMaxConcurrentInvestigations: true,
+        incidentAiMaxConcurrentInvestigations: true,
+        alertAiMaxConcurrentInvestigations: true,
+      },
       props: { isRoot: true },
     });
 
+    let configuredConcurrencyCap: number | undefined =
+      project?.aiMaxConcurrentInvestigations;
+
+    if (run.triggeredByIncidentId) {
+      configuredConcurrencyCap = project?.incidentAiMaxConcurrentInvestigations;
+    } else if (run.triggeredByAlertId) {
+      configuredConcurrencyCap = project?.alertAiMaxConcurrentInvestigations;
+    }
+
     const concurrencyCap: number = Math.min(
       MAX_CONCURRENT_INVESTIGATIONS,
       Math.max(
         MIN_CONCURRENT_INVESTIGATIONS,
-        project?.aiMaxConcurrentInvestigations ??
-          DEFAULT_MAX_CONCURRENT_INVESTIGATIONS,
+        configuredConcurrencyCap ?? DEFAULT_MAX_CONCURRENT_INVESTIGATIONS,
       ),
     );
 
+    const subjectLaneQuery: Query<AIRun> = this.getSubjectLaneQuery(run);
+
     /*
      * Investigations, remediation plans and remediation executions share
-     * the same per-project cap — all are short and storm-shaped on
-     * incident/alert creation, so one pool keeps the cost model simple.
+     * the cap inside their incident, alert or subjectless lane. Work in one
+     * lane never consumes another lane's concurrency slots.
      */
     const runningCount: number = (
       await AIRunService.countBy({
@@ -575,6 +654,7 @@ export default class AIInvestigationQueue {
             AIRunType.RemediationExecution,
           ]),
           status: AIRunStatus.Running,
+          ...subjectLaneQuery,
         },
         props: { isRoot: true },
       })
@@ -595,10 +675,10 @@ export default class AIInvestigationQueue {
      * kinds share ONE combined sub-cap, so together they always leave the
      * reserved slot reachable. Floored at 1 so a project running at the
      * minimum cap of 1 still drains its background work (slower, one at a
-     * time) instead of deadlocking the lane entirely — the interactive lane
-     * keeps its priority through the global cap check above, which a single
-     * running background run cannot outlast for long (both kinds are
-     * read-only and short).
+     * time) instead of deadlocking the lane entirely — foreground work keeps
+     * its priority through the lane cap check above, which a single running
+     * background run cannot outlast for long (both kinds are read-only and
+     * short).
      */
     const isBackgroundLane: boolean =
       Boolean(run.triggeredByAiInsightId) ||
@@ -622,6 +702,7 @@ export default class AIInvestigationQueue {
             runType: AIRunType.Investigation,
             status: AIRunStatus.Running,
             triggeredByAiInsightId: QueryHelper.notNull(),
+            ...subjectLaneQuery,
           },
           props: { isRoot: true },
         })
@@ -636,6 +717,7 @@ export default class AIInvestigationQueue {
               AIRunType.RemediationExecution,
             ]),
             status: AIRunStatus.Running,
+            ...subjectLaneQuery,
           },
           props: { isRoot: true },
         })
@@ -653,7 +735,10 @@ export default class AIInvestigationQueue {
     }
 
     const budget: AutonomousBudgetStatus =
-      await AIService.getAutonomousDailyBudgetStatus(run.projectId);
+      await AIService.getAutonomousDailyBudgetStatus(run.projectId, {
+        incidentId: run.triggeredByIncidentId,
+        alertId: run.triggeredByAlertId,
+      });
 
     if (budget.exhausted) {
       logger.debug(
@@ -665,6 +750,35 @@ export default class AIInvestigationQueue {
     return true;
   }
 
+  /*
+   * Scope queue counts to one independent lane. The concrete subject id is
+   * deliberately not compared: all incidents share the incident cap, all
+   * alerts share the alert cap, and subjectless work must have neither link.
+   */
+  private static getSubjectLaneQuery(run: {
+    triggeredByIncidentId?: ObjectID | undefined;
+    triggeredByAlertId?: ObjectID | undefined;
+  }): Query<AIRun> {
+    if (run.triggeredByIncidentId) {
+      return {
+        triggeredByIncidentId: QueryHelper.notNull(),
+        triggeredByAlertId: QueryHelper.isNull(),
+      };
+    }
+
+    if (run.triggeredByAlertId) {
+      return {
+        triggeredByIncidentId: QueryHelper.isNull(),
+        triggeredByAlertId: QueryHelper.notNull(),
+      };
+    }
+
+    return {
+      triggeredByIncidentId: QueryHelper.isNull(),
+      triggeredByAlertId: QueryHelper.isNull(),
+    };
+  }
+
   /*
    * The atomic claim: Queued -> Running, guarded on the attemptCount the
    * caller observed, so a stale queue snapshot can neither double-claim
@@ -709,6 +823,7 @@ export default class AIInvestigationQueue {
         await AIRunService.attemptStatusTransition({
           aiRunId: run.id,
           fromStatus: AIRunStatus.Running,
+          expectedAttemptCount: attempt,
           set: {
             status: AIRunStatus.Error,
             completedAt: OneUptimeDate.getCurrentDate(),
@@ -737,6 +852,7 @@ export default class AIInvestigationQueue {
         await AIRunService.attemptStatusTransition({
           aiRunId: run.id,
           fromStatus: AIRunStatus.Running,
+          expectedAttemptCount: attempt,
           set: {
             status: AIRunStatus.Error,
             completedAt: OneUptimeDate.getCurrentDate(),
@@ -817,6 +933,7 @@ export default class AIInvestigationQueue {
     await AIRunService.attemptStatusTransition({
       aiRunId: run.id,
       fromStatus: AIRunStatus.Running,
+      expectedAttemptCount: attempt,
       set: {
         status: AIRunStatus.Error,
         completedAt: OneUptimeDate.getCurrentDate(),

package/Server/Utils/AI/SRE/InvestigationSubjectLock.ts

@@ -0,0 +1,66 @@
+import ObjectID from "../../../../Types/ObjectID";
+import BadDataException from "../../../../Types/Exception/BadDataException";
+import Semaphore, { SemaphoreMutex } from "../../../Infrastructure/Semaphore";
+import logger from "../../Logger";
+
+/*
+ * Investigation creation and FixFromIncident authorization must share one
+ * linearization point. Without it, a new Investigation can be inserted after
+ * a fix trigger's "latest run" read but before the CodeFix INSERT, allowing a
+ * task grounded in an analysis that is no longer authoritative.
+ */
+export const INVESTIGATION_SUBJECT_LOCK_NAMESPACE: string =
+  "InvestigationSubjectLifecycle";
+
+export interface InvestigationSubjectLockData {
+  projectId: ObjectID;
+  incidentId?: ObjectID | undefined;
+  alertId?: ObjectID | undefined;
+}
+
+export function getInvestigationSubjectLockKey(
+  data: InvestigationSubjectLockData,
+): string {
+  if (Boolean(data.incidentId) === Boolean(data.alertId)) {
+    throw new BadDataException(
+      "Exactly one incident or alert subject is required.",
+    );
+  }
+
+  const subjectKey: string = data.incidentId
+    ? `incident-${data.incidentId.toString()}`
+    : `alert-${data.alertId!.toString()}`;
+
+  return `${data.projectId.toString()}-${subjectKey}`;
+}
+
+export default class InvestigationSubjectLock {
+  public static async runExclusive<T>(
+    data: InvestigationSubjectLockData,
+    operation: () => Promise<T>,
+  ): Promise<T> {
+    const key: string = getInvestigationSubjectLockKey(data);
+    const mutex: SemaphoreMutex = await Semaphore.lock({
+      key,
+      namespace: INVESTIGATION_SUBJECT_LOCK_NAMESPACE,
+      lockTimeout: 30 * 1000,
+      acquireTimeout: 10 * 1000,
+    });
+
+    try {
+      return await operation();
+    } finally {
+      try {
+        await Semaphore.release(mutex);
+      } catch (error) {
+        /*
+         * The Redis mutex expires automatically. Do not hide the operation's
+         * result (or its more useful error) when best-effort release fails.
+         */
+        logger.error(
+          `AI: failed to release the investigation subject lock ${key}; it will expire automatically: ${error}`,
+        );
+      }
+    }
+  }
+}