@oneuptime/common 11.7.4 → 12.0.0

package/Server/Utils/AI/SRE/InvestigationQueue.ts

@@ -99,9 +99,11 @@ export interface QueuedRunRef {
   id: ObjectID;
   projectId: ObjectID;
   attemptCount: number;
+  runType?: AIRunType | undefined;
   triggeredByIncidentId?: ObjectID | undefined;
   triggeredByAlertId?: ObjectID | undefined;
   triggeredByAiInsightId?: ObjectID | undefined;
+  triggeredByAutoRemediationSuggestionId?: ObjectID | undefined;
 }
 
 export default class AIInvestigationQueue {
@@ -122,6 +124,22 @@ export default class AIInvestigationQueue {
     subjectAlertId?: ObjectID | undefined;
     subjectMonitorId?: ObjectID | undefined;
     subjectAIInsightId?: ObjectID | undefined;
+    /*
+     * When set, the run is a remediation run (not an Investigation): by
+     * default a read-only RemediationPlan that picks a runbook for this
+     * suggestion, or a RemediationExecution when remediationRunType says so.
+     * dispatch() routes on runType FIRST, so the incident/alert subject ids
+     * above may still be set for dashboard linkage.
+     */
+    subjectAutoRemediationSuggestionId?: ObjectID | undefined;
+    /*
+     * Only meaningful with subjectAutoRemediationSuggestionId: which kind
+     * of remediation run to enqueue. Defaults to RemediationPlan.
+     */
+    remediationRunType?:
+      | AIRunType.RemediationPlan
+      | AIRunType.RemediationExecution
+      | undefined;
   }): Promise<ObjectID | null> {
     const { projectId } = data;
 
@@ -147,9 +165,13 @@ export default class AIInvestigationQueue {
       return null;
     }
 
+    const runType: AIRunType = data.subjectAutoRemediationSuggestionId
+      ? data.remediationRunType || AIRunType.RemediationPlan
+      : AIRunType.Investigation;
+
     const run: AIRun = new AIRun();
     run.projectId = projectId;
-    run.runType = AIRunType.Investigation;
+    run.runType = runType;
     run.status = AIRunStatus.Queued;
 
     if (data.subjectIncidentId) {
@@ -164,6 +186,10 @@ export default class AIInvestigationQueue {
     if (data.subjectAIInsightId) {
       run.triggeredByAiInsightId = data.subjectAIInsightId;
     }
+    if (data.subjectAutoRemediationSuggestionId) {
+      run.triggeredByAutoRemediationSuggestionId =
+        data.subjectAutoRemediationSuggestionId;
+    }
 
     let createdRun: AIRun;
     try {
@@ -181,9 +207,12 @@ export default class AIInvestigationQueue {
       id: createdRun.id!,
       projectId,
       attemptCount: 0,
+      runType,
       triggeredByIncidentId: data.subjectIncidentId,
       triggeredByAlertId: data.subjectAlertId,
       triggeredByAiInsightId: data.subjectAIInsightId,
+      triggeredByAutoRemediationSuggestionId:
+        data.subjectAutoRemediationSuggestionId,
     }).catch((error: Error) => {
       logger.error(
         `AI: inline processing of queued run ${createdRun.id?.toString()} failed: ${error}`,
@@ -236,18 +265,28 @@ export default class AIInvestigationQueue {
 
     const expiredRuns: Array<AIRun> = await AIRunService.findBy({
       query: {
-        runType: AIRunType.Investigation,
+        runType: QueryHelper.any([
+          AIRunType.Investigation,
+          AIRunType.RemediationPlan,
+          AIRunType.RemediationExecution,
+        ]),
         status: AIRunStatus.Queued,
         createdAt: QueryHelper.lessThan(expiryThreshold),
       },
-      select: { _id: true },
+      select: {
+        _id: true,
+        projectId: true,
+        runType: true,
+        triggeredByIncidentId: true,
+        triggeredByAlertId: true,
+      },
       limit: 100,
       skip: 0,
       props: { isRoot: true },
     });
 
     for (const expired of expiredRuns) {
-      await AIRunService.attemptStatusTransition({
+      const expiredCount: number = await AIRunService.attemptStatusTransition({
         aiRunId: expired.id!,
         fromStatus: AIRunStatus.Queued,
         set: {
@@ -256,20 +295,42 @@ export default class AIInvestigationQueue {
           errorMessage: `Expired in the investigation queue after ${QUEUE_TTL_MINUTES} minutes — a first-pass analysis this late would no longer be useful. The project may have been at its concurrency cap or daily token budget.`,
         },
       });
+
+      /*
+       * An expired incident/alert investigation is terminally settled —
+       * release the subject to auto-remediation (RCA-first ordering must
+       * not swallow remediation when the RCA never ran). Guarded on the
+       * WON transition so a run claimed between the query and this
+       * transition is settled by its executor instead.
+       */
+      if (expiredCount > 0) {
+        await this.handOffSettledInvestigationToRemediation({
+          runType: expired.runType,
+          projectId: expired.projectId,
+          triggeredByIncidentId: expired.triggeredByIncidentId,
+          triggeredByAlertId: expired.triggeredByAlertId,
+        });
+      }
     }
 
     const queuedRuns: Array<AIRun> = await AIRunService.findBy({
       query: {
-        runType: AIRunType.Investigation,
+        runType: QueryHelper.any([
+          AIRunType.Investigation,
+          AIRunType.RemediationPlan,
+          AIRunType.RemediationExecution,
+        ]),
         status: AIRunStatus.Queued,
       },
       select: {
         _id: true,
         projectId: true,
         attemptCount: true,
+        runType: true,
         triggeredByIncidentId: true,
         triggeredByAlertId: true,
         triggeredByAiInsightId: true,
+        triggeredByAutoRemediationSuggestionId: true,
       },
       sort: { createdAt: SortOrder.Ascending },
       limit: POLLER_BATCH_SIZE,
@@ -282,9 +343,12 @@ export default class AIInvestigationQueue {
         id: queued.id!,
         projectId: queued.projectId!,
         attemptCount: queued.attemptCount || 0,
+        runType: queued.runType,
         triggeredByIncidentId: queued.triggeredByIncidentId,
         triggeredByAlertId: queued.triggeredByAlertId,
         triggeredByAiInsightId: queued.triggeredByAiInsightId,
+        triggeredByAutoRemediationSuggestionId:
+          queued.triggeredByAutoRemediationSuggestionId,
       };
 
       try {
@@ -320,6 +384,12 @@ export default class AIInvestigationQueue {
    * remain, otherwise mark the run Error. The transition guards on Running
    * so a run that already completed (e.g. only postAnalysis failed) is
    * never clobbered or re-run.
+   *
+   * Returns what actually happened, so callers can settle post-terminal
+   * work (the remediation hand-off) exactly once: "requeued" — a later
+   * attempt owns the run now; "finalized" — THIS call moved the run to
+   * Error, the run is settled; "noop" — another actor moved the run first
+   * (that actor owns settlement).
    */
   @CaptureSpan()
   public static async failOrRequeue(data: {
@@ -327,7 +397,7 @@ export default class AIInvestigationQueue {
     attemptCount: number;
     errorMessage: string;
     isPermanent: boolean;
-  }): Promise<void> {
+  }): Promise<"requeued" | "finalized" | "noop"> {
     const truncatedMessage: string = data.errorMessage.substring(0, 400);
 
     if (!data.isPermanent && data.attemptCount < MAX_INVESTIGATION_ATTEMPTS) {
@@ -344,11 +414,12 @@ export default class AIInvestigationQueue {
         logger.debug(
           `AI: requeued run ${data.aiRunId.toString()} after attempt ${data.attemptCount} failed.`,
         );
+        return "requeued";
       }
-      return;
+      return "noop";
     }
 
-    await AIRunService.attemptStatusTransition({
+    const finalized: number = await AIRunService.attemptStatusTransition({
       aiRunId: data.aiRunId,
       fromStatus: AIRunStatus.Running,
       set: {
@@ -357,17 +428,30 @@ export default class AIInvestigationQueue {
         errorMessage: truncatedMessage,
       },
     });
+
+    return finalized > 0 ? "finalized" : "noop";
   }
 
   /*
    * Called by the stale-run sweeper for an Investigation run whose
    * heartbeat went silent (the pod running it died). Requeues while
    * attempts remain; otherwise marks it Stale as before.
+   *
+   * The optional subject fields let the sweeper hand a terminally-staled
+   * incident/alert investigation to auto-remediation: remediation is
+   * deferred until the investigation settles (RCA-first ordering), so a
+   * run that dies without retries left must still release the subject to
+   * the rule engine — otherwise a dead investigation would silently
+   * swallow remediation for that incident/alert.
    */
   @CaptureSpan()
   public static async requeueOrMarkStale(run: {
     id: ObjectID;
     attemptCount: number;
+    runType?: AIRunType | undefined;
+    projectId?: ObjectID | undefined;
+    triggeredByIncidentId?: ObjectID | undefined;
+    triggeredByAlertId?: ObjectID | undefined;
   }): Promise<"requeued" | "stale"> {
     if ((run.attemptCount || 0) < MAX_INVESTIGATION_ATTEMPTS) {
       const requeued: number = await AIRunService.attemptStatusTransition({
@@ -384,7 +468,7 @@ export default class AIInvestigationQueue {
       }
     }
 
-    await AIRunService.attemptStatusTransition({
+    const staled: number = await AIRunService.attemptStatusTransition({
       aiRunId: run.id,
       fromStatus: AIRunStatus.Running,
       set: {
@@ -395,9 +479,64 @@ export default class AIInvestigationQueue {
       },
     });
 
+    /*
+     * Only the actor that WON the terminal transition may settle — a lost
+     * transition means the run moved on and another actor owns settlement.
+     */
+    if (staled > 0) {
+      await this.handOffSettledInvestigationToRemediation(run);
+    }
+
     return "stale";
   }
 
+  /*
+   * Release a terminally-settled incident/alert investigation to the
+   * auto-remediation rule engine (the RCA-first ordering's terminal
+   * fallback for runs that expired or went stale — the happy path hands
+   * off from the engine via InvestigationRequest.onSettled). Only fires
+   * for Investigation runs with an incident/alert subject; remediation
+   * runs and insight triage never hand off. Never throws.
+   */
+  private static async handOffSettledInvestigationToRemediation(run: {
+    runType?: AIRunType | undefined;
+    projectId?: ObjectID | undefined;
+    triggeredByIncidentId?: ObjectID | undefined;
+    triggeredByAlertId?: ObjectID | undefined;
+  }): Promise<void> {
+    if (run.runType !== AIRunType.Investigation) {
+      return;
+    }
+
+    if (
+      !run.projectId ||
+      (!run.triggeredByIncidentId && !run.triggeredByAlertId)
+    ) {
+      return;
+    }
+
+    try {
+      /*
+       * Lazy require: RemediationHandoff imports the rule engine, which
+       * imports this queue — a top-level import here would be circular at
+       * module-init time (same pattern as the runner dispatch below).
+       */
+      const remediationHandoff: typeof import("./RemediationHandoff").default =
+        // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+        require("./RemediationHandoff").default;
+
+      await remediationHandoff.runForSettledInvestigation({
+        projectId: run.projectId,
+        incidentId: run.triggeredByIncidentId,
+        alertId: run.triggeredByAlertId,
+      });
+    } catch (error) {
+      logger.error(
+        `AI: remediation hand-off after a settled investigation failed: ${error}`,
+      );
+    }
+  }
+
   /*
    * The claim-time cost gates: concurrency cap, the preventive-lane sub-cap
    * and the daily budget. A run failing these stays Queued — the poller
@@ -421,11 +560,20 @@ export default class AIInvestigationQueue {
       ),
     );
 
+    /*
+     * Investigations, remediation plans and remediation executions share
+     * the same per-project cap — all are short and storm-shaped on
+     * incident/alert creation, so one pool keeps the cost model simple.
+     */
     const runningCount: number = (
       await AIRunService.countBy({
         query: {
           projectId: run.projectId,
-          runType: AIRunType.Investigation,
+          runType: QueryHelper.any([
+            AIRunType.Investigation,
+            AIRunType.RemediationPlan,
+            AIRunType.RemediationExecution,
+          ]),
           status: AIRunStatus.Running,
         },
         props: { isRoot: true },
@@ -440,21 +588,33 @@ export default class AIInvestigationQueue {
     }
 
     /*
-     * The preventive lane's sub-cap: insight triage may never occupy the
-     * slot(s) reserved for incident/alert RCA (see
-     * INSIGHT_TRIAGE_RESERVED_SLOTS). Floored at 1 so a project running at
-     * the minimum cap of 1 still triages its insights (slower, one at a
+     * The background lane's sub-cap: insight triage AND remediation plans
+     * may never occupy the slot(s) reserved for incident/alert RCA (see
+     * INSIGHT_TRIAGE_RESERVED_SLOTS) — the RCA is what a paged human is
+     * waiting for, and both background kinds are storm-shaped. The two
+     * kinds share ONE combined sub-cap, so together they always leave the
+     * reserved slot reachable. Floored at 1 so a project running at the
+     * minimum cap of 1 still drains its background work (slower, one at a
      * time) instead of deadlocking the lane entirely — the interactive lane
      * keeps its priority through the global cap check above, which a single
-     * running triage run cannot outlast for long (triage is read-only and
-     * short).
+     * running background run cannot outlast for long (both kinds are
+     * read-only and short).
      */
-    if (run.triggeredByAiInsightId) {
-      const triageCap: number = Math.max(
+    const isBackgroundLane: boolean =
+      Boolean(run.triggeredByAiInsightId) ||
+      run.runType === AIRunType.RemediationPlan ||
+      run.runType === AIRunType.RemediationExecution;
+
+    if (isBackgroundLane) {
+      const backgroundCap: number = Math.max(
         1,
         concurrencyCap - INSIGHT_TRIAGE_RESERVED_SLOTS,
       );
 
+      /*
+       * Triage runs are Investigations with an insight subject; plan runs
+       * are their own runType — the two counts never overlap.
+       */
       const runningTriageCount: number = (
         await AIRunService.countBy({
           query: {
@@ -467,9 +627,26 @@ export default class AIInvestigationQueue {
         })
       ).toNumber();
 
-      if (runningTriageCount >= triageCap) {
+      const runningPlanCount: number = (
+        await AIRunService.countBy({
+          query: {
+            projectId: run.projectId,
+            runType: QueryHelper.any([
+              AIRunType.RemediationPlan,
+              AIRunType.RemediationExecution,
+            ]),
+            status: AIRunStatus.Running,
+          },
+          props: { isRoot: true },
+        })
+      ).toNumber();
+
+      const runningBackgroundCount: number =
+        runningTriageCount + runningPlanCount;
+
+      if (runningBackgroundCount >= backgroundCap) {
         logger.debug(
-          `AI: leaving insight triage run ${run.id.toString()} queued — ${runningTriageCount} triage runs already running (triage lane cap: ${triageCap} of ${concurrencyCap}).`,
+          `AI: leaving background run ${run.id.toString()} queued — ${runningBackgroundCount} background runs (triage + remediation plans) already running (background lane cap: ${backgroundCap} of ${concurrencyCap}).`,
         );
         return false;
       }
@@ -521,6 +698,68 @@ export default class AIInvestigationQueue {
     run: QueuedRunRef,
     attempt: number,
   ): Promise<void> {
+    /*
+     * RemediationPlan runs also carry triggeredByIncidentId/AlertId (for
+     * dashboard linkage), so the runType discriminator MUST come before the
+     * subject-column routing below — otherwise a plan run would be executed
+     * as an RCA investigation.
+     */
+    if (run.runType === AIRunType.RemediationPlan) {
+      if (!run.triggeredByAutoRemediationSuggestionId) {
+        await AIRunService.attemptStatusTransition({
+          aiRunId: run.id,
+          fromStatus: AIRunStatus.Running,
+          set: {
+            status: AIRunStatus.Error,
+            completedAt: OneUptimeDate.getCurrentDate(),
+            errorMessage:
+              "Queued remediation plan has no suggestion to plan for.",
+          },
+        });
+        return;
+      }
+
+      const remediationPlanRunner: typeof import("../Remediation/RemediationPlanRunner").default =
+        // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+        require("../Remediation/RemediationPlanRunner").default;
+
+      await remediationPlanRunner.executePlan({
+        aiRunId: run.id,
+        projectId: run.projectId,
+        suggestionId: run.triggeredByAutoRemediationSuggestionId,
+        attemptCount: attempt,
+      });
+      return;
+    }
+
+    if (run.runType === AIRunType.RemediationExecution) {
+      if (!run.triggeredByAutoRemediationSuggestionId) {
+        await AIRunService.attemptStatusTransition({
+          aiRunId: run.id,
+          fromStatus: AIRunStatus.Running,
+          set: {
+            status: AIRunStatus.Error,
+            completedAt: OneUptimeDate.getCurrentDate(),
+            errorMessage:
+              "Queued remediation execution has no suggestion to work on.",
+          },
+        });
+        return;
+      }
+
+      const remediationExecutionRunner: typeof import("../Remediation/RemediationExecutionRunner").default =
+        // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+        require("../Remediation/RemediationExecutionRunner").default;
+
+      await remediationExecutionRunner.executeRemediation({
+        aiRunId: run.id,
+        projectId: run.projectId,
+        suggestionId: run.triggeredByAutoRemediationSuggestionId,
+        attemptCount: attempt,
+      });
+      return;
+    }
+
     if (run.triggeredByIncidentId) {
       /*
        * Lazy require: the runners import this queue to enqueue, so a

package/Server/Utils/AI/SRE/PostedRootCause.ts

@@ -0,0 +1,82 @@
+import IncidentFeedService from "../../../Services/IncidentFeedService";
+import AlertFeedService from "../../../Services/AlertFeedService";
+import IncidentFeed, {
+  IncidentFeedEventType,
+} from "../../../../Models/DatabaseModels/IncidentFeed";
+import AlertFeed, {
+  AlertFeedEventType,
+} from "../../../../Models/DatabaseModels/AlertFeed";
+import ObjectID from "../../../../Types/ObjectID";
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+
+/*
+ * The analysis an AI investigation posted for a subject.
+ *
+ * The RootCause feed item is the only persisted copy — the runner's
+ * postAnalysis is its sole writer, and the grader, the eval corpus and the
+ * fix-from-incident recipe all read it. Anything else that wants the RCA
+ * should come through here rather than re-deriving the query, so a change to
+ * where the analysis lives moves one place.
+ */
+export default class PostedRootCause {
+  @CaptureSpan()
+  public static async getForIncident(
+    incidentId: ObjectID,
+  ): Promise<string | null> {
+    const feedItem: IncidentFeed | null = await IncidentFeedService.findOneBy({
+      query: {
+        incidentId: incidentId,
+        incidentFeedEventType: IncidentFeedEventType.RootCause,
+      },
+      select: {
+        feedInfoInMarkdown: true,
+      },
+      sort: {
+        createdAt: SortOrder.Descending,
+      },
+      props: { isRoot: true },
+    });
+
+    const analysis: string = (feedItem?.feedInfoInMarkdown || "").trim();
+
+    return analysis || null;
+  }
+
+  @CaptureSpan()
+  public static async getForAlert(alertId: ObjectID): Promise<string | null> {
+    const feedItem: AlertFeed | null = await AlertFeedService.findOneBy({
+      query: {
+        alertId: alertId,
+        alertFeedEventType: AlertFeedEventType.RootCause,
+      },
+      select: {
+        feedInfoInMarkdown: true,
+      },
+      sort: {
+        createdAt: SortOrder.Descending,
+      },
+      props: { isRoot: true },
+    });
+
+    const analysis: string = (feedItem?.feedInfoInMarkdown || "").trim();
+
+    return analysis || null;
+  }
+
+  @CaptureSpan()
+  public static async getForSubject(data: {
+    incidentId?: ObjectID | undefined;
+    alertId?: ObjectID | undefined;
+  }): Promise<string | null> {
+    if (data.incidentId) {
+      return PostedRootCause.getForIncident(data.incidentId);
+    }
+
+    if (data.alertId) {
+      return PostedRootCause.getForAlert(data.alertId);
+    }
+
+    return null;
+  }
+}

package/Server/Utils/AI/SRE/README.md

@@ -6,5 +6,6 @@ Wake-on-signal AI SRE: when an incident or alert is created (and the project has
 - `IncidentInvestigationRunner.ts` / `AlertInvestigationRunner.ts` — per-subject context assembly + result posting
 - `IncidentPostmortemRunner.ts` — auto-draft postmortem when an incident resolves
 - `AIMemory.ts` — recurrence context from past resolved incidents (retrieval-only v1)
-
-**Roadmap & status:** [Internal/Roadmap/AISentinelExecution.md](../../../../../Internal/Roadmap/AISentinelExecution.md) (tracker — update it in the same PR when you change behavior here) and [Internal/Roadmap/AISentinelVision.md](../../../../../Internal/Roadmap/AISentinelVision.md) (strategy).
+- `RemediationHandoff.ts` — the investigation → auto-remediation hand-off (RCA-first ordering). The incident/alert create hooks defer `AutoRemediationRuleEngineService` whenever an investigation was enqueued (the runners' `investigateNew*` return "enqueued"); this module releases the subject to the rule engine when that investigation **settles** — any terminal outcome. Settle-once discipline is inherited from the run-status CAS transitions (the same discipline that keeps `postAnalysis` single-shot): only the actor that WON a terminal transition calls the hand-off. The settle sites: Completed and finalized-Error via `InvestigationRequest.onSettled` / the runners' `failOrRequeue` outcome, queue-TTL expiry and terminal-stale via `InvestigationQueue`'s private hand-off. A failed investigation still hands off — remediation degrades to its pre-RCA behavior, never silently disappears; with no investigation enqueued, the create hook applies rules immediately and this module is never involved.
+- `ConfidenceSignal.ts` — the structured, server-verified confidence signal (deterministic evidence floor + one constrained classification call), with per-consumer fail directions
+- `FixFromIncidentTaskTrigger.ts` — the FixFromIncident recipe's two trigger forms: the human "Open Fix PR from this analysis" button (no opt-in flag — the click is the gate), and the automatic form for projects opted in via `Project.enableAutomaticCodeFixes` (default false). The automatic form fires from the runners' `postAnalysis`, only on `AIConfidenceSignal.shouldAutoEnqueueCodeFixTask` — a POSITIVE confident classification; the evidence floor and a failed classification both fail toward doing nothing. Never throws; pre-checks the daily fix-run budget as a quiet skip; same repo + per-subject dedupe gates as the button; the enqueued run is system-authored (no userId).

package/Server/Utils/AI/SRE/RemediationHandoff.ts

@@ -0,0 +1,82 @@
+import ObjectID from "../../../../Types/ObjectID";
+import AutoRemediationRuleEngineService from "../../../Services/AutoRemediationRuleEngineService";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+
+/*
+ * AI SRE — the investigation → remediation hand-off (RCA-first ordering).
+ *
+ * Auto-remediation used to fire from the incident/alert create hook in
+ * PARALLEL with the AI investigation, so the remediation planner usually ran
+ * before any root cause analysis existed and could only read the RCA
+ * best-effort. Now the create hook DEFERS remediation whenever an
+ * investigation was enqueued for the subject, and this module releases the
+ * subject to the rule engine only after that investigation SETTLES — so the
+ * planner always has the posted RCA (when one was produced) as its input.
+ *
+ * "Settles" means any terminal outcome, exactly once:
+ *   - Completed (analysis posted, or nothing worth posting) — via
+ *     InvestigationRequest.onSettled from AIInvestigationEngine;
+ *   - Error after retries were exhausted — via the failOrRequeue outcome in
+ *     the engine/runners;
+ *   - Cancelled (queue TTL expiry) or Stale (heartbeat death, no retries
+ *     left) — via AIInvestigationQueue's terminal transitions.
+ *
+ * Exactly-once is inherited from the run-status CAS transitions: only the
+ * actor that WON a terminal transition calls this, the same discipline that
+ * keeps postAnalysis from double-posting. A failed investigation still hands
+ * off — remediation must degrade to its pre-RCA behavior, never silently
+ * disappear because the AI lane broke.
+ *
+ * When NO investigation is enqueued at creation (opt-out, gates, budget),
+ * the create hook applies the rules immediately and this module is never
+ * involved — remediation does not depend on the AI lane being enabled.
+ */
+export default class RemediationHandoff {
+  /*
+   * Release a settled investigation's subject to the auto-remediation rule
+   * engine. Exactly one of incidentId/alertId is expected (an investigation
+   * has exactly one subject). Never throws — the hand-off runs after
+   * terminal run transitions and inside postAnalysis follow-ups, where a
+   * failure must be logged, not propagated.
+   */
+  @CaptureSpan()
+  public static async runForSettledInvestigation(data: {
+    projectId: ObjectID;
+    incidentId?: ObjectID | undefined;
+    alertId?: ObjectID | undefined;
+  }): Promise<void> {
+    try {
+      if (data.incidentId) {
+        logger.debug(
+          `AI: investigation settled for incident ${data.incidentId.toString()} — handing off to auto-remediation.`,
+        );
+
+        await AutoRemediationRuleEngineService.applyRulesToIncidentById({
+          incidentId: data.incidentId,
+          projectId: data.projectId,
+        });
+        return;
+      }
+
+      if (data.alertId) {
+        logger.debug(
+          `AI: investigation settled for alert ${data.alertId.toString()} — handing off to auto-remediation.`,
+        );
+
+        await AutoRemediationRuleEngineService.applyRulesToAlertById({
+          alertId: data.alertId,
+          projectId: data.projectId,
+        });
+      }
+    } catch (error) {
+      logger.error(
+        `AI: remediation hand-off failed for ${
+          data.incidentId
+            ? `incident ${data.incidentId.toString()}`
+            : `alert ${data.alertId?.toString()}`
+        }: ${error}`,
+      );
+    }
+  }
+}

package/Server/Utils/AI/SRE/SubjectCodeFixRun.ts

@@ -1,6 +1,8 @@
 import ObjectID from "../../../../Types/ObjectID";
 import AIRunType from "../../../../Types/AI/AIRunType";
-import AIRunStatus from "../../../../Types/AI/AIRunStatus";
+import AIRunStatus, {
+  AIRunStatusHelper,
+} from "../../../../Types/AI/AIRunStatus";
 import CodeFixTaskType from "../../../../Types/AI/CodeFixTaskType";
 import CodeFixTaskContext from "../../../../Types/AI/CodeFixTaskContext";
 import CodeRepositoryType from "../../../../Types/CodeRepository/CodeRepositoryType";
@@ -65,12 +67,7 @@ export default class SubjectCodeFixRun {
         ...(data.incidentId
           ? { triggeredByIncidentId: data.incidentId }
           : { triggeredByAlertId: data.alertId! }),
-        status: QueryHelper.notIn([
-          AIRunStatus.Completed,
-          AIRunStatus.Error,
-          AIRunStatus.Cancelled,
-          AIRunStatus.Stale,
-        ]),
+        status: QueryHelper.notIn(AIRunStatusHelper.terminalStatuses()),
       },
       select: { _id: true },
       props: { isRoot: true },
@@ -96,12 +93,7 @@ export default class SubjectCodeFixRun {
         projectId: data.projectId,
         runType: AIRunType.CodeFix,
         codeFixTaskType: CodeFixTaskType.FixPerformance,
-        status: QueryHelper.notIn([
-          AIRunStatus.Completed,
-          AIRunStatus.Error,
-          AIRunStatus.Cancelled,
-          AIRunStatus.Stale,
-        ]),
+        status: QueryHelper.notIn(AIRunStatusHelper.terminalStatuses()),
       },
       select: { _id: true, taskContext: true },
       limit: LIMIT_PER_PROJECT,
@@ -135,12 +127,7 @@ export default class SubjectCodeFixRun {
         projectId: data.projectId,
         runType: AIRunType.CodeFix,
         codeFixTaskType: data.taskType,
-        status: QueryHelper.notIn([
-          AIRunStatus.Completed,
-          AIRunStatus.Error,
-          AIRunStatus.Cancelled,
-          AIRunStatus.Stale,
-        ]),
+        status: QueryHelper.notIn(AIRunStatusHelper.terminalStatuses()),
       },
       select: { _id: true, taskContext: true },
       limit: LIMIT_PER_PROJECT,