@oneuptime/common 12.0.4 → 12.0.5

package/Server/Utils/AI/SRE/InvestigationGrader.ts

@@ -146,6 +146,7 @@ export default class InvestigationGrader {
         select: {
           _id: true,
           autoGrade: true,
+          completedAt: true,
         },
         sort: { createdAt: SortOrder.Descending },
         props: { isRoot: true },
@@ -175,13 +176,16 @@ export default class InvestigationGrader {
       }
 
       /*
-       * The posted analysis: the latest RootCause feed item — the AI's
-       * postAnalysis is its only writer, and it is the only persisted copy
-       * of the analysis (the same source the fix recipes and the remediation
-       * planner read, through this one helper).
+       * The posted analysis owned by this exact run. A newer investigation or
+       * a human RootCause item may exist by resolution time; neither may be
+       * graded as though it came from the selected investigation.
        */
       const analysisMarkdown: string | null =
-        await PostedRootCause.getForIncident(incidentId);
+        await PostedRootCause.getForInvestigation({
+          incidentId,
+          aiRunId: run.id!,
+          runCompletedAt: run.completedAt,
+        });
 
       if (!analysisMarkdown) {
         logger.debug(

package/Server/Utils/AI/SRE/InvestigationQueue.ts

@@ -1,7 +1,9 @@
 import ObjectID from "../../../../Types/ObjectID";
 import OneUptimeDate from "../../../../Types/Date";
+import Query from "../../../../Types/BaseDatabase/Query";
 import AIRunType from "../../../../Types/AI/AIRunType";
 import AIRunStatus from "../../../../Types/AI/AIRunStatus";
+import AIRunCodeFixRecommendation from "../../../../Types/AI/AIRunCodeFixRecommendation";
 import AIRun from "../../../../Models/DatabaseModels/AIRun";
 import Project from "../../../../Models/DatabaseModels/Project";
 import AIRunService from "../../../Services/AIRunService";
@@ -11,6 +13,7 @@ import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
 import QueryHelper from "../../../Types/Database/QueryHelper";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
+import InvestigationSubjectLock from "./InvestigationSubjectLock";
 
 /*
  * AI SRE — the durable investigation queue (Phase 2's first item; Q1
@@ -47,11 +50,11 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
 export const MAX_INVESTIGATION_ATTEMPTS: number = 2;
 
 /*
- * G4 cost guardrail: at most this many investigations may be Running per
- * project at once (per-project override in
- * Project.aiMaxConcurrentInvestigations). Enforced at CLAIM time, so a storm
- * queues (bounded by dedupe + severity gates + this TTL) and drains at cap
- * rate instead of being dropped.
+ * G4 cost guardrail: at most this many investigations may be Running in an
+ * incident, alert or subjectless lane at once. Each lane has its own project
+ * override; subjectless work keeps Project.aiMaxConcurrentInvestigations.
+ * Enforced at CLAIM time, so a storm queues (bounded by dedupe + severity
+ * gates + this TTL) and drains at cap rate instead of being dropped.
  */
 export const DEFAULT_MAX_CONCURRENT_INVESTIGATIONS: number = 3;
 /*
@@ -62,7 +65,7 @@ const MIN_CONCURRENT_INVESTIGATIONS: number = 1;
 const MAX_CONCURRENT_INVESTIGATIONS: number = 25;
 
 /*
- * Lane priority. Two kinds of run share this queue and the per-project
+ * Lane priority. Two kinds of run share this queue and each subject lane's
  * concurrency cap above:
  *   - the INTERACTIVE lane (incident/alert RCA) — a human is waiting;
  *   - the PREVENTIVE lane (AI-insight triage, identified by
@@ -143,6 +146,13 @@ export default class AIInvestigationQueue {
   }): Promise<ObjectID | null> {
     const { projectId } = data;
 
+    if (data.subjectIncidentId && data.subjectAlertId) {
+      logger.error(
+        `AI: not enqueueing investigation for project ${projectId.toString()} — a run cannot belong to both an incident and an alert.`,
+      );
+      return null;
+    }
+
     /*
      * Budget quiet-skip at enqueue: when the daily budget is already
      * exhausted there is no point recording intent that the TTL would
@@ -150,7 +160,10 @@ export default class AIInvestigationQueue {
      */
     try {
       const budget: AutonomousBudgetStatus =
-        await AIService.getAutonomousDailyBudgetStatus(projectId);
+        await AIService.getAutonomousDailyBudgetStatus(projectId, {
+          incidentId: data.subjectIncidentId,
+          alertId: data.subjectAlertId,
+        });
 
       if (budget.exhausted) {
         logger.debug(
@@ -173,6 +186,13 @@ export default class AIInvestigationQueue {
     run.projectId = projectId;
     run.runType = runType;
     run.status = AIRunStatus.Queued;
+    /*
+     * Queued and Running rows must never advertise an unresolved code-fix
+     * decision. The engine atomically moves eligible incident/alert runs to
+     * Pending only when it wins the Running -> Completed transition; until
+     * then the fail-closed default is NotRecommended.
+     */
+    run.codeFixRecommendation = AIRunCodeFixRecommendation.NotRecommended;
 
     if (data.subjectIncidentId) {
       run.triggeredByIncidentId = data.subjectIncidentId;
@@ -193,10 +213,32 @@ export default class AIInvestigationQueue {
 
     let createdRun: AIRun;
     try {
-      createdRun = await AIRunService.create({
-        data: run,
-        props: { isRoot: true },
-      });
+      const createRun: () => Promise<AIRun> = async (): Promise<AIRun> => {
+        return AIRunService.create({
+          data: run,
+          props: { isRoot: true },
+        });
+      };
+
+      /*
+       * Linearize subject Investigation creation with FixFromIncident's
+       * final latest-source validation and INSERT. Whichever writer gets the
+       * shared lock first becomes authoritative: a fix inserted first is
+       * valid at that instant; an Investigation inserted first is observed by
+       * the fix gate and makes the stale request fail closed.
+       */
+      createdRun =
+        runType === AIRunType.Investigation &&
+        (data.subjectIncidentId || data.subjectAlertId)
+          ? await InvestigationSubjectLock.runExclusive(
+              {
+                projectId,
+                incidentId: data.subjectIncidentId,
+                alertId: data.subjectAlertId,
+              },
+              createRun,
+            )
+          : await createRun();
     } catch (error) {
       logger.error(`AI: failed to enqueue investigation run: ${error}`);
       return null;
@@ -291,6 +333,7 @@ export default class AIInvestigationQueue {
         fromStatus: AIRunStatus.Queued,
         set: {
           status: AIRunStatus.Cancelled,
+          codeFixRecommendation: AIRunCodeFixRecommendation.NotRecommended,
           completedAt: OneUptimeDate.getCurrentDate(),
           errorMessage: `Expired in the investigation queue after ${QUEUE_TTL_MINUTES} minutes — a first-pass analysis this late would no longer be useful. The project may have been at its concurrency cap or daily token budget.`,
         },
@@ -404,8 +447,10 @@ export default class AIInvestigationQueue {
       const requeued: number = await AIRunService.attemptStatusTransition({
         aiRunId: data.aiRunId,
         fromStatus: AIRunStatus.Running,
+        expectedAttemptCount: data.attemptCount,
         set: {
           status: AIRunStatus.Queued,
+          codeFixRecommendation: AIRunCodeFixRecommendation.NotRecommended,
           errorMessage: `Attempt ${data.attemptCount} failed and the run was requeued: ${truncatedMessage}`,
         },
       });
@@ -422,8 +467,10 @@ export default class AIInvestigationQueue {
     const finalized: number = await AIRunService.attemptStatusTransition({
       aiRunId: data.aiRunId,
       fromStatus: AIRunStatus.Running,
+      expectedAttemptCount: data.attemptCount,
       set: {
         status: AIRunStatus.Error,
+        codeFixRecommendation: AIRunCodeFixRecommendation.NotRecommended,
         completedAt: OneUptimeDate.getCurrentDate(),
         errorMessage: truncatedMessage,
       },
@@ -448,17 +495,21 @@ export default class AIInvestigationQueue {
   public static async requeueOrMarkStale(run: {
     id: ObjectID;
     attemptCount: number;
+    lastHeartbeatAt?: Date | undefined;
     runType?: AIRunType | undefined;
     projectId?: ObjectID | undefined;
     triggeredByIncidentId?: ObjectID | undefined;
     triggeredByAlertId?: ObjectID | undefined;
-  }): Promise<"requeued" | "stale"> {
+  }): Promise<"requeued" | "stale" | "noop"> {
     if ((run.attemptCount || 0) < MAX_INVESTIGATION_ATTEMPTS) {
       const requeued: number = await AIRunService.attemptStatusTransition({
         aiRunId: run.id,
         fromStatus: AIRunStatus.Running,
+        expectedAttemptCount: run.attemptCount,
+        expectedLastHeartbeatAt: run.lastHeartbeatAt,
         set: {
           status: AIRunStatus.Queued,
+          codeFixRecommendation: AIRunCodeFixRecommendation.NotRecommended,
           errorMessage: `Attempt ${run.attemptCount} stopped reporting progress (the server processing it may have restarted) and the run was requeued.`,
         },
       });
@@ -471,8 +522,11 @@ export default class AIInvestigationQueue {
     const staled: number = await AIRunService.attemptStatusTransition({
       aiRunId: run.id,
       fromStatus: AIRunStatus.Running,
+      expectedAttemptCount: run.attemptCount,
+      expectedLastHeartbeatAt: run.lastHeartbeatAt,
       set: {
         status: AIRunStatus.Stale,
+        codeFixRecommendation: AIRunCodeFixRecommendation.NotRecommended,
         completedAt: OneUptimeDate.getCurrentDate(),
         errorMessage:
           "The run stopped reporting progress and was marked as stale after exhausting its retry attempts. The server processing it may have restarted.",
@@ -485,9 +539,10 @@ export default class AIInvestigationQueue {
      */
     if (staled > 0) {
       await this.handOffSettledInvestigationToRemediation(run);
+      return "stale";
     }
 
-    return "stale";
+    return "noop";
   }
 
   /*
@@ -544,26 +599,50 @@ export default class AIInvestigationQueue {
    * errors.
    */
   private static async passesClaimGates(run: QueuedRunRef): Promise<boolean> {
-    // Per-project cap override, defaulting to 3 and clamped to [1, 25].
+    if (run.triggeredByIncidentId && run.triggeredByAlertId) {
+      logger.error(
+        `AI: leaving run ${run.id.toString()} queued — a run cannot belong to both an incident and an alert.`,
+      );
+      return false;
+    }
+
+    /*
+     * Each subject lane owns its own concurrency pool. Insight triage and any
+     * future subjectless queue work retain the legacy project-wide fallback.
+     */
     const project: Project | null = await ProjectService.findOneById({
       id: run.projectId,
-      select: { aiMaxConcurrentInvestigations: true },
+      select: {
+        aiMaxConcurrentInvestigations: true,
+        incidentAiMaxConcurrentInvestigations: true,
+        alertAiMaxConcurrentInvestigations: true,
+      },
       props: { isRoot: true },
     });
 
+    let configuredConcurrencyCap: number | undefined =
+      project?.aiMaxConcurrentInvestigations;
+
+    if (run.triggeredByIncidentId) {
+      configuredConcurrencyCap = project?.incidentAiMaxConcurrentInvestigations;
+    } else if (run.triggeredByAlertId) {
+      configuredConcurrencyCap = project?.alertAiMaxConcurrentInvestigations;
+    }
+
     const concurrencyCap: number = Math.min(
       MAX_CONCURRENT_INVESTIGATIONS,
       Math.max(
         MIN_CONCURRENT_INVESTIGATIONS,
-        project?.aiMaxConcurrentInvestigations ??
-          DEFAULT_MAX_CONCURRENT_INVESTIGATIONS,
+        configuredConcurrencyCap ?? DEFAULT_MAX_CONCURRENT_INVESTIGATIONS,
       ),
     );
 
+    const subjectLaneQuery: Query<AIRun> = this.getSubjectLaneQuery(run);
+
     /*
      * Investigations, remediation plans and remediation executions share
-     * the same per-project cap — all are short and storm-shaped on
-     * incident/alert creation, so one pool keeps the cost model simple.
+     * the cap inside their incident, alert or subjectless lane. Work in one
+     * lane never consumes another lane's concurrency slots.
      */
     const runningCount: number = (
       await AIRunService.countBy({
@@ -575,6 +654,7 @@ export default class AIInvestigationQueue {
             AIRunType.RemediationExecution,
           ]),
           status: AIRunStatus.Running,
+          ...subjectLaneQuery,
         },
         props: { isRoot: true },
       })
@@ -595,10 +675,10 @@ export default class AIInvestigationQueue {
      * kinds share ONE combined sub-cap, so together they always leave the
      * reserved slot reachable. Floored at 1 so a project running at the
      * minimum cap of 1 still drains its background work (slower, one at a
-     * time) instead of deadlocking the lane entirely — the interactive lane
-     * keeps its priority through the global cap check above, which a single
-     * running background run cannot outlast for long (both kinds are
-     * read-only and short).
+     * time) instead of deadlocking the lane entirely — foreground work keeps
+     * its priority through the lane cap check above, which a single running
+     * background run cannot outlast for long (both kinds are read-only and
+     * short).
      */
     const isBackgroundLane: boolean =
       Boolean(run.triggeredByAiInsightId) ||
@@ -622,6 +702,7 @@ export default class AIInvestigationQueue {
             runType: AIRunType.Investigation,
             status: AIRunStatus.Running,
             triggeredByAiInsightId: QueryHelper.notNull(),
+            ...subjectLaneQuery,
           },
           props: { isRoot: true },
         })
@@ -636,6 +717,7 @@ export default class AIInvestigationQueue {
               AIRunType.RemediationExecution,
             ]),
             status: AIRunStatus.Running,
+            ...subjectLaneQuery,
           },
           props: { isRoot: true },
         })
@@ -653,7 +735,10 @@ export default class AIInvestigationQueue {
     }
 
     const budget: AutonomousBudgetStatus =
-      await AIService.getAutonomousDailyBudgetStatus(run.projectId);
+      await AIService.getAutonomousDailyBudgetStatus(run.projectId, {
+        incidentId: run.triggeredByIncidentId,
+        alertId: run.triggeredByAlertId,
+      });
 
     if (budget.exhausted) {
       logger.debug(
@@ -665,6 +750,35 @@ export default class AIInvestigationQueue {
     return true;
   }
 
+  /*
+   * Scope queue counts to one independent lane. The concrete subject id is
+   * deliberately not compared: all incidents share the incident cap, all
+   * alerts share the alert cap, and subjectless work must have neither link.
+   */
+  private static getSubjectLaneQuery(run: {
+    triggeredByIncidentId?: ObjectID | undefined;
+    triggeredByAlertId?: ObjectID | undefined;
+  }): Query<AIRun> {
+    if (run.triggeredByIncidentId) {
+      return {
+        triggeredByIncidentId: QueryHelper.notNull(),
+        triggeredByAlertId: QueryHelper.isNull(),
+      };
+    }
+
+    if (run.triggeredByAlertId) {
+      return {
+        triggeredByIncidentId: QueryHelper.isNull(),
+        triggeredByAlertId: QueryHelper.notNull(),
+      };
+    }
+
+    return {
+      triggeredByIncidentId: QueryHelper.isNull(),
+      triggeredByAlertId: QueryHelper.isNull(),
+    };
+  }
+
   /*
    * The atomic claim: Queued -> Running, guarded on the attemptCount the
    * caller observed, so a stale queue snapshot can neither double-claim
@@ -709,6 +823,7 @@ export default class AIInvestigationQueue {
         await AIRunService.attemptStatusTransition({
           aiRunId: run.id,
           fromStatus: AIRunStatus.Running,
+          expectedAttemptCount: attempt,
           set: {
             status: AIRunStatus.Error,
             completedAt: OneUptimeDate.getCurrentDate(),
@@ -737,6 +852,7 @@ export default class AIInvestigationQueue {
         await AIRunService.attemptStatusTransition({
           aiRunId: run.id,
           fromStatus: AIRunStatus.Running,
+          expectedAttemptCount: attempt,
           set: {
             status: AIRunStatus.Error,
             completedAt: OneUptimeDate.getCurrentDate(),
@@ -817,6 +933,7 @@ export default class AIInvestigationQueue {
     await AIRunService.attemptStatusTransition({
       aiRunId: run.id,
       fromStatus: AIRunStatus.Running,
+      expectedAttemptCount: attempt,
       set: {
         status: AIRunStatus.Error,
         completedAt: OneUptimeDate.getCurrentDate(),

package/Server/Utils/AI/SRE/InvestigationSubjectLock.ts

@@ -0,0 +1,66 @@
+import ObjectID from "../../../../Types/ObjectID";
+import BadDataException from "../../../../Types/Exception/BadDataException";
+import Semaphore, { SemaphoreMutex } from "../../../Infrastructure/Semaphore";
+import logger from "../../Logger";
+
+/*
+ * Investigation creation and FixFromIncident authorization must share one
+ * linearization point. Without it, a new Investigation can be inserted after
+ * a fix trigger's "latest run" read but before the CodeFix INSERT, allowing a
+ * task grounded in an analysis that is no longer authoritative.
+ */
+export const INVESTIGATION_SUBJECT_LOCK_NAMESPACE: string =
+  "InvestigationSubjectLifecycle";
+
+export interface InvestigationSubjectLockData {
+  projectId: ObjectID;
+  incidentId?: ObjectID | undefined;
+  alertId?: ObjectID | undefined;
+}
+
+export function getInvestigationSubjectLockKey(
+  data: InvestigationSubjectLockData,
+): string {
+  if (Boolean(data.incidentId) === Boolean(data.alertId)) {
+    throw new BadDataException(
+      "Exactly one incident or alert subject is required.",
+    );
+  }
+
+  const subjectKey: string = data.incidentId
+    ? `incident-${data.incidentId.toString()}`
+    : `alert-${data.alertId!.toString()}`;
+
+  return `${data.projectId.toString()}-${subjectKey}`;
+}
+
+export default class InvestigationSubjectLock {
+  public static async runExclusive<T>(
+    data: InvestigationSubjectLockData,
+    operation: () => Promise<T>,
+  ): Promise<T> {
+    const key: string = getInvestigationSubjectLockKey(data);
+    const mutex: SemaphoreMutex = await Semaphore.lock({
+      key,
+      namespace: INVESTIGATION_SUBJECT_LOCK_NAMESPACE,
+      lockTimeout: 30 * 1000,
+      acquireTimeout: 10 * 1000,
+    });
+
+    try {
+      return await operation();
+    } finally {
+      try {
+        await Semaphore.release(mutex);
+      } catch (error) {
+        /*
+         * The Redis mutex expires automatically. Do not hide the operation's
+         * result (or its more useful error) when best-effort release fails.
+         */
+        logger.error(
+          `AI: failed to release the investigation subject lock ${key}; it will expire automatically: ${error}`,
+        );
+      }
+    }
+  }
+}

package/Server/Utils/AI/SRE/PostedRootCause.ts

@@ -8,8 +8,26 @@ import AlertFeed, {
 } from "../../../../Models/DatabaseModels/AlertFeed";
 import ObjectID from "../../../../Types/ObjectID";
 import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
+import QueryHelper from "../../../Types/Database/QueryHelper";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
+export const AI_ROOT_CAUSE_HEADING: string =
+  "AI — Automated Root Cause Analysis";
+
+export interface PostedRootCauseQueryOptions {
+  /* Exact ownership for newly posted AI investigation reports. */
+  aiRunId?: ObjectID | undefined;
+  /* Upgrade-only lookup for RootCause rows written before aiRunId existed. */
+  withoutAIRunId?: boolean | undefined;
+  /*
+   * A subject can be investigated more than once. When a caller is resolving
+   * the analysis for one particular run, it must not accidentally return a
+   * RootCause item posted by an older run while the new one is still being
+   * finalized.
+   */
+  createdAtOrAfter?: Date | undefined;
+}
+
 /*
  * The analysis an AI investigation posted for a subject.
  *
@@ -20,14 +38,64 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  * where the analysis lives moves one place.
  */
 export default class PostedRootCause {
+  /*
+   * Resolve one run's report exactly. A narrow compatibility path keeps
+   * reports written before aiRunId was deployed visible. Deployment time is
+   * not the migration file's build timestamp, so compatibility cannot use a
+   * wall-clock cutover. Instead, only null-associated rows posted after the
+   * run completed and carrying the server's branded AI heading qualify.
+   */
+  @CaptureSpan()
+  public static async getForInvestigation(data: {
+    incidentId?: ObjectID | undefined;
+    alertId?: ObjectID | undefined;
+    aiRunId: ObjectID;
+    runCompletedAt?: Date | undefined;
+  }): Promise<string | null> {
+    const exactAnalysis: string | null = await this.getForSubject({
+      incidentId: data.incidentId,
+      alertId: data.alertId,
+      aiRunId: data.aiRunId,
+    });
+
+    if (exactAnalysis) {
+      return exactAnalysis;
+    }
+
+    if (!data.runCompletedAt) {
+      return null;
+    }
+
+    const legacyAnalysis: string | null = await this.getForSubject({
+      incidentId: data.incidentId,
+      alertId: data.alertId,
+      withoutAIRunId: true,
+      createdAtOrAfter: data.runCompletedAt,
+    });
+
+    return legacyAnalysis?.includes(AI_ROOT_CAUSE_HEADING)
+      ? legacyAnalysis
+      : null;
+  }
+
   @CaptureSpan()
   public static async getForIncident(
     incidentId: ObjectID,
+    options?: PostedRootCauseQueryOptions | undefined,
   ): Promise<string | null> {
     const feedItem: IncidentFeed | null = await IncidentFeedService.findOneBy({
       query: {
         incidentId: incidentId,
         incidentFeedEventType: IncidentFeedEventType.RootCause,
+        ...(options?.aiRunId ? { aiRunId: options.aiRunId } : {}),
+        ...(options?.withoutAIRunId ? { aiRunId: QueryHelper.isNull() } : {}),
+        ...(options?.createdAtOrAfter
+          ? {
+              createdAt: QueryHelper.greaterThanEqualTo(
+                options.createdAtOrAfter,
+              ),
+            }
+          : {}),
       },
       select: {
         feedInfoInMarkdown: true,
@@ -44,11 +112,23 @@ export default class PostedRootCause {
   }
 
   @CaptureSpan()
-  public static async getForAlert(alertId: ObjectID): Promise<string | null> {
+  public static async getForAlert(
+    alertId: ObjectID,
+    options?: PostedRootCauseQueryOptions | undefined,
+  ): Promise<string | null> {
     const feedItem: AlertFeed | null = await AlertFeedService.findOneBy({
       query: {
         alertId: alertId,
         alertFeedEventType: AlertFeedEventType.RootCause,
+        ...(options?.aiRunId ? { aiRunId: options.aiRunId } : {}),
+        ...(options?.withoutAIRunId ? { aiRunId: QueryHelper.isNull() } : {}),
+        ...(options?.createdAtOrAfter
+          ? {
+              createdAt: QueryHelper.greaterThanEqualTo(
+                options.createdAtOrAfter,
+              ),
+            }
+          : {}),
       },
       select: {
         feedInfoInMarkdown: true,
@@ -68,13 +148,24 @@ export default class PostedRootCause {
   public static async getForSubject(data: {
     incidentId?: ObjectID | undefined;
     alertId?: ObjectID | undefined;
+    aiRunId?: ObjectID | undefined;
+    withoutAIRunId?: boolean | undefined;
+    createdAtOrAfter?: Date | undefined;
   }): Promise<string | null> {
     if (data.incidentId) {
-      return PostedRootCause.getForIncident(data.incidentId);
+      return PostedRootCause.getForIncident(data.incidentId, {
+        aiRunId: data.aiRunId,
+        withoutAIRunId: data.withoutAIRunId,
+        createdAtOrAfter: data.createdAtOrAfter,
+      });
     }
 
     if (data.alertId) {
-      return PostedRootCause.getForAlert(data.alertId);
+      return PostedRootCause.getForAlert(data.alertId, {
+        aiRunId: data.aiRunId,
+        withoutAIRunId: data.withoutAIRunId,
+        createdAtOrAfter: data.createdAtOrAfter,
+      });
     }
 
     return null;

package/Server/Utils/AI/SRE/README.md

@@ -8,4 +8,4 @@ Wake-on-signal AI SRE: when an incident or alert is created (and the project has
 - `AIMemory.ts` — recurrence context from past resolved incidents (retrieval-only v1)
 - `RemediationHandoff.ts` — the investigation → auto-remediation hand-off (RCA-first ordering). The incident/alert create hooks defer `AutoRemediationRuleEngineService` whenever an investigation was enqueued (the runners' `investigateNew*` return "enqueued"); this module releases the subject to the rule engine when that investigation **settles** — any terminal outcome. Settle-once discipline is inherited from the run-status CAS transitions (the same discipline that keeps `postAnalysis` single-shot): only the actor that WON a terminal transition calls the hand-off. The settle sites: Completed and finalized-Error via `InvestigationRequest.onSettled` / the runners' `failOrRequeue` outcome, queue-TTL expiry and terminal-stale via `InvestigationQueue`'s private hand-off. A failed investigation still hands off — remediation degrades to its pre-RCA behavior, never silently disappears; with no investigation enqueued, the create hook applies rules immediately and this module is never involved.
 - `ConfidenceSignal.ts` — the structured, server-verified confidence signal (deterministic evidence floor + one constrained classification call), with per-consumer fail directions
-- `FixFromIncidentTaskTrigger.ts` — the FixFromIncident recipe's two trigger forms: the human "Open Fix PR from this analysis" button (no opt-in flag — the click is the gate), and the automatic form for projects opted in via `Project.enableAutomaticCodeFixes` (default false). The automatic form fires from the runners' `postAnalysis`, only on `AIConfidenceSignal.shouldAutoEnqueueCodeFixTask` — a POSITIVE confident classification; the evidence floor and a failed classification both fail toward doing nothing. Never throws; pre-checks the daily fix-run budget as a quiet skip; same repo + per-subject dedupe gates as the button; the enqueued run is system-authored (no userId).
+- `FixFromIncidentTaskTrigger.ts` — the FixFromIncident recipe's two trigger forms: the human "Open Fix PR from this analysis" button (shown only when the investigation's structured classification recommends a repository code change), and the automatic form for projects opted in through the independent incident or alert automatic-code-fix setting (default false). Both use the same fail-closed code-fix recommendation: operational, infrastructure, external, user-error, inconclusive and failed classifications never offer or open a code-fix PR. The automatic form fires only after the exact posted analysis and its Recommended decision are durably stored; it never throws, pre-checks the subject lane's daily fix-run budget as a quiet skip, and shares the same repo + serialized per-subject dedupe gates as the button. The enqueued automatic run is system-authored (no userId).