@oneuptime/common 11.5.4 → 11.5.5

package/Server/Utils/AI/{Sentinel → SRE}/InvestigationQueue.ts

@@ -13,7 +13,7 @@ import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
 /*
- * Sentinel — the durable investigation queue (Phase 2's first item; Q1
+ * AI SRE — the durable investigation queue (Phase 2's first item; Q1
  * decided as the DB-claim pattern on AIRun rows, no new infrastructure).
  *
  * The problem it replaces: investigations ran as detached in-process
@@ -61,6 +61,30 @@ export const DEFAULT_MAX_CONCURRENT_INVESTIGATIONS: number = 3;
 const MIN_CONCURRENT_INVESTIGATIONS: number = 1;
 const MAX_CONCURRENT_INVESTIGATIONS: number = 25;
 
+/*
+ * Lane priority. Two kinds of run share this queue and the per-project
+ * concurrency cap above:
+ *   - the INTERACTIVE lane (incident/alert RCA) — a human is waiting;
+ *   - the PREVENTIVE lane (AI-insight triage, identified by
+ *     triggeredByAiInsightId) — nobody is waiting, and one scan tick
+ *     can file up to MAX_NEW_INSIGHTS_PER_PROJECT_PER_SCAN (10) of them.
+ *
+ * Without a sub-cap the preventive lane can hold every slot, and since the
+ * poller drains oldest-first, an incident that fires after a scan queues
+ * BEHIND that backlog — the exact RCA latency the product promises.
+ *
+ * So the preventive lane may hold at most (cap - RESERVED) slots: at least
+ * one slot is always unreachable by triage. Combined with the inline kick
+ * that enqueue() fires for every run, that reserved slot is enough on its
+ * own — an incident/alert enqueue calls processRun immediately, passes the
+ * gates against a cap that triage cannot have saturated, and dispatches
+ * without ever touching the poller (so it can also never be TTL-expired
+ * behind triage). Splitting the poller's oldest-first query into two lane
+ * queries would therefore only re-order runs that are ALREADY late, at the
+ * cost of an extra query every tick — deliberately not done.
+ */
+export const INSIGHT_TRIAGE_RESERVED_SLOTS: number = 1;
+
 /*
  * A first-pass RCA is only useful while the incident is fresh. Queued runs
  * older than this are expired rather than run late — this also caps queue
@@ -77,14 +101,19 @@ export interface QueuedRunRef {
   attemptCount: number;
   triggeredByIncidentId?: ObjectID | undefined;
   triggeredByAlertId?: ObjectID | undefined;
+  triggeredByAiInsightId?: ObjectID | undefined;
 }
 
-export default class SentinelInvestigationQueue {
+export default class AIInvestigationQueue {
   /*
    * Record the durable intent to investigate, then try to process it
    * immediately (detached — the poller is the safety net if this pod dies).
    * Callers have already passed the subject-specific gates (opt-in,
    * severity floor, dedupe window).
+   *
+   * Returns the created run's id (so subject rows can link back to it), or
+   * null when the enqueue was quiet-skipped (budget) or failed — enqueue
+   * itself never throws.
    */
   @CaptureSpan()
   public static async enqueue(data: {
@@ -92,7 +121,8 @@ export default class SentinelInvestigationQueue {
     subjectIncidentId?: ObjectID | undefined;
     subjectAlertId?: ObjectID | undefined;
     subjectMonitorId?: ObjectID | undefined;
-  }): Promise<void> {
+    subjectAIInsightId?: ObjectID | undefined;
+  }): Promise<ObjectID | null> {
     const { projectId } = data;
 
     /*
@@ -106,15 +136,15 @@ export default class SentinelInvestigationQueue {
 
       if (budget.exhausted) {
         logger.debug(
-          `Sentinel: not enqueueing investigation for project ${projectId.toString()} — daily autonomous token budget exhausted (${budget.usedTokensToday} of ${budget.limitInTokens} tokens used today).`,
+          `AI: not enqueueing investigation for project ${projectId.toString()} — daily autonomous token budget exhausted (${budget.usedTokensToday} of ${budget.limitInTokens} tokens used today).`,
         );
-        return;
+        return null;
       }
     } catch (error) {
       logger.error(
-        `Sentinel: budget check failed, not enqueueing investigation: ${error}`,
+        `AI: budget check failed, not enqueueing investigation: ${error}`,
       );
-      return;
+      return null;
     }
 
     const run: AIRun = new AIRun();
@@ -131,6 +161,9 @@ export default class SentinelInvestigationQueue {
     if (data.subjectMonitorId) {
       run.monitorId = data.subjectMonitorId;
     }
+    if (data.subjectAIInsightId) {
+      run.triggeredByAiInsightId = data.subjectAIInsightId;
+    }
 
     let createdRun: AIRun;
     try {
@@ -139,8 +172,8 @@ export default class SentinelInvestigationQueue {
         props: { isRoot: true },
       });
     } catch (error) {
-      logger.error(`Sentinel: failed to enqueue investigation run: ${error}`);
-      return;
+      logger.error(`AI: failed to enqueue investigation run: ${error}`);
+      return null;
     }
 
     // Inline kick — preserves the 1-3 minute RCA latency on the happy path.
@@ -150,11 +183,14 @@ export default class SentinelInvestigationQueue {
       attemptCount: 0,
       triggeredByIncidentId: data.subjectIncidentId,
       triggeredByAlertId: data.subjectAlertId,
+      triggeredByAiInsightId: data.subjectAIInsightId,
     }).catch((error: Error) => {
       logger.error(
-        `Sentinel: inline processing of queued run ${createdRun.id?.toString()} failed: ${error}`,
+        `AI: inline processing of queued run ${createdRun.id?.toString()} failed: ${error}`,
       );
     });
+
+    return createdRun.id || null;
   }
 
   /*
@@ -179,7 +215,7 @@ export default class SentinelInvestigationQueue {
       await this.dispatch(run, attempt);
     } catch (error) {
       logger.error(
-        `Sentinel: failed to process queued run ${run.id.toString()}: ${error}`,
+        `AI: failed to process queued run ${run.id.toString()}: ${error}`,
       );
     }
   }
@@ -233,6 +269,7 @@ export default class SentinelInvestigationQueue {
         attemptCount: true,
         triggeredByIncidentId: true,
         triggeredByAlertId: true,
+        triggeredByAiInsightId: true,
       },
       sort: { createdAt: SortOrder.Ascending },
       limit: POLLER_BATCH_SIZE,
@@ -247,6 +284,7 @@ export default class SentinelInvestigationQueue {
         attemptCount: queued.attemptCount || 0,
         triggeredByIncidentId: queued.triggeredByIncidentId,
         triggeredByAlertId: queued.triggeredByAlertId,
+        triggeredByAiInsightId: queued.triggeredByAiInsightId,
       };
 
       try {
@@ -266,12 +304,12 @@ export default class SentinelInvestigationQueue {
          */
         this.dispatch(ref, attempt).catch((error: Error) => {
           logger.error(
-            `Sentinel: detached execution of run ${ref.id.toString()} failed: ${error}`,
+            `AI: detached execution of run ${ref.id.toString()} failed: ${error}`,
           );
         });
       } catch (error) {
         logger.error(
-          `Sentinel: poller failed on queued run ${ref.id.toString()}: ${error}`,
+          `AI: poller failed on queued run ${ref.id.toString()}: ${error}`,
         );
       }
     }
@@ -304,7 +342,7 @@ export default class SentinelInvestigationQueue {
 
       if (requeued > 0) {
         logger.debug(
-          `Sentinel: requeued run ${data.aiRunId.toString()} after attempt ${data.attemptCount} failed.`,
+          `AI: requeued run ${data.aiRunId.toString()} after attempt ${data.attemptCount} failed.`,
         );
       }
       return;
@@ -361,9 +399,10 @@ export default class SentinelInvestigationQueue {
   }
 
   /*
-   * The claim-time cost gates: concurrency cap and daily budget. A run
-   * failing these stays Queued — the poller retries and the TTL expires
-   * what never fits. Fails cheap (skip) on gate errors.
+   * The claim-time cost gates: concurrency cap, the preventive-lane sub-cap
+   * and the daily budget. A run failing these stays Queued — the poller
+   * retries and the TTL expires what never fits. Fails cheap (skip) on gate
+   * errors.
    */
   private static async passesClaimGates(run: QueuedRunRef): Promise<boolean> {
     // Per-project cap override, defaulting to 3 and clamped to [1, 25].
@@ -395,17 +434,53 @@ export default class SentinelInvestigationQueue {
 
     if (runningCount >= concurrencyCap) {
       logger.debug(
-        `Sentinel: leaving run ${run.id.toString()} queued — ${runningCount} investigations already running (cap: ${concurrencyCap}).`,
+        `AI: leaving run ${run.id.toString()} queued — ${runningCount} investigations already running (cap: ${concurrencyCap}).`,
       );
       return false;
     }
 
+    /*
+     * The preventive lane's sub-cap: insight triage may never occupy the
+     * slot(s) reserved for incident/alert RCA (see
+     * INSIGHT_TRIAGE_RESERVED_SLOTS). Floored at 1 so a project running at
+     * the minimum cap of 1 still triages its insights (slower, one at a
+     * time) instead of deadlocking the lane entirely — the interactive lane
+     * keeps its priority through the global cap check above, which a single
+     * running triage run cannot outlast for long (triage is read-only and
+     * short).
+     */
+    if (run.triggeredByAiInsightId) {
+      const triageCap: number = Math.max(
+        1,
+        concurrencyCap - INSIGHT_TRIAGE_RESERVED_SLOTS,
+      );
+
+      const runningTriageCount: number = (
+        await AIRunService.countBy({
+          query: {
+            projectId: run.projectId,
+            runType: AIRunType.Investigation,
+            status: AIRunStatus.Running,
+            triggeredByAiInsightId: QueryHelper.notNull(),
+          },
+          props: { isRoot: true },
+        })
+      ).toNumber();
+
+      if (runningTriageCount >= triageCap) {
+        logger.debug(
+          `AI: leaving insight triage run ${run.id.toString()} queued — ${runningTriageCount} triage runs already running (triage lane cap: ${triageCap} of ${concurrencyCap}).`,
+        );
+        return false;
+      }
+    }
+
     const budget: AutonomousBudgetStatus =
       await AIService.getAutonomousDailyBudgetStatus(run.projectId);
 
     if (budget.exhausted) {
       logger.debug(
-        `Sentinel: leaving run ${run.id.toString()} queued — daily autonomous token budget exhausted.`,
+        `AI: leaving run ${run.id.toString()} queued — daily autonomous token budget exhausted.`,
       );
       return false;
     }
@@ -479,6 +554,26 @@ export default class SentinelInvestigationQueue {
       return;
     }
 
+    /*
+     * AI-insight triage runs carry their subject in
+     * triggeredByAiInsightId — they must be recognized here as
+     * subject-BEARING, or the fallthrough below would fail every one of
+     * them as subject-less.
+     */
+    if (run.triggeredByAiInsightId) {
+      const insightTriageRunner: typeof import("./Insights/InsightTriageRunner").default =
+        // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+        require("./Insights/InsightTriageRunner").default;
+
+      await insightTriageRunner.executeTriage({
+        aiRunId: run.id,
+        projectId: run.projectId,
+        sentinelInsightId: run.triggeredByAiInsightId,
+        attemptCount: attempt,
+      });
+      return;
+    }
+
     // A queued investigation without a subject cannot be executed.
     await AIRunService.attemptStatusTransition({
       aiRunId: run.id,

package/Server/Utils/AI/{Sentinel → SRE}/README.md

@@ -1,10 +1,10 @@
-# Sentinel — autonomous investigation engine
+# AI SRE — autonomous investigation engine
 
-Wake-on-signal AI SRE: when an incident or alert is created (and the project has opted in), Sentinel runs a read-only, budgeted, citation-minting investigation over the project's own telemetry and posts a root-cause analysis to the subject's feed, with a glass-box `AIRun`/`AIRunEvent` trail behind a live dashboard panel.
+Wake-on-signal AI SRE: when an incident or alert is created (and the project has opted in), AI runs a read-only, budgeted, citation-minting investigation over the project's own telemetry and posts a root-cause analysis to the subject's feed, with a glass-box `AIRun`/`AIRunEvent` trail behind a live dashboard panel.
 
-- `SentinelInvestigationEngine.ts` — shared run lifecycle, budgets, persona, quiet-mode confidence gate
+- `AIInvestigationEngine.ts` — shared run lifecycle, budgets, persona, quiet-mode confidence gate
 - `IncidentInvestigationRunner.ts` / `AlertInvestigationRunner.ts` — per-subject context assembly + result posting
 - `IncidentPostmortemRunner.ts` — auto-draft postmortem when an incident resolves
-- `SentinelMemory.ts` — recurrence context from past resolved incidents (retrieval-only v1)
+- `AIMemory.ts` — recurrence context from past resolved incidents (retrieval-only v1)
 
 **Roadmap & status:** [Internal/Roadmap/AISentinelExecution.md](../../../../../Internal/Roadmap/AISentinelExecution.md) (tracker — update it in the same PR when you change behavior here) and [Internal/Roadmap/AISentinelVision.md](../../../../../Internal/Roadmap/AISentinelVision.md) (strategy).

package/Server/Utils/AI/SRE/SubjectCodeFixRun.ts

@@ -0,0 +1,172 @@
+import ObjectID from "../../../../Types/ObjectID";
+import AIRunType from "../../../../Types/AI/AIRunType";
+import AIRunStatus from "../../../../Types/AI/AIRunStatus";
+import CodeFixTaskType from "../../../../Types/AI/CodeFixTaskType";
+import CodeFixTaskContext from "../../../../Types/AI/CodeFixTaskContext";
+import CodeRepositoryType from "../../../../Types/CodeRepository/CodeRepositoryType";
+import { LIMIT_PER_PROJECT } from "../../../../Types/Database/LimitMax";
+import AIRun from "../../../../Models/DatabaseModels/AIRun";
+import AIRunService from "../../../Services/AIRunService";
+import CodeRepositoryService from "../../../Services/CodeRepositoryService";
+import FixRunBudget from "../CodeFix/FixRunBudget";
+import QueryHelper from "../../../Types/Database/QueryHelper";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+
+/*
+ * Shared plumbing for the non-exception CodeFix recipes: the incident/alert
+ * subject recipes (ImproveInstrumentation, FixFromIncident) and the
+ * trace-evidence recipe (FixPerformance). All enqueue AIRun(CodeFix) rows
+ * keyed to something other than a telemetry exception, all need a
+ * GitHub-App-connected repository the agent can actually clone and push,
+ * and all dedupe to at most one non-terminal run per (context, recipe).
+ * The triggers differ (automatic-after-inconclusive vs. user button) — the
+ * enqueue idiom must not.
+ */
+export default class SubjectCodeFixRun {
+  /*
+   * Only GitHub-App-connected repositories can be cloned and pushed by the
+   * agent (mirrors CodeRepositoryService.resolveRepositoryForException).
+   */
+  @CaptureSpan()
+  public static async hasGitHubAppConnectedRepository(
+    projectId: ObjectID,
+  ): Promise<boolean> {
+    const connectedRepositoryCount: number = (
+      await CodeRepositoryService.countBy({
+        query: {
+          projectId,
+          repositoryHostedAt: CodeRepositoryType.GitHub,
+          gitHubAppInstallationId: QueryHelper.notNull(),
+        },
+        props: { isRoot: true },
+      })
+    ).toNumber();
+
+    return connectedRepositoryCount > 0;
+  }
+
+  /*
+   * Per-(subject, recipe) dedupe guard (mirrors the per-(exception,
+   * taskType) guard in TelemetryExceptionService): at most one non-terminal
+   * run of a recipe per incident/alert, so repeated triggers on the same
+   * subject never fan out into duplicate PRs. Exactly one of
+   * incidentId/alertId is expected.
+   */
+  @CaptureSpan()
+  public static async findNonTerminalRunForSubject(data: {
+    taskType: CodeFixTaskType;
+    incidentId?: ObjectID | undefined;
+    alertId?: ObjectID | undefined;
+  }): Promise<AIRun | null> {
+    return AIRunService.findOneBy({
+      query: {
+        runType: AIRunType.CodeFix,
+        codeFixTaskType: data.taskType,
+        ...(data.incidentId
+          ? { triggeredByIncidentId: data.incidentId }
+          : { triggeredByAlertId: data.alertId! }),
+        status: QueryHelper.notIn([
+          AIRunStatus.Completed,
+          AIRunStatus.Error,
+          AIRunStatus.Cancelled,
+          AIRunStatus.Stale,
+        ]),
+      },
+      select: { _id: true },
+      props: { isRoot: true },
+    });
+  }
+
+  /*
+   * Per-trace dedupe for the FixPerformance recipe: at most one
+   * non-terminal FixPerformance run per (project, traceId). The traceId
+   * lives inside the JSON taskContext, which the Query layer cannot filter
+   * on — so this scans the project's non-terminal FixPerformance runs
+   * (bounded and rare: they exist only between a user click and the PR)
+   * and matches the traceId in memory. Honest per-trace dedupe without a
+   * dedicated column.
+   */
+  @CaptureSpan()
+  public static async findNonTerminalPerformanceFixRunForTrace(data: {
+    projectId: ObjectID;
+    traceId: string;
+  }): Promise<AIRun | null> {
+    const activeRuns: Array<AIRun> = await AIRunService.findBy({
+      query: {
+        projectId: data.projectId,
+        runType: AIRunType.CodeFix,
+        codeFixTaskType: CodeFixTaskType.FixPerformance,
+        status: QueryHelper.notIn([
+          AIRunStatus.Completed,
+          AIRunStatus.Error,
+          AIRunStatus.Cancelled,
+          AIRunStatus.Stale,
+        ]),
+      },
+      select: { _id: true, taskContext: true },
+      limit: LIMIT_PER_PROJECT,
+      skip: 0,
+      props: { isRoot: true },
+    });
+
+    return (
+      activeRuns.find((run: AIRun): boolean => {
+        return run.taskContext?.traceId === data.traceId;
+      }) || null
+    );
+  }
+
+  /*
+   * Record the durable intent as a Queued CodeFix AIRun that the external
+   * agent worker claims via /ai-agent-task/get-pending-task. `userId` is
+   * attribution for human-triggered recipes (who clicked the button);
+   * automatic triggers pass none and stay system-authored. Subject-less
+   * recipes (FixPerformance) pass neither incidentId nor alertId and carry
+   * their trigger-time evidence in `taskContext` instead.
+   *
+   * Created as root: AIRun rows are server-written only (empty create ACL);
+   * callers must have gated access before enqueueing.
+   *
+   * Throws BadDataException when the project is over its daily fix-run
+   * budget (G11 guardrail, enforced here so EVERY subject/perf recipe is
+   * covered). User-triggered callers surface the message; the automatic
+   * instrumentation trigger pre-checks the budget and skips quietly, with
+   * its never-throws wrapper as the backstop.
+   */
+  @CaptureSpan()
+  public static async enqueueSubjectCodeFixRun(data: {
+    projectId: ObjectID;
+    taskType: CodeFixTaskType;
+    incidentId?: ObjectID | undefined;
+    alertId?: ObjectID | undefined;
+    userId?: ObjectID | undefined;
+    taskContext?: CodeFixTaskContext | undefined;
+  }): Promise<AIRun> {
+    await FixRunBudget.assertWithinBudget(data.projectId);
+
+    const run: AIRun = new AIRun();
+    run.projectId = data.projectId;
+    run.runType = AIRunType.CodeFix;
+    run.codeFixTaskType = data.taskType;
+    run.status = AIRunStatus.Queued;
+
+    if (data.incidentId) {
+      run.triggeredByIncidentId = data.incidentId;
+    } else if (data.alertId) {
+      run.triggeredByAlertId = data.alertId;
+    }
+
+    if (data.userId) {
+      run.userId = data.userId;
+    }
+
+    if (data.taskContext) {
+      run.taskContext = data.taskContext;
+    }
+
+    return AIRunService.create({
+      data: run,
+      props: { isRoot: true },
+    });
+  }
+}

package/Server/Utils/AI/Toolbox/{SentinelActionTools.ts → AIActionTools.ts}

@@ -28,7 +28,7 @@ import {
 } from "./ToolTypes";
 
 /*
- * Sentinel "action belt" — Phase 0.
+ * AI "action belt" — Phase 0.
  *
  * Mutating tools that let the copilot OPERATE the platform (not just answer
  * questions), each a thin wrapper over an already-RBAC-tested service method.
@@ -261,6 +261,27 @@ export const RunRunbookTool: ObservabilityTool = {
       );
     }
 
+    /*
+     * Only link an incident the caller can actually read in this project.
+     * Looking it up under the user's RBAC (not isRoot) rejects both a
+     * cross-tenant id and one the user has no access to — so the tool cannot
+     * be used to staple another project's incident onto an execution.
+     */
+    if (incidentId) {
+      const linkedIncident: Incident | null = await IncidentService.findOneById(
+        {
+          id: incidentId,
+          select: { _id: true },
+          props: ctx.props,
+        },
+      );
+      if (!linkedIncident) {
+        throw new BadDataException(
+          "Incident not found (or you do not have access to it).",
+        );
+      }
+    }
+
     const execution: RunbookExecution | null =
       await RunbookRuleEngineService.startRunbookFor({
         projectId: ctx.projectId,

package/Server/Utils/AI/Toolbox/Index.ts

@@ -33,7 +33,7 @@ import {
   RunRunbookTool,
   PostIncidentStatusUpdateTool,
   ChangeIncidentSeverityTool,
-} from "./SentinelActionTools";
+} from "./AIActionTools";
 import AIChatPermissionMode from "../../../../Types/AI/AIChatPermissionMode";
 
 export interface ToolCallOutcome {
@@ -77,7 +77,7 @@ export default class AIToolbox {
     ResolveIncidentTool,
     AcknowledgeAlertTool,
     ResolveAlertTool,
-    // Sentinel action belt (Phase 0) — operate the platform, not just answer.
+    // AI action belt (Phase 0) — operate the platform, not just answer.
     PageOnCallPolicyTool,
     RunRunbookTool,
     PostIncidentStatusUpdateTool,