@oneuptime/common 11.4.0 → 11.5.0

package/build/dist/Server/Utils/AI/Sentinel/InvestigationQueue.js

@@ -0,0 +1,428 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import OneUptimeDate from "../../../../Types/Date";
+import AIRunType from "../../../../Types/AI/AIRunType";
+import AIRunStatus from "../../../../Types/AI/AIRunStatus";
+import AIRun from "../../../../Models/DatabaseModels/AIRun";
+import AIRunService from "../../../Services/AIRunService";
+import ProjectService from "../../../Services/ProjectService";
+import AIService from "../../../Services/AIService";
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
+import QueryHelper from "../../../Types/Database/QueryHelper";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+/*
+ * Sentinel — the durable investigation queue (Phase 2's first item; Q1
+ * decided as the DB-claim pattern on AIRun rows, no new infrastructure).
+ *
+ * The problem it replaces: investigations ran as detached in-process
+ * promises, so a pod restart orphaned them silently (Deviations log D2).
+ *
+ * The shape:
+ *   1. enqueue() records the durable intent as an AIRun in status Queued
+ *      BEFORE any expensive work, then immediately tries to process it
+ *      in-process — same latency as before when the pod stays alive.
+ *   2. Claims go through AIRunService.attemptStatusTransition — one
+ *      conditional UPDATE guarded on BOTH status=Queued and the expected
+ *      attemptCount — so the enqueueing pod and the poller can race and
+ *      exactly one wins, and a stale queue snapshot can never claim (or
+ *      re-number) a run that moved on. attemptCount can therefore never
+ *      exceed MAX_INVESTIGATION_ATTEMPTS.
+ *   3. A Workers poller claims whatever the inline path left behind
+ *      (pod died, cap was full, budget was exhausted) and expires runs
+ *      that sat queued past their usefulness window. The poller claims
+ *      sequentially (so the concurrency cap sees each claim) but executes
+ *      detached, keeping the every-minute tick fast.
+ *   4. Failed attempts requeue (transient errors and stale heartbeats)
+ *      until MAX_ATTEMPTS, then finalize as Error/Stale — G9's retry
+ *      policy. Permanent errors (bad configuration) never retry.
+ *
+ * Mid-run message-level checkpointing is deliberately NOT here: a retried
+ * investigation re-runs from the top, which is safe because investigations
+ * are read-only until the single postAnalysis at the end.
+ */
+// Initial attempt + one retry.
+export const MAX_INVESTIGATION_ATTEMPTS = 2;
+/*
+ * G4 cost guardrail: at most this many investigations may be Running per
+ * project at once (per-project override in
+ * Project.aiMaxConcurrentInvestigations). Enforced at CLAIM time, so a storm
+ * queues (bounded by dedupe + severity gates + this TTL) and drains at cap
+ * rate instead of being dropped.
+ */
+export const DEFAULT_MAX_CONCURRENT_INVESTIGATIONS = 3;
+/*
+ * Clamp bounds for the per-project override. Pausing has its own switches
+ * (the opt-in toggles, daily limit 0), so the floor is 1, not 0.
+ */
+const MIN_CONCURRENT_INVESTIGATIONS = 1;
+const MAX_CONCURRENT_INVESTIGATIONS = 25;
+/*
+ * A first-pass RCA is only useful while the incident is fresh. Queued runs
+ * older than this are expired rather than run late — this also caps queue
+ * growth when the daily budget blocks claiming for hours.
+ */
+export const QUEUE_TTL_MINUTES = 30;
+// How many queued runs one poller tick will try to claim.
+const POLLER_BATCH_SIZE = 10;
+export default class SentinelInvestigationQueue {
+    /*
+     * Record the durable intent to investigate, then try to process it
+     * immediately (detached — the poller is the safety net if this pod dies).
+     * Callers have already passed the subject-specific gates (opt-in,
+     * severity floor, dedupe window).
+     */
+    static async enqueue(data) {
+        const { projectId } = data;
+        /*
+         * Budget quiet-skip at enqueue: when the daily budget is already
+         * exhausted there is no point recording intent that the TTL would
+         * expire anyway. Fails cheap (skip) like all the cost gates.
+         */
+        try {
+            const budget = await AIService.getAutonomousDailyBudgetStatus(projectId);
+            if (budget.exhausted) {
+                logger.debug(`Sentinel: not enqueueing investigation for project ${projectId.toString()} — daily autonomous token budget exhausted (${budget.usedTokensToday} of ${budget.limitInTokens} tokens used today).`);
+                return;
+            }
+        }
+        catch (error) {
+            logger.error(`Sentinel: budget check failed, not enqueueing investigation: ${error}`);
+            return;
+        }
+        const run = new AIRun();
+        run.projectId = projectId;
+        run.runType = AIRunType.Investigation;
+        run.status = AIRunStatus.Queued;
+        if (data.subjectIncidentId) {
+            run.triggeredByIncidentId = data.subjectIncidentId;
+        }
+        if (data.subjectAlertId) {
+            run.triggeredByAlertId = data.subjectAlertId;
+        }
+        if (data.subjectMonitorId) {
+            run.monitorId = data.subjectMonitorId;
+        }
+        let createdRun;
+        try {
+            createdRun = await AIRunService.create({
+                data: run,
+                props: { isRoot: true },
+            });
+        }
+        catch (error) {
+            logger.error(`Sentinel: failed to enqueue investigation run: ${error}`);
+            return;
+        }
+        // Inline kick — preserves the 1-3 minute RCA latency on the happy path.
+        this.processRun({
+            id: createdRun.id,
+            projectId,
+            attemptCount: 0,
+            triggeredByIncidentId: data.subjectIncidentId,
+            triggeredByAlertId: data.subjectAlertId,
+        }).catch((error) => {
+            var _a;
+            logger.error(`Sentinel: inline processing of queued run ${(_a = createdRun.id) === null || _a === void 0 ? void 0 : _a.toString()} failed: ${error}`);
+        });
+    }
+    /*
+     * Claim a queued run and execute it to completion. Safe to call from
+     * multiple places concurrently — the claim is one conditional UPDATE, so
+     * exactly one caller wins. Leaves the run Queued (for the poller / TTL)
+     * when the concurrency cap is full or the budget is exhausted.
+     */
+    static async processRun(run) {
+        try {
+            if (!(await this.passesClaimGates(run))) {
+                return;
+            }
+            const attempt = await this.claim(run);
+            if (attempt === null) {
+                return;
+            }
+            await this.dispatch(run, attempt);
+        }
+        catch (error) {
+            logger.error(`Sentinel: failed to process queued run ${run.id.toString()}: ${error}`);
+        }
+    }
+    /*
+     * One poller tick: expire runs that queued past their usefulness window,
+     * then claim a batch of the rest. Claims happen sequentially — each claim
+     * sets Running immediately, so the next iteration's concurrency-cap count
+     * sees it — but execution is detached so the tick finishes in seconds
+     * instead of holding the job open for the length of the investigations.
+     * Called every minute from Workers; also the recovery path for runs
+     * orphaned by pod restarts.
+     */
+    static async processQueuedRuns() {
+        const expiryThreshold = OneUptimeDate.getSomeMinutesAgo(QUEUE_TTL_MINUTES);
+        const expiredRuns = await AIRunService.findBy({
+            query: {
+                runType: AIRunType.Investigation,
+                status: AIRunStatus.Queued,
+                createdAt: QueryHelper.lessThan(expiryThreshold),
+            },
+            select: { _id: true },
+            limit: 100,
+            skip: 0,
+            props: { isRoot: true },
+        });
+        for (const expired of expiredRuns) {
+            await AIRunService.attemptStatusTransition({
+                aiRunId: expired.id,
+                fromStatus: AIRunStatus.Queued,
+                set: {
+                    status: AIRunStatus.Cancelled,
+                    completedAt: OneUptimeDate.getCurrentDate(),
+                    errorMessage: `Expired in the investigation queue after ${QUEUE_TTL_MINUTES} minutes — a first-pass analysis this late would no longer be useful. The project may have been at its concurrency cap or daily token budget.`,
+                },
+            });
+        }
+        const queuedRuns = await AIRunService.findBy({
+            query: {
+                runType: AIRunType.Investigation,
+                status: AIRunStatus.Queued,
+            },
+            select: {
+                _id: true,
+                projectId: true,
+                attemptCount: true,
+                triggeredByIncidentId: true,
+                triggeredByAlertId: true,
+            },
+            sort: { createdAt: SortOrder.Ascending },
+            limit: POLLER_BATCH_SIZE,
+            skip: 0,
+            props: { isRoot: true },
+        });
+        for (const queued of queuedRuns) {
+            const ref = {
+                id: queued.id,
+                projectId: queued.projectId,
+                attemptCount: queued.attemptCount || 0,
+                triggeredByIncidentId: queued.triggeredByIncidentId,
+                triggeredByAlertId: queued.triggeredByAlertId,
+            };
+            try {
+                if (!(await this.passesClaimGates(ref))) {
+                    continue;
+                }
+                const attempt = await this.claim(ref);
+                if (attempt === null) {
+                    continue;
+                }
+                /*
+                 * Execute detached — the run is claimed (Running + heartbeat), so
+                 * the sweeper owns recovery if this pod dies mid-flight.
+                 */
+                this.dispatch(ref, attempt).catch((error) => {
+                    logger.error(`Sentinel: detached execution of run ${ref.id.toString()} failed: ${error}`);
+                });
+            }
+            catch (error) {
+                logger.error(`Sentinel: poller failed on queued run ${ref.id.toString()}: ${error}`);
+            }
+        }
+    }
+    /*
+     * Finalize a failed attempt: requeue transient failures while attempts
+     * remain, otherwise mark the run Error. The transition guards on Running
+     * so a run that already completed (e.g. only postAnalysis failed) is
+     * never clobbered or re-run.
+     */
+    static async failOrRequeue(data) {
+        const truncatedMessage = data.errorMessage.substring(0, 400);
+        if (!data.isPermanent && data.attemptCount < MAX_INVESTIGATION_ATTEMPTS) {
+            const requeued = await AIRunService.attemptStatusTransition({
+                aiRunId: data.aiRunId,
+                fromStatus: AIRunStatus.Running,
+                set: {
+                    status: AIRunStatus.Queued,
+                    errorMessage: `Attempt ${data.attemptCount} failed and the run was requeued: ${truncatedMessage}`,
+                },
+            });
+            if (requeued > 0) {
+                logger.debug(`Sentinel: requeued run ${data.aiRunId.toString()} after attempt ${data.attemptCount} failed.`);
+            }
+            return;
+        }
+        await AIRunService.attemptStatusTransition({
+            aiRunId: data.aiRunId,
+            fromStatus: AIRunStatus.Running,
+            set: {
+                status: AIRunStatus.Error,
+                completedAt: OneUptimeDate.getCurrentDate(),
+                errorMessage: truncatedMessage,
+            },
+        });
+    }
+    /*
+     * Called by the stale-run sweeper for an Investigation run whose
+     * heartbeat went silent (the pod running it died). Requeues while
+     * attempts remain; otherwise marks it Stale as before.
+     */
+    static async requeueOrMarkStale(run) {
+        if ((run.attemptCount || 0) < MAX_INVESTIGATION_ATTEMPTS) {
+            const requeued = await AIRunService.attemptStatusTransition({
+                aiRunId: run.id,
+                fromStatus: AIRunStatus.Running,
+                set: {
+                    status: AIRunStatus.Queued,
+                    errorMessage: `Attempt ${run.attemptCount} stopped reporting progress (the server processing it may have restarted) and the run was requeued.`,
+                },
+            });
+            if (requeued > 0) {
+                return "requeued";
+            }
+        }
+        await AIRunService.attemptStatusTransition({
+            aiRunId: run.id,
+            fromStatus: AIRunStatus.Running,
+            set: {
+                status: AIRunStatus.Stale,
+                completedAt: OneUptimeDate.getCurrentDate(),
+                errorMessage: "The run stopped reporting progress and was marked as stale after exhausting its retry attempts. The server processing it may have restarted.",
+            },
+        });
+        return "stale";
+    }
+    /*
+     * The claim-time cost gates: concurrency cap and daily budget. A run
+     * failing these stays Queued — the poller retries and the TTL expires
+     * what never fits. Fails cheap (skip) on gate errors.
+     */
+    static async passesClaimGates(run) {
+        var _a;
+        // Per-project cap override, defaulting to 3 and clamped to [1, 25].
+        const project = await ProjectService.findOneById({
+            id: run.projectId,
+            select: { aiMaxConcurrentInvestigations: true },
+            props: { isRoot: true },
+        });
+        const concurrencyCap = Math.min(MAX_CONCURRENT_INVESTIGATIONS, Math.max(MIN_CONCURRENT_INVESTIGATIONS, (_a = project === null || project === void 0 ? void 0 : project.aiMaxConcurrentInvestigations) !== null && _a !== void 0 ? _a : DEFAULT_MAX_CONCURRENT_INVESTIGATIONS));
+        const runningCount = (await AIRunService.countBy({
+            query: {
+                projectId: run.projectId,
+                runType: AIRunType.Investigation,
+                status: AIRunStatus.Running,
+            },
+            props: { isRoot: true },
+        })).toNumber();
+        if (runningCount >= concurrencyCap) {
+            logger.debug(`Sentinel: leaving run ${run.id.toString()} queued — ${runningCount} investigations already running (cap: ${concurrencyCap}).`);
+            return false;
+        }
+        const budget = await AIService.getAutonomousDailyBudgetStatus(run.projectId);
+        if (budget.exhausted) {
+            logger.debug(`Sentinel: leaving run ${run.id.toString()} queued — daily autonomous token budget exhausted.`);
+            return false;
+        }
+        return true;
+    }
+    /*
+     * The atomic claim: Queued -> Running, guarded on the attemptCount the
+     * caller observed, so a stale queue snapshot can neither double-claim
+     * nor reset the attempt numbering. Returns the claimed attempt number,
+     * or null when another actor won.
+     */
+    static async claim(run) {
+        const attempt = (run.attemptCount || 0) + 1;
+        const claimedCount = await AIRunService.attemptStatusTransition({
+            aiRunId: run.id,
+            fromStatus: AIRunStatus.Queued,
+            expectedAttemptCount: run.attemptCount || 0,
+            set: {
+                status: AIRunStatus.Running,
+                startedAt: OneUptimeDate.getCurrentDate(),
+                lastHeartbeatAt: OneUptimeDate.getCurrentDate(),
+                attemptCount: attempt,
+            },
+        });
+        if (claimedCount === 0) {
+            return null;
+        }
+        return attempt;
+    }
+    // Route a claimed run to its subject's executor.
+    static async dispatch(run, attempt) {
+        if (run.triggeredByIncidentId) {
+            /*
+             * Lazy require: the runners import this queue to enqueue, so a
+             * top-level import here would be circular at module-init time
+             * (same pattern as DatabaseService -> AuditLogService).
+             */
+            const incidentRunner = 
+            // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+            require("./IncidentInvestigationRunner").default;
+            await incidentRunner.executeInvestigation({
+                aiRunId: run.id,
+                projectId: run.projectId,
+                incidentId: run.triggeredByIncidentId,
+                attemptCount: attempt,
+            });
+            return;
+        }
+        if (run.triggeredByAlertId) {
+            const alertRunner = 
+            // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+            require("./AlertInvestigationRunner").default;
+            await alertRunner.executeInvestigation({
+                aiRunId: run.id,
+                projectId: run.projectId,
+                alertId: run.triggeredByAlertId,
+                attemptCount: attempt,
+            });
+            return;
+        }
+        // A queued investigation without a subject cannot be executed.
+        await AIRunService.attemptStatusTransition({
+            aiRunId: run.id,
+            fromStatus: AIRunStatus.Running,
+            set: {
+                status: AIRunStatus.Error,
+                completedAt: OneUptimeDate.getCurrentDate(),
+                errorMessage: "Queued investigation has no subject to investigate.",
+            },
+        });
+    }
+}
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], SentinelInvestigationQueue, "enqueue", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], SentinelInvestigationQueue, "processRun", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", []),
+    __metadata("design:returntype", Promise)
+], SentinelInvestigationQueue, "processQueuedRuns", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], SentinelInvestigationQueue, "failOrRequeue", null);
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], SentinelInvestigationQueue, "requeueOrMarkStale", null);
+//# sourceMappingURL=InvestigationQueue.js.map

package/build/dist/Server/Utils/AI/Sentinel/InvestigationQueue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InvestigationQueue.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/Sentinel/InvestigationQueue.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,aAAa,MAAM,wBAAwB,CAAC;AACnD,OAAO,SAAS,MAAM,gCAAgC,CAAC;AACvD,OAAO,WAAW,MAAM,kCAAkC,CAAC;AAC3D,OAAO,KAAK,MAAM,yCAAyC,CAAC;AAE5D,OAAO,YAAY,MAAM,gCAAgC,CAAC;AAC1D,OAAO,cAAc,MAAM,kCAAkC,CAAC;AAC9D,OAAO,SAAqC,MAAM,6BAA6B,CAAC;AAChF,OAAO,SAAS,MAAM,0CAA0C,CAAC;AACjE,OAAO,WAAW,MAAM,qCAAqC,CAAC;AAC9D,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,WAAW,MAAM,6BAA6B,CAAC;AAEtD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,+BAA+B;AAC/B,MAAM,CAAC,MAAM,0BAA0B,GAAW,CAAC,CAAC;AAEpD;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,qCAAqC,GAAW,CAAC,CAAC;AAC/D;;;GAGG;AACH,MAAM,6BAA6B,GAAW,CAAC,CAAC;AAChD,MAAM,6BAA6B,GAAW,EAAE,CAAC;AAEjD;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAW,EAAE,CAAC;AAE5C,0DAA0D;AAC1D,MAAM,iBAAiB,GAAW,EAAE,CAAC;AAUrC,MAAM,CAAC,OAAO,OAAO,0BAA0B;IAC7C;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,IAK3B;QACC,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;QAE3B;;;;WAIG;QACH,IAAI,CAAC;YACH,MAAM,MAAM,GACV,MAAM,SAAS,CAAC,8BAA8B,CAAC,SAAS,CAAC,CAAC;YAE5D,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,MAAM,CAAC,KAAK,CACV,sDAAsD,SAAS,CAAC,QAAQ,EAAE,+CAA+C,MAAM,CAAC,eAAe,OAAO,MAAM,CAAC,aAAa,sBAAsB,CACjM,CAAC;gBACF,OAAO;YACT,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV,gEAAgE,KAAK,EAAE,CACxE,CAAC;YACF,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAU,IAAI,KAAK,EAAE,CAAC;QAC/B,GAAG,CAAC,SAAS,GAAG,SAAS,CAAC;QAC1B,GAAG,CAAC,OAAO,GAAG,SAAS,CAAC,aAAa,CAAC;QACtC,GAAG,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC;QAEhC,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,GAAG,CAAC,qBAAqB,GAAG,IAAI,CAAC,iBAAiB,CAAC;QACrD,CAAC;QACD,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,GAAG,CAAC,kBAAkB,GAAG,IAAI,CAAC,cAAc,CAAC;QAC/C,CAAC;QACD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,GAAG,CAAC,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC;QACxC,CAAC;QAED,IAAI,UAAiB,CAAC;QACtB,IAAI,CAAC;YACH,UAAU,GAAG,MAAM,YAAY,CAAC,MAAM,CAAC;gBACrC,IAAI,EAAE,GAAG;gBACT,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CAAC,kDAAkD,KAAK,EAAE,CAAC,CAAC;YACxE,OAAO;QACT,CAAC;QAED,wEAAwE;QACxE,IAAI,CAAC,UAAU,CAAC;YACd,EAAE,EAAE,UAAU,CAAC,EAAG;YAClB,SAAS;YACT,YAAY,EAAE,CAAC;YACf,qBAAqB,EAAE,IAAI,CAAC,iBAAiB;YAC7C,kBAAkB,EAAE,IAAI,CAAC,cAAc;SACxC,CAAC,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE;;YACxB,MAAM,CAAC,KAAK,CACV,6CAA6C,MAAA,UAAU,CAAC,EAAE,0CAAE,QAAQ,EAAE,YAAY,KAAK,EAAE,CAC1F,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,GAAiB;QAC9C,IAAI,CAAC;YACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;gBACxC,OAAO;YACT,CAAC;YAED,MAAM,OAAO,GAAkB,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAErD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;gBACrB,OAAO;YACT,CAAC;YAED,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACpC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,CAAC,KAAK,CACV,0CAA0C,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CACxE,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;;;;OAQG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,iBAAiB;QACnC,MAAM,eAAe,GACnB,aAAa,CAAC,iBAAiB,CAAC,iBAAiB,CAAC,CAAC;QAErD,MAAM,WAAW,GAAiB,MAAM,YAAY,CAAC,MAAM,CAAC;YAC1D,KAAK,EAAE;gBACL,OAAO,EAAE,SAAS,CAAC,aAAa;gBAChC,MAAM,EAAE,WAAW,CAAC,MAAM;gBAC1B,SAAS,EAAE,WAAW,CAAC,QAAQ,CAAC,eAAe,CAAC;aACjD;YACD,MAAM,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE;YACrB,KAAK,EAAE,GAAG;YACV,IAAI,EAAE,CAAC;YACP,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CAAC;QAEH,KAAK,MAAM,OAAO,IAAI,WAAW,EAAE,CAAC;YAClC,MAAM,YAAY,CAAC,uBAAuB,CAAC;gBACzC,OAAO,EAAE,OAAO,CAAC,EAAG;gBACpB,UAAU,EAAE,WAAW,CAAC,MAAM;gBAC9B,GAAG,EAAE;oBACH,MAAM,EAAE,WAAW,CAAC,SAAS;oBAC7B,WAAW,EAAE,aAAa,CAAC,cAAc,EAAE;oBAC3C,YAAY,EAAE,4CAA4C,iBAAiB,+IAA+I;iBAC3N;aACF,CAAC,CAAC;QACL,CAAC;QAED,MAAM,UAAU,GAAiB,MAAM,YAAY,CAAC,MAAM,CAAC;YACzD,KAAK,EAAE;gBACL,OAAO,EAAE,SAAS,CAAC,aAAa;gBAChC,MAAM,EAAE,WAAW,CAAC,MAAM;aAC3B;YACD,MAAM,EAAE;gBACN,GAAG,EAAE,IAAI;gBACT,SAAS,EAAE,IAAI;gBACf,YAAY,EAAE,IAAI;gBAClB,qBAAqB,EAAE,IAAI;gBAC3B,kBAAkB,EAAE,IAAI;aACzB;YACD,IAAI,EAAE,EAAE,SAAS,EAAE,SAAS,CAAC,SAAS,EAAE;YACxC,KAAK,EAAE,iBAAiB;YACxB,IAAI,EAAE,CAAC;YACP,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CAAC;QAEH,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,MAAM,GAAG,GAAiB;gBACxB,EAAE,EAAE,MAAM,CAAC,EAAG;gBACd,SAAS,EAAE,MAAM,CAAC,SAAU;gBAC5B,YAAY,EAAE,MAAM,CAAC,YAAY,IAAI,CAAC;gBACtC,qBAAqB,EAAE,MAAM,CAAC,qBAAqB;gBACnD,kBAAkB,EAAE,MAAM,CAAC,kBAAkB;aAC9C,CAAC;YAEF,IAAI,CAAC;gBACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;oBACxC,SAAS;gBACX,CAAC;gBAED,MAAM,OAAO,GAAkB,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAErD,IAAI,OAAO,KAAK,IAAI,EAAE,CAAC;oBACrB,SAAS;gBACX,CAAC;gBAED;;;mBAGG;gBACH,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAY,EAAE,EAAE;oBACjD,MAAM,CAAC,KAAK,CACV,uCAAuC,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,EAAE,CAC5E,CAAC;gBACJ,CAAC,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,CAAC,KAAK,CACV,yCAAyC,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,KAAK,KAAK,EAAE,CACvE,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAKjC;QACC,MAAM,gBAAgB,GAAW,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;QAErE,IAAI,CAAC,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,YAAY,GAAG,0BAA0B,EAAE,CAAC;YACxE,MAAM,QAAQ,GAAW,MAAM,YAAY,CAAC,uBAAuB,CAAC;gBAClE,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,UAAU,EAAE,WAAW,CAAC,OAAO;gBAC/B,GAAG,EAAE;oBACH,MAAM,EAAE,WAAW,CAAC,MAAM;oBAC1B,YAAY,EAAE,WAAW,IAAI,CAAC,YAAY,qCAAqC,gBAAgB,EAAE;iBAClG;aACF,CAAC,CAAC;YAEH,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBACjB,MAAM,CAAC,KAAK,CACV,0BAA0B,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,kBAAkB,IAAI,CAAC,YAAY,UAAU,CAC/F,CAAC;YACJ,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,YAAY,CAAC,uBAAuB,CAAC;YACzC,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,UAAU,EAAE,WAAW,CAAC,OAAO;YAC/B,GAAG,EAAE;gBACH,MAAM,EAAE,WAAW,CAAC,KAAK;gBACzB,WAAW,EAAE,aAAa,CAAC,cAAc,EAAE;gBAC3C,YAAY,EAAE,gBAAgB;aAC/B;SACF,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,kBAAkB,CAAC,GAGtC;QACC,IAAI,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,0BAA0B,EAAE,CAAC;YACzD,MAAM,QAAQ,GAAW,MAAM,YAAY,CAAC,uBAAuB,CAAC;gBAClE,OAAO,EAAE,GAAG,CAAC,EAAE;gBACf,UAAU,EAAE,WAAW,CAAC,OAAO;gBAC/B,GAAG,EAAE;oBACH,MAAM,EAAE,WAAW,CAAC,MAAM;oBAC1B,YAAY,EAAE,WAAW,GAAG,CAAC,YAAY,qGAAqG;iBAC/I;aACF,CAAC,CAAC;YAEH,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;gBACjB,OAAO,UAAU,CAAC;YACpB,CAAC;QACH,CAAC;QAED,MAAM,YAAY,CAAC,uBAAuB,CAAC;YACzC,OAAO,EAAE,GAAG,CAAC,EAAE;YACf,UAAU,EAAE,WAAW,CAAC,OAAO;YAC/B,GAAG,EAAE;gBACH,MAAM,EAAE,WAAW,CAAC,KAAK;gBACzB,WAAW,EAAE,aAAa,CAAC,cAAc,EAAE;gBAC3C,YAAY,EACV,8IAA8I;aACjJ;SACF,CAAC,CAAC;QAEH,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACK,MAAM,CAAC,KAAK,CAAC,gBAAgB,CAAC,GAAiB;;QACrD,oEAAoE;QACpE,MAAM,OAAO,GAAmB,MAAM,cAAc,CAAC,WAAW,CAAC;YAC/D,EAAE,EAAE,GAAG,CAAC,SAAS;YACjB,MAAM,EAAE,EAAE,6BAA6B,EAAE,IAAI,EAAE;YAC/C,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CAAC;QAEH,MAAM,cAAc,GAAW,IAAI,CAAC,GAAG,CACrC,6BAA6B,EAC7B,IAAI,CAAC,GAAG,CACN,6BAA6B,EAC7B,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,6BAA6B,mCACpC,qCAAqC,CACxC,CACF,CAAC;QAEF,MAAM,YAAY,GAAW,CAC3B,MAAM,YAAY,CAAC,OAAO,CAAC;YACzB,KAAK,EAAE;gBACL,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,OAAO,EAAE,SAAS,CAAC,aAAa;gBAChC,MAAM,EAAE,WAAW,CAAC,OAAO;aAC5B;YACD,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CACH,CAAC,QAAQ,EAAE,CAAC;QAEb,IAAI,YAAY,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,CAAC,KAAK,CACV,yBAAyB,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,aAAa,YAAY,yCAAyC,cAAc,IAAI,CAC/H,CAAC;YACF,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,MAAM,GACV,MAAM,SAAS,CAAC,8BAA8B,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAEhE,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,MAAM,CAAC,KAAK,CACV,yBAAyB,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,oDAAoD,CAC/F,CAAC;YACF,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACK,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,GAAiB;QAC1C,MAAM,OAAO,GAAW,CAAC,GAAG,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAEpD,MAAM,YAAY,GAAW,MAAM,YAAY,CAAC,uBAAuB,CAAC;YACtE,OAAO,EAAE,GAAG,CAAC,EAAE;YACf,UAAU,EAAE,WAAW,CAAC,MAAM;YAC9B,oBAAoB,EAAE,GAAG,CAAC,YAAY,IAAI,CAAC;YAC3C,GAAG,EAAE;gBACH,MAAM,EAAE,WAAW,CAAC,OAAO;gBAC3B,SAAS,EAAE,aAAa,CAAC,cAAc,EAAE;gBACzC,eAAe,EAAE,aAAa,CAAC,cAAc,EAAE;gBAC/C,YAAY,EAAE,OAAO;aACtB;SACF,CAAC,CAAC;QAEH,IAAI,YAAY,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,iDAAiD;IACzC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAC3B,GAAiB,EACjB,OAAe;QAEf,IAAI,GAAG,CAAC,qBAAqB,EAAE,CAAC;YAC9B;;;;eAIG;YACH,MAAM,cAAc;YAClB,qGAAqG;YACrG,OAAO,CAAC,+BAA+B,CAAC,CAAC,OAAO,CAAC;YAEnD,MAAM,cAAc,CAAC,oBAAoB,CAAC;gBACxC,OAAO,EAAE,GAAG,CAAC,EAAE;gBACf,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,UAAU,EAAE,GAAG,CAAC,qBAAqB;gBACrC,YAAY,EAAE,OAAO;aACtB,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,IAAI,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC3B,MAAM,WAAW;YACf,qGAAqG;YACrG,OAAO,CAAC,4BAA4B,CAAC,CAAC,OAAO,CAAC;YAEhD,MAAM,WAAW,CAAC,oBAAoB,CAAC;gBACrC,OAAO,EAAE,GAAG,CAAC,EAAE;gBACf,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,OAAO,EAAE,GAAG,CAAC,kBAAkB;gBAC/B,YAAY,EAAE,OAAO;aACtB,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,+DAA+D;QAC/D,MAAM,YAAY,CAAC,uBAAuB,CAAC;YACzC,OAAO,EAAE,GAAG,CAAC,EAAE;YACf,UAAU,EAAE,WAAW,CAAC,OAAO;YAC/B,GAAG,EAAE;gBACH,MAAM,EAAE,WAAW,CAAC,KAAK;gBACzB,WAAW,EAAE,aAAa,CAAC,cAAc,EAAE;gBAC3C,YAAY,EAAE,qDAAqD;aACpE;SACF,CAAC,CAAC;IACL,CAAC;CACF;AAnZqB;IADnB,WAAW,EAAE;;;;+CAqEb;AASmB;IADnB,WAAW,EAAE;;;;kDAmBb;AAYmB;IADnB,WAAW,EAAE;;;;yDAkFb;AASmB;IADnB,WAAW,EAAE;;;;qDAoCb;AAQmB;IADnB,WAAW,EAAE;;;;0DAgCb"}

package/build/dist/Server/Utils/AI/Sentinel/SentinelInvestigationEngine.js

@@ -9,15 +9,14 @@ var __metadata = (this && this.__metadata) || function (k, v) {
 };
 import ObjectID from "../../../../Types/ObjectID";
 import OneUptimeDate from "../../../../Types/Date";
-import AIRunType from "../../../../Types/AI/AIRunType";
 import AIRunStatus from "../../../../Types/AI/AIRunStatus";
 import AIRunEventType from "../../../../Types/AI/AIRunEventType";
-import AIRun from "../../../../Models/DatabaseModels/AIRun";
 import AIRunEvent from "../../../../Models/DatabaseModels/AIRunEvent";
 import AIRunService from "../../../Services/AIRunService";
 import AIRunEventService from "../../../Services/AIRunEventService";
 import ProjectService from "../../../Services/ProjectService";
 import LlmProviderService from "../../../Services/LlmProviderService";
+import SentinelInvestigationQueue from "./InvestigationQueue";
 import ObservabilityAssistant from "../Chat/ObservabilityAssistant";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
@@ -47,6 +46,12 @@ const MAX_OUTPUT_TOKENS = 2000;
  * a non-answer should never loudly page the on-call.
  */
 const INCONCLUSIVE_RE = /inconclusive[^.\n]*insufficient signal/i;
+/*
+ * Failures a retry cannot fix within the run's usefulness window: missing/
+ * broken provider configuration and budget exhaustion (both messages minted
+ * by our own gating in AIService/LLMService, so they are stable to match).
+ */
+const PERMANENT_FAILURE_RE = /no llm provider configured|llm provider type is not configured|token budget exhausted/i;
 // Maps a live agent step to the AIRunEvent type persisted for the glass-box trail.
 const STEP_EVENT_TYPE = {
     llm_started: AIRunEventType.LlmCallStarted,
@@ -59,7 +64,7 @@ const INVESTIGATION_PERSONA = `You are "Sentinel", OneUptime's autonomous AI Sit
 
 Investigate like a senior on-call engineer:
 - Start from the affected monitors/services named in the signal.
-- Use your read tools to inspect the telemetry AROUND the signal's creation time: recent exceptions and their trends, error/latency metrics versus their normal range, failing traces, relevant logs, and recent changes / deploys.
+- Use your read tools to inspect the telemetry AROUND the signal's creation time: recent exceptions and their trends, error/latency metrics versus their normal range (use baseline_anomaly to judge a metric against its learned hour-of-week baseline quantitatively instead of eyeballing), failing traces, relevant logs, and recent changes / deploys.
 - Form the single most likely root-cause hypothesis. If the evidence is inconclusive, say so plainly and list what you checked — do NOT guess a cause the data does not support.
 - If the context lists past resolved incidents, check whether this is a RECURRENCE. If the current signal matches one, say so explicitly, reference that incident number, and note how it was resolved before — but still verify against the current telemetry.
 
@@ -107,37 +112,27 @@ export default class SentinelInvestigationEngine {
         return true;
     }
     /*
-     * Run one investigation end-to-end. Never throws — failures are logged and
-     * recorded on the AIRun.
+     * Execute an already-CLAIMED (Running) investigation run end-to-end.
+     * Called by SentinelInvestigationQueue after a successful CAS claim —
+     * cap/budget gating and run creation live in the queue now. Never throws;
+     * failures are handed to the queue's retry policy (failOrRequeue).
      */
-    static async investigate(request) {
-        const { projectId } = request;
-        // Record the run up front so it is auditable even if it fails.
-        const run = new AIRun();
-        run.projectId = projectId;
-        run.runType = AIRunType.Investigation;
-        run.status = AIRunStatus.Running;
-        run.startedAt = OneUptimeDate.getCurrentDate();
-        run.lastHeartbeatAt = OneUptimeDate.getCurrentDate();
-        if (request.subjectIncidentId) {
-            run.triggeredByIncidentId = request.subjectIncidentId;
-        }
-        if (request.subjectAlertId) {
-            run.triggeredByAlertId = request.subjectAlertId;
-        }
-        let createdRun;
+    static async executeRun(data) {
+        const { aiRunId, projectId, request } = data;
+        /*
+         * Retried runs already have events from earlier attempts; continue the
+         * sequence so the glass-box trail stays ordered and shows every attempt.
+         */
+        let sequence = 0;
         try {
-            createdRun = await AIRunService.create({
-                data: run,
+            sequence = (await AIRunEventService.countBy({
+                query: { aiRunId },
                 props: { isRoot: true },
-            });
+            })).toNumber();
         }
         catch (error) {
-            logger.error(`Sentinel: failed to create investigation run: ${error}`);
-            return;
+            logger.error(`Sentinel: failed to read event count for run ${aiRunId.toString()}; starting sequence at 0: ${error}`);
         }
-        const aiRunId = createdRun.id;
-        let sequence = 0;
         await this.emitEvent({
             projectId,
             aiRunId,
@@ -168,10 +163,12 @@ export default class SentinelInvestigationEngine {
                 resultSummary,
                 citationId: step.citationId,
             });
-            // Keep the run visibly alive for the stale-run sweeper + live UI.
-            if (step.type === "llm_started") {
-                await this.touchHeartbeat(aiRunId);
-            }
+            /*
+             * Keep the run visibly alive for the stale-run sweeper + live UI on
+             * EVERY step — a slow self-hosted LLM call can approach the sweeper's
+             * timeout, so the heartbeat must be as frequent as we can make it.
+             */
+            await this.touchHeartbeat(aiRunId);
         };
         try {
             const result = await ObservabilityAssistant.answerQuestion({
@@ -187,9 +184,18 @@ export default class SentinelInvestigationEngine {
                 maxOutputTokens: MAX_OUTPUT_TOKENS,
                 onStep,
             });
-            await AIRunService.updateOneBy({
-                query: { _id: aiRunId.toString(), status: AIRunStatus.Running },
-                data: {
+            /*
+             * Atomic Running -> Completed. If we did NOT win this transition,
+             * another actor moved the run while we executed — most likely the
+             * stale sweeper falsely requeued it (slow LLM call outlasting the
+             * heartbeat window) and a second attempt is or will be running. In
+             * that case DO NOT post the analysis: the winning attempt will, and
+             * posting here would duplicate the RCA in the feed and workspace.
+             */
+            const completedCount = await AIRunService.attemptStatusTransition({
+                aiRunId,
+                fromStatus: AIRunStatus.Running,
+                set: {
                     status: AIRunStatus.Completed,
                     completedAt: OneUptimeDate.getCurrentDate(),
                     lastHeartbeatAt: OneUptimeDate.getCurrentDate(),
@@ -197,8 +203,11 @@ export default class SentinelInvestigationEngine {
                     toolCallCount: result.toolCallCount,
                     totalTokens: result.totalTokens,
                 },
-                props: { isRoot: true },
             });
+            if (completedCount === 0) {
+                logger.warn(`Sentinel: run ${aiRunId.toString()} finished but was no longer Running (likely requeued as stale mid-flight); skipping postAnalysis to avoid a duplicate RCA.`);
+                return;
+            }
             await this.emitEvent({
                 projectId,
                 aiRunId,
@@ -220,22 +229,29 @@ export default class SentinelInvestigationEngine {
         }
         catch (error) {
             const message = error instanceof Error ? error.message : String(error);
-            await AIRunService.updateOneBy({
-                query: { _id: aiRunId.toString(), status: AIRunStatus.Running },
-                data: {
-                    status: AIRunStatus.Error,
-                    completedAt: OneUptimeDate.getCurrentDate(),
-                    errorMessage: message.substring(0, 480),
-                },
-                props: { isRoot: true },
-            });
             await this.emitEvent({
                 projectId,
                 aiRunId,
                 sequence: sequence++,
                 eventType: AIRunEventType.RunFailed,
             });
-            logger.error(`Sentinel: investigation failed (run ${aiRunId.toString()}): ${message}`);
+            /*
+             * Hand the failure to the queue's retry policy: transient errors
+             * requeue while attempts remain; permanent ones finalize as Error
+             * since retrying cannot help. Classification is by message, NOT by
+             * exception type: LLMService wraps transient provider failures (429s,
+             * 5xx, timeouts) in BadDataException too, so type-based classification
+             * would wrongly make the exact failures retries exist for permanent.
+             * Only configuration/budget gating — which a retry cannot change
+             * within the run's usefulness window — counts as permanent.
+             */
+            await SentinelInvestigationQueue.failOrRequeue({
+                aiRunId,
+                attemptCount: data.attemptCount,
+                errorMessage: message,
+                isPermanent: PERMANENT_FAILURE_RE.test(message),
+            });
+            logger.error(`Sentinel: investigation attempt ${data.attemptCount} failed (run ${aiRunId.toString()}): ${message}`);
         }
     }
     // Wrap the agent's analysis with Sentinel branding + an evidence list.
@@ -307,5 +323,5 @@ __decorate([
     __metadata("design:type", Function),
     __metadata("design:paramtypes", [Object]),
     __metadata("design:returntype", Promise)
-], SentinelInvestigationEngine, "investigate", null);
+], SentinelInvestigationEngine, "executeRun", null);
 //# sourceMappingURL=SentinelInvestigationEngine.js.map