@oneuptime/common 11.5.3 → 11.5.5

package/Server/Utils/AI/{Sentinel/SentinelInvestigationEngine.ts → SRE/AIInvestigationEngine.ts}

@@ -14,7 +14,8 @@ import AIRunService from "../../../Services/AIRunService";
 import AIRunEventService from "../../../Services/AIRunEventService";
 import ProjectService from "../../../Services/ProjectService";
 import LlmProviderService from "../../../Services/LlmProviderService";
-import SentinelInvestigationQueue from "./InvestigationQueue";
+import AIInvestigationQueue from "./InvestigationQueue";
+import AIConfidenceSignal, { ConfidenceSignal } from "./ConfidenceSignal";
 import ObservabilityAssistant, {
   ObservabilityAssistantResult,
   ObservabilityAssistantStep,
@@ -24,14 +25,16 @@ import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
 /*
- * Sentinel — the shared autonomous-investigation engine.
+ * AI SRE — the shared autonomous-investigation engine.
  *
  * A single subject-agnostic core that powers "wake on signal" investigations
  * for both incidents and alerts (and, later, anomalies). It:
  *   1. records the run as an AIRun(Investigation) + AIRunEvents (audit trail),
  *   2. runs the existing read-only, tool-grounded, citation-minting agent loop
- *      (ObservabilityAssistant) with the Sentinel persona and a larger budget,
- *   3. brands the cited analysis, judges confidence, and
+ *      (ObservabilityAssistant) with the AI persona and a larger budget,
+ *   3. brands the cited analysis, judges confidence via the structured
+ *      ConfidenceSignal (deterministic evidence floor + one constrained
+ *      classification call — G6: no control flow from free-form prose), and
  *   4. hands the finished analysis back to the caller to post to the subject's
  *      timeline.
  *
@@ -45,13 +48,6 @@ const MAX_TOOL_CALLS: number = 12;
 const MAX_WALL_CLOCK_MS: number = 150 * 1000;
 const MAX_OUTPUT_TOKENS: number = 2000;
 
-/*
- * Sentinel's own contract: it writes exactly this phrase when it cannot
- * determine a cause. We match it deterministically to drive "quiet mode" —
- * a non-answer should never loudly page the on-call.
- */
-const INCONCLUSIVE_RE: RegExp = /inconclusive[^.\n]*insufficient signal/i;
-
 /*
  * Failures a retry cannot fix within the run's usefulness window: missing/
  * broken provider configuration and budget exhaustion (both messages minted
@@ -70,7 +66,7 @@ const STEP_EVENT_TYPE: Record<ObservabilityAssistantStepType, AIRunEventType> =
     tool_failed: AIRunEventType.ToolCallFailed,
   };
 
-const INVESTIGATION_PERSONA: string = `You are "Sentinel", OneUptime's autonomous AI Site Reliability Engineer. You have been woken automatically because a NEW signal (an incident or alert) was just declared in this project — no human has asked you a question yet. Investigate it proactively and produce a first-pass root cause analysis that the on-call engineer will read the moment they are paged.
+const INVESTIGATION_PERSONA: string = `You are OneUptime AI, OneUptime's autonomous AI Site Reliability Engineer. You have been woken automatically because a NEW signal (an incident or alert) was just declared in this project — no human has asked you a question yet. Investigate it proactively and produce a first-pass root cause analysis that the on-call engineer will read the moment they are paged.
 
 Investigate like a senior on-call engineer:
 - Start from the affected monitors/services named in the signal.
@@ -80,34 +76,37 @@ Investigate like a senior on-call engineer:
 
 Write your final answer as a concise incident-response note with exactly these sections (use these markdown headings):
 **Summary** — one or two sentences a paged engineer can read in five seconds.
-**Most likely root cause** — your hypothesis, with only the confidence the evidence actually supports, each factual claim carrying its [C#] citation. If you could not determine it, write "Inconclusive — insufficient signal" and explain what is missing.
+**Most likely root cause** — your hypothesis, with only the confidence the evidence actually supports, each factual claim carrying its [C#] citation. If you could not determine it, say so plainly and explain what is missing.
 **Evidence** — the key findings that support or rule out the hypothesis, each cited.
 **Suggested next steps** — concrete actions for the on-call engineer.
 
 Keep it tight and skimmable. You are read-only: never claim to have changed anything.`;
 
 // Which signal an investigation is about — each has its own per-project opt-in.
-export type SentinelSubjectType = "Incident" | "Alert";
+export type AISubjectType = "Incident" | "Alert";
 
 export interface InvestigationRequest {
-  // Label recorded on LlmLog, e.g. "Sentinel Incident Investigation".
+  // Label recorded on LlmLog, e.g. "AI Incident Investigation".
   feature: string;
   // A compact markdown summary of the subject that seeds the investigation.
   contextSummary: string;
   /*
-   * Called with the finished, branded, cited analysis so the caller can post it
-   * to the subject's timeline. `isConfident` is false when Sentinel reported it
-   * could not determine the cause — callers use it to post QUIETLY (no loud
-   * workspace ping) rather than paging a non-answer.
+   * Called with the finished, branded, cited analysis so the caller can post
+   * it to the subject's timeline. `confidence` is the structured,
+   * server-verified G6 signal (see ConfidenceSignal.ts) — callers must route
+   * every control-flow decision through its helpers
+   * (shouldSendWorkspaceNotification / shouldEnqueueInstrumentationTask),
+   * never through the analysis prose; the helpers encode each consumer's
+   * fail direction when the classification itself failed.
    */
   postAnalysis: (data: {
     analysisMarkdown: string;
-    isConfident: boolean;
+    confidence: ConfidenceSignal;
     result: ObservabilityAssistantResult;
   }) => Promise<void>;
 }
 
-export default class SentinelInvestigationEngine {
+export default class AIInvestigationEngine {
   /*
    * Shared gate: AI enabled, the subject's auto-investigation opt-in on, and an
    * LLM provider configured. Incidents and alerts each have their own opt-in so
@@ -117,7 +116,7 @@ export default class SentinelInvestigationEngine {
   @CaptureSpan()
   public static async isEnabledForProject(
     projectId: ObjectID,
-    subjectType: SentinelSubjectType,
+    subjectType: AISubjectType,
   ): Promise<boolean> {
     const project: Project | null = await ProjectService.findOneById({
       id: projectId,
@@ -151,7 +150,7 @@ export default class SentinelInvestigationEngine {
 
     if (!llmProvider) {
       logger.debug(
-        `Sentinel: skipping investigation for project ${projectId.toString()} — no LLM provider configured.`,
+        `AI: skipping investigation for project ${projectId.toString()} — no LLM provider configured.`,
       );
       return false;
     }
@@ -161,7 +160,7 @@ export default class SentinelInvestigationEngine {
 
   /*
    * Execute an already-CLAIMED (Running) investigation run end-to-end.
-   * Called by SentinelInvestigationQueue after a successful CAS claim —
+   * Called by AIInvestigationQueue after a successful CAS claim —
    * cap/budget gating and run creation live in the queue now. Never throws;
    * failures are handed to the queue's retry policy (failOrRequeue).
    */
@@ -188,7 +187,7 @@ export default class SentinelInvestigationEngine {
       ).toNumber();
     } catch (error) {
       logger.error(
-        `Sentinel: failed to read event count for run ${aiRunId.toString()}; starting sequence at 0: ${error}`,
+        `AI: failed to read event count for run ${aiRunId.toString()}; starting sequence at 0: ${error}`,
       );
     }
 
@@ -277,7 +276,7 @@ export default class SentinelInvestigationEngine {
 
       if (completedCount === 0) {
         logger.warn(
-          `Sentinel: run ${aiRunId.toString()} finished but was no longer Running (likely requeued as stale mid-flight); skipping postAnalysis to avoid a duplicate RCA.`,
+          `AI: run ${aiRunId.toString()} finished but was no longer Running (likely requeued as stale mid-flight); skipping postAnalysis to avoid a duplicate RCA.`,
         );
         return;
       }
@@ -293,21 +292,43 @@ export default class SentinelInvestigationEngine {
 
       if (!analysis) {
         logger.debug(
-          `Sentinel: investigation (run ${aiRunId.toString()}) produced no analysis; nothing posted.`,
+          `AI: investigation (run ${aiRunId.toString()}) produced no analysis; nothing posted.`,
         );
         return;
       }
 
-      const isConfident: boolean = !INCONCLUSIVE_RE.test(analysis);
+      /*
+       * G6: judge confidence via the structured signal — the deterministic
+       * evidence floor over this run's own server-minted citations, then
+       * (only when the floor passes) one constrained classification call.
+       * Budget accounting: that call fires AFTER the agent loop finished, so
+       * it is deliberately OUTSIDE the per-run caps above (MAX_LLM_CALLS /
+       * MAX_OUTPUT_TOKENS govern the loop, whose counts were already
+       * persisted at the Completed transition). It is still metered in
+       * LlmLog under an AUTONOMOUS_AI_FEATURES feature, so the G4 daily
+       * autonomous budget covers it; a budget rejection degrades the signal
+       * to "classification-failed" — never a run failure. Running it after
+       * the WON Completed transition also means a falsely-requeued duplicate
+       * attempt can never double-spend it.
+       */
+      const confidence: ConfidenceSignal =
+        await AIConfidenceSignal.computeConfidenceSignal({
+          projectId,
+          aiRunId,
+          analysisMarkdown: analysis,
+          evidence: AIConfidenceSignal.evidenceFromCitations(
+            result.citations || [],
+          ),
+        });
 
       await request.postAnalysis({
         analysisMarkdown: this.buildBrandedMarkdown(result, analysis),
-        isConfident,
+        confidence,
         result,
       });
 
       logger.debug(
-        `Sentinel: investigation complete (run ${aiRunId.toString()}, confident=${isConfident}, ${result.llmCallCount} LLM calls, ${result.toolCallCount} tools, ${result.totalTokens} tokens).`,
+        `AI: investigation complete (run ${aiRunId.toString()}, confident=${confidence.confident} via ${confidence.source}, ${result.llmCallCount} LLM calls, ${result.toolCallCount} tools, ${result.totalTokens} tokens).`,
       );
     } catch (error) {
       const message: string =
@@ -330,7 +351,7 @@ export default class SentinelInvestigationEngine {
        * Only configuration/budget gating — which a retry cannot change
        * within the run's usefulness window — counts as permanent.
        */
-      await SentinelInvestigationQueue.failOrRequeue({
+      await AIInvestigationQueue.failOrRequeue({
         aiRunId,
         attemptCount: data.attemptCount,
         errorMessage: message,
@@ -338,17 +359,17 @@ export default class SentinelInvestigationEngine {
       });
 
       logger.error(
-        `Sentinel: investigation attempt ${data.attemptCount} failed (run ${aiRunId.toString()}): ${message}`,
+        `AI: investigation attempt ${data.attemptCount} failed (run ${aiRunId.toString()}): ${message}`,
       );
     }
   }
 
-  // Wrap the agent's analysis with Sentinel branding + an evidence list.
+  // Wrap the agent's analysis with AI branding + an evidence list.
   private static buildBrandedMarkdown(
     result: ObservabilityAssistantResult,
     analysisMarkdown: string,
   ): string {
-    let markdown: string = `## 🧠 Sentinel — Automated Root Cause Analysis\n\n${analysisMarkdown}`;
+    let markdown: string = `## 🧠 AI — Automated Root Cause Analysis\n\n${analysisMarkdown}`;
 
     const citations: Array<AIChatCitation> = result.citations || [];
 
@@ -359,7 +380,7 @@ export default class SentinelInvestigationEngine {
       }
     }
 
-    markdown += `\n\n---\n*Investigated automatically by OneUptime Sentinel — read-only, ${result.toolCallCount} quer${
+    markdown += `\n\n---\n*Investigated automatically by OneUptime AI — read-only, ${result.toolCallCount} quer${
       result.toolCallCount === 1 ? "y" : "ies"
     } run across your own telemetry${
       result.modelName ? ` using ${result.modelName}` : ""
@@ -379,7 +400,7 @@ export default class SentinelInvestigationEngine {
         props: { isRoot: true },
       });
     } catch (error) {
-      logger.error(`Sentinel: heartbeat update failed: ${error}`);
+      logger.error(`AI: heartbeat update failed: ${error}`);
     }
   }
 
@@ -419,7 +440,7 @@ export default class SentinelInvestigationEngine {
       });
     } catch (error) {
       // Events are best-effort telemetry — never fail the run because of them.
-      logger.error(`Sentinel: failed to emit AIRunEvent: ${error}`);
+      logger.error(`AI: failed to emit AIRunEvent: ${error}`);
     }
   }
 }

package/Server/Utils/AI/{Sentinel/SentinelMemory.ts → SRE/AIMemory.ts}

@@ -7,7 +7,7 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
 import logger from "../../Logger";
 
 /*
- * Sentinel — episodic memory (Phase 4, "I've seen this before").
+ * AI SRE — episodic memory (Phase 4, "I've seen this before").
  *
  * The first, no-new-storage cut of the recurrence short-circuit: instead of a
  * vector store, retrieve past RESOLVED incidents in the same project that share
@@ -28,7 +28,7 @@ const MAX_ROOT_CAUSE_CHARS: number = 300;
 const SHARED_MONITOR_SCORE: number = 2;
 const SHARED_LABEL_SCORE: number = 1;
 
-export default class SentinelMemory {
+export default class AIMemory {
   /*
    * Returns a markdown "past resolved incidents" section to append to the
    * investigation context, or an empty string when there is nothing useful.
@@ -160,7 +160,7 @@ export default class SentinelMemory {
 
       return markdown;
     } catch (error) {
-      logger.error(`Sentinel: failed to load recurrence context: ${error}`);
+      logger.error(`AI: failed to load recurrence context: ${error}`);
       return "";
     }
   }

package/Server/Utils/AI/{Sentinel → SRE}/AlertInvestigationRunner.ts

@@ -16,19 +16,22 @@ import QueryHelper from "../../../Types/Database/QueryHelper";
 import AlertAIContextBuilder, {
   AlertContextData,
 } from "../AlertAIContextBuilder";
-import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
-import SentinelInvestigationQueue from "./InvestigationQueue";
+import { AI_ALERT_INVESTIGATION_FEATURE } from "../../../Services/AIService";
+import AIInvestigationEngine from "./AIInvestigationEngine";
+import AIInvestigationQueue from "./InvestigationQueue";
+import AIConfidenceSignal, { ConfidenceSignal } from "./ConfidenceSignal";
+import InstrumentationTaskTrigger from "./InstrumentationTaskTrigger";
 import { ObservabilityAssistantResult } from "../Chat/ObservabilityAssistant";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
 /*
- * Sentinel — alert investigation.
+ * AI SRE — alert investigation.
  *
  * Alerts are the earlier, "left of boom" signal. When a new alert is declared,
- * Sentinel investigates it (read-only) and posts a cited root-cause analysis to
+ * AI investigates it (read-only) and posts a cited root-cause analysis to
  * the alert timeline + Slack/Teams. Symmetric to incident investigation and
- * built on the same shared SentinelInvestigationEngine.
+ * built on the same shared AIInvestigationEngine.
  *
  * Gated by the same per-project opt-in as incidents, PLUS alert-specific cost
  * gates (alert volume can be far higher than incidents):
@@ -52,7 +55,7 @@ export interface AlertGateDecision {
   monitorId?: ObjectID | undefined;
 }
 
-export default class SentinelAlertInvestigationRunner {
+export default class AIAlertInvestigationRunner {
   /*
    * Entry point called (fire-and-forget) from AlertService.onCreateSuccess.
    * Cheap: gates + a Queued AIRun row; execution happens via the queue.
@@ -67,10 +70,7 @@ export default class SentinelAlertInvestigationRunner {
 
     try {
       if (
-        !(await SentinelInvestigationEngine.isEnabledForProject(
-          projectId,
-          "Alert",
-        ))
+        !(await AIInvestigationEngine.isEnabledForProject(projectId, "Alert"))
       ) {
         return;
       }
@@ -82,26 +82,26 @@ export default class SentinelAlertInvestigationRunner {
 
       if (!gate.investigate) {
         logger.debug(
-          `Sentinel: skipping investigation for alert ${alertId.toString()} — ${gate.reason}.`,
+          `AI: skipping investigation for alert ${alertId.toString()} — ${gate.reason}.`,
         );
         return;
       }
 
-      await SentinelInvestigationQueue.enqueue({
+      await AIInvestigationQueue.enqueue({
         projectId,
         subjectAlertId: alertId,
         subjectMonitorId: gate.monitorId,
       });
     } catch (error) {
       logger.error(
-        `Sentinel: unexpected error enqueueing investigation for alert ${alertId.toString()}: ${error}`,
+        `AI: unexpected error enqueueing investigation for alert ${alertId.toString()}: ${error}`,
       );
     }
   }
 
   /*
    * Execute a claimed run: assemble alert context and hand it to the engine.
-   * Called by SentinelInvestigationQueue after a successful claim. Never
+   * Called by AIInvestigationQueue after a successful claim. Never
    * throws — failures are handed to the queue's retry policy so a claimed
    * run is always finalized or requeued.
    */
@@ -127,7 +127,7 @@ export default class SentinelAlertInvestigationRunner {
        * Context assembly failed — the run is claimed, so hand it to the
        * retry policy rather than leaving it Running until the sweeper.
        */
-      await SentinelInvestigationQueue.failOrRequeue({
+      await AIInvestigationQueue.failOrRequeue({
         aiRunId,
         attemptCount,
         errorMessage: `Failed to build alert context: ${
@@ -138,16 +138,17 @@ export default class SentinelAlertInvestigationRunner {
       return;
     }
 
-    await SentinelInvestigationEngine.executeRun({
+    await AIInvestigationEngine.executeRun({
       aiRunId,
       projectId,
       attemptCount,
       request: {
-        feature: "Sentinel Alert Investigation",
+        // Persisted LlmLog label — owned by AIService (G4 budget list).
+        feature: AI_ALERT_INVESTIGATION_FEATURE,
         contextSummary,
         postAnalysis: async (postData: {
           analysisMarkdown: string;
-          isConfident: boolean;
+          confidence: ConfidenceSignal;
           result: ObservabilityAssistantResult;
         }): Promise<void> => {
           await AlertFeedService.createAlertFeedItem({
@@ -158,12 +159,41 @@ export default class SentinelAlertInvestigationRunner {
             feedInfoInMarkdown: postData.analysisMarkdown,
             /*
              * Quiet mode: an inconclusive investigation posts to the timeline
-             * but does NOT loudly ping the workspace / on-call.
+             * but does NOT loudly ping the workspace / on-call. Fail
+             * direction (G6): when the confidence classification itself
+             * failed, the ping SENDS — quiet mode fails louder, not silent.
              */
             workspaceNotification: {
-              sendWorkspaceNotification: postData.isConfident,
+              sendWorkspaceNotification:
+                AIConfidenceSignal.shouldSendWorkspaceNotification(
+                  postData.confidence,
+                ),
             },
           });
+
+          /*
+           * Inconclusive means the telemetry was insufficient — for
+           * opted-in projects (Project.enableInstrumentationFixTasks,
+           * default false), queue an ImproveInstrumentation fix task that
+           * opens a PR adding the missing observability. Runs strictly
+           * AFTER the analysis is posted, and the trigger never throws, so
+           * the investigation can neither be blocked nor failed by it.
+           * Fail direction (G6): only a POSITIVE inconclusive verdict
+           * (deterministic floor or classification) enqueues — a failed
+           * classification does NOT create a PR.
+           */
+          if (
+            AIConfidenceSignal.shouldEnqueueInstrumentationTask(
+              postData.confidence,
+            )
+          ) {
+            await InstrumentationTaskTrigger.enqueueForInconclusiveInvestigation(
+              {
+                projectId,
+                alertId,
+              },
+            );
+          }
         },
       },
     });