@oneuptime/common 11.3.28 → 11.4.1

package/Server/Utils/AI/Chat/ObservabilityAssistant.ts

@@ -1,6 +1,7 @@
 import DatabaseCommonInteractionProps from "../../../../Types/BaseDatabase/DatabaseCommonInteractionProps";
 import OneUptimeDate from "../../../../Types/Date";
 import ObjectID from "../../../../Types/ObjectID";
+import { JSONObject } from "../../../../Types/JSON";
 import { AIChatCitation } from "../../../../Types/AI/AIChatTypes";
 import AIService, { AILogResponse } from "../../../Services/AIService";
 import logger from "../../Logger";
@@ -32,6 +33,29 @@ export interface ObservabilityAssistantPriorTurn {
   content: string;
 }
 
+/*
+ * A single "thinking step" emitted live as the agent loop runs, so callers can
+ * narrate the investigation in real time ("Reading trace ✓ 0.4s"). This is the
+ * step-level narration the OneUptime UI renders by polling AIRunEvent rows.
+ */
+export type ObservabilityAssistantStepType =
+  | "llm_started"
+  | "llm_completed"
+  | "tool_started"
+  | "tool_completed"
+  | "tool_failed";
+
+export interface ObservabilityAssistantStep {
+  type: ObservabilityAssistantStepType;
+  toolName?: string | undefined;
+  toolArguments?: JSONObject | undefined;
+  rowCount?: number | undefined;
+  durationMs?: number | undefined;
+  citationId?: string | undefined;
+  totalTokens?: number | undefined;
+  errorMessage?: string | undefined;
+}
+
 export interface ObservabilityAssistantRequest {
   projectId: ObjectID;
   userId?: ObjectID | undefined;
@@ -44,6 +68,29 @@ export interface ObservabilityAssistantRequest {
   llmProviderId?: ObjectID | undefined;
   // Label recorded on LlmLog, e.g. "Slack ChatOps".
   feature: string;
+  /*
+   * Extra instructions appended to the base observability system prompt — used
+   * to give an autonomous run (e.g. Sentinel incident investigation) a distinct
+   * persona and task framing while keeping the same hard rules (citations, no
+   * fabrication, read-only). The base prompt's grounding rules always win.
+   */
+  systemInstructions?: string | undefined;
+  /*
+   * Optional budget overrides. Autonomous investigations get more room than an
+   * interactive chat-ops answer, which must return in a single quick message.
+   * Omitted values fall back to the interactive defaults.
+   */
+  maxLlmCalls?: number | undefined;
+  maxToolCalls?: number | undefined;
+  maxWallClockMs?: number | undefined;
+  maxOutputTokens?: number | undefined;
+  /*
+   * Optional live-narration hook, fired for each LLM/tool step as the loop runs.
+   * Used by autonomous runs to persist a per-step AIRunEvent trail so the UI can
+   * "watch it think". Best-effort — implementations must not throw; a slow or
+   * failing handler should not break the run.
+   */
+  onStep?: ((step: ObservabilityAssistantStep) => Promise<void>) | undefined;
 }
 
 export interface ObservabilityAssistantResult {
@@ -70,19 +117,50 @@ export default class ObservabilityAssistant {
   ): Promise<ObservabilityAssistantResult> {
     const startedAtMs: number = Date.now();
 
+    const maxLlmCalls: number = request.maxLlmCalls ?? MAX_LLM_CALLS;
+    const maxToolCalls: number = request.maxToolCalls ?? MAX_TOOL_CALLS;
+    const maxWallClockMs: number = request.maxWallClockMs ?? MAX_WALL_CLOCK_MS;
+    const maxOutputTokens: number =
+      request.maxOutputTokens ?? MAX_OUTPUT_TOKENS;
+
+    // Best-effort live-narration emitter — never throws into the loop.
+    const emitStep: (
+      step: ObservabilityAssistantStep,
+    ) => Promise<void> = async (
+      step: ObservabilityAssistantStep,
+    ): Promise<void> => {
+      if (!request.onStep) {
+        return;
+      }
+      try {
+        await request.onStep(step);
+      } catch (err) {
+        logger.error(`ObservabilityAssistant onStep failed: ${err}`);
+      }
+    };
+
     const toolContext: ToolContext = {
       projectId: request.projectId,
       props: request.props,
     };
 
+    let systemPromptContent: string = buildObservabilityChatSystemPrompt({
+      currentTime: OneUptimeDate.getCurrentDate(),
+      /*
+       * Slack/Teams and autonomous investigations have no approval UI, so this
+       * surface stays strictly read-only.
+       */
+      permissionMode: AIChatPermissionMode.ReadOnly,
+    });
+
+    if (request.systemInstructions) {
+      systemPromptContent += `\n\n${request.systemInstructions}`;
+    }
+
     const messages: Array<LLMMessage> = [
       {
         role: "system",
-        content: buildObservabilityChatSystemPrompt({
-          currentTime: OneUptimeDate.getCurrentDate(),
-          // Slack/Teams have no approval UI, so this surface stays read-only.
-          permissionMode: AIChatPermissionMode.ReadOnly,
-        }),
+        content: systemPromptContent,
       },
     ];
 
@@ -104,9 +182,9 @@ export default class ObservabilityAssistant {
 
     while (true) {
       const budgetExhausted: boolean =
-        llmCallCount >= MAX_LLM_CALLS - 1 ||
-        toolCallCount >= MAX_TOOL_CALLS ||
-        Date.now() - startedAtMs >= MAX_WALL_CLOCK_MS;
+        llmCallCount >= maxLlmCalls - 1 ||
+        toolCallCount >= maxToolCalls ||
+        Date.now() - startedAtMs >= maxWallClockMs;
 
       if (budgetExhausted) {
         messages.push({
@@ -116,6 +194,8 @@ export default class ObservabilityAssistant {
         });
       }
 
+      await emitStep({ type: "llm_started" });
+
       const response: AILogResponse = await AIService.executeWithLogging({
         projectId: request.projectId,
         userId: request.userId,
@@ -125,7 +205,7 @@ export default class ObservabilityAssistant {
         tools: budgetExhausted
           ? undefined
           : AIToolbox.getLlmToolDefinitions(AIChatPermissionMode.ReadOnly),
-        maxTokens: MAX_OUTPUT_TOKENS,
+        maxTokens: maxOutputTokens,
         temperature: TEMPERATURE,
         // Chat-ops content is per-user — do not persist previews to LlmLog.
         storeContentPreviews: false,
@@ -134,6 +214,11 @@ export default class ObservabilityAssistant {
       llmCallCount++;
       totalTokens += response.llmLog.totalTokens || 0;
 
+      await emitStep({
+        type: "llm_completed",
+        totalTokens: response.llmLog.totalTokens || 0,
+      });
+
       if (!providerName) {
         providerName = response.llmLog.llmProviderName;
         modelName = response.llmLog.modelName;
@@ -152,8 +237,8 @@ export default class ObservabilityAssistant {
 
         for (const toolCall of response.toolCalls) {
           const overBudget: boolean =
-            toolCallCount >= MAX_TOOL_CALLS ||
-            Date.now() - startedAtMs >= MAX_WALL_CLOCK_MS;
+            toolCallCount >= maxToolCalls ||
+            Date.now() - startedAtMs >= maxWallClockMs;
 
           if (overBudget) {
             messages.push({
@@ -176,6 +261,14 @@ export default class ObservabilityAssistant {
             continue;
           }
 
+          const toolStartedAtMs: number = Date.now();
+
+          await emitStep({
+            type: "tool_started",
+            toolName: toolCall.name,
+            toolArguments: toolCall.arguments,
+          });
+
           const outcome: ToolCallOutcome = await AIToolbox.executeTool({
             name: toolCall.name,
             args: toolCall.arguments,
@@ -183,6 +276,13 @@ export default class ObservabilityAssistant {
           });
 
           if (!outcome.success || !outcome.result) {
+            await emitStep({
+              type: "tool_failed",
+              toolName: toolCall.name,
+              durationMs: Date.now() - toolStartedAtMs,
+              errorMessage: outcome.errorMessage || outcome.textForLlm,
+            });
+
             messages.push({
               role: "tool",
               toolCallId: toolCall.id,
@@ -202,6 +302,14 @@ export default class ObservabilityAssistant {
             target: outcome.result.citationTarget,
           });
 
+          await emitStep({
+            type: "tool_completed",
+            toolName: toolCall.name,
+            durationMs: Date.now() - toolStartedAtMs,
+            rowCount: outcome.result.rowCount,
+            citationId: citationId,
+          });
+
           const escapedText: string = escapeToolResultContent(
             outcome.textForLlm,
           );

package/Server/Utils/AI/Sentinel/AlertInvestigationRunner.ts

@@ -0,0 +1,141 @@
+import ObjectID from "../../../../Types/ObjectID";
+import OneUptimeDate from "../../../../Types/Date";
+import { Blue500 } from "../../../../Types/BrandColors";
+import { AlertFeedEventType } from "../../../../Models/DatabaseModels/AlertFeed";
+import AlertFeedService from "../../../Services/AlertFeedService";
+import AlertAIContextBuilder, {
+  AlertContextData,
+} from "../AlertAIContextBuilder";
+import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
+import { ObservabilityAssistantResult } from "../Chat/ObservabilityAssistant";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+
+/*
+ * Sentinel — alert investigation.
+ *
+ * Alerts are the earlier, "left of boom" signal. When a new alert is declared,
+ * Sentinel investigates it (read-only) and posts a cited root-cause analysis to
+ * the alert timeline + Slack/Teams. Symmetric to incident investigation and
+ * built on the same shared SentinelInvestigationEngine.
+ *
+ * Gated by the same per-project opt-in as incidents. Alert volume can be higher
+ * than incidents, so severity / rate gating is a deliberate follow-up.
+ */
+export default class SentinelAlertInvestigationRunner {
+  /*
+   * Entry point called (fire-and-forget) from AlertService.onCreateSuccess.
+   * Never throws — all failures are logged and recorded on the AIRun.
+   */
+  @CaptureSpan()
+  public static async investigateNewAlert(data: {
+    alertId: ObjectID;
+    projectId: ObjectID;
+  }): Promise<void> {
+    const { alertId, projectId } = data;
+
+    try {
+      if (
+        !(await SentinelInvestigationEngine.isEnabledForProject(
+          projectId,
+          "Alert",
+        ))
+      ) {
+        return;
+      }
+
+      const contextData: AlertContextData =
+        await AlertAIContextBuilder.buildAlertContext({
+          alertId,
+        });
+
+      await SentinelInvestigationEngine.investigate({
+        projectId,
+        feature: "Sentinel Alert Investigation",
+        subjectAlertId: alertId,
+        contextSummary: this.buildAlertSummary(contextData),
+        postAnalysis: async (postData: {
+          analysisMarkdown: string;
+          isConfident: boolean;
+          result: ObservabilityAssistantResult;
+        }): Promise<void> => {
+          await AlertFeedService.createAlertFeedItem({
+            alertId,
+            projectId,
+            alertFeedEventType: AlertFeedEventType.RootCause,
+            displayColor: Blue500,
+            feedInfoInMarkdown: postData.analysisMarkdown,
+            /*
+             * Quiet mode: an inconclusive investigation posts to the timeline
+             * but does NOT loudly ping the workspace / on-call.
+             */
+            workspaceNotification: {
+              sendWorkspaceNotification: postData.isConfident,
+            },
+          });
+        },
+      });
+    } catch (error) {
+      logger.error(
+        `Sentinel: unexpected error investigating alert ${alertId.toString()}: ${error}`,
+      );
+    }
+  }
+
+  // Build a compact alert record to seed the investigation.
+  private static buildAlertSummary(contextData: AlertContextData): string {
+    const { alert, internalNotes } = contextData;
+
+    const lines: Array<string> = [];
+    lines.push("# Alert");
+    lines.push(`Title: ${alert.title || "N/A"}`);
+
+    if (alert.description) {
+      lines.push(`Description: ${alert.description}`);
+    }
+
+    lines.push(`Severity: ${alert.alertSeverity?.name || "N/A"}`);
+    lines.push(`Current state: ${alert.currentAlertState?.name || "N/A"}`);
+    lines.push(
+      `Declared at: ${
+        alert.createdAt
+          ? OneUptimeDate.getDateAsFormattedString(alert.createdAt)
+          : "N/A"
+      }`,
+    );
+
+    if (alert.monitor?.name) {
+      lines.push(`Affected monitor: ${alert.monitor.name}`);
+    }
+
+    if (alert.labels && alert.labels.length > 0) {
+      lines.push(
+        `Labels: ${alert.labels
+          .map((l: { name?: string }) => {
+            return l.name;
+          })
+          .filter(Boolean)
+          .join(", ")}`,
+      );
+    }
+
+    if (alert.rootCause) {
+      lines.push(`Root cause (as recorded so far): ${alert.rootCause}`);
+    }
+
+    const recentNotes: Array<string> = (internalNotes || [])
+      .slice(-5)
+      .map((note: { note?: string }) => {
+        return note.note ? `- ${note.note}` : "";
+      })
+      .filter(Boolean);
+
+    if (recentNotes.length > 0) {
+      lines.push("");
+      lines.push("Recent internal notes:");
+      lines.push(...recentNotes);
+    }
+
+    return lines.join("\n");
+  }
+}

package/Server/Utils/AI/Sentinel/IncidentInvestigationRunner.ts

@@ -0,0 +1,178 @@
+import ObjectID from "../../../../Types/ObjectID";
+import OneUptimeDate from "../../../../Types/Date";
+import { Blue500 } from "../../../../Types/BrandColors";
+import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
+import IncidentFeedService from "../../../Services/IncidentFeedService";
+import IncidentAIContextBuilder, {
+  IncidentContextData,
+} from "../IncidentAIContextBuilder";
+import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
+import SentinelMemory from "./SentinelMemory";
+import { ObservabilityAssistantResult } from "../Chat/ObservabilityAssistant";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+
+/*
+ * Sentinel — incident investigation.
+ *
+ * When a new incident is declared, wakes Sentinel to investigate it (read-only)
+ * and post a cited root-cause analysis into the incident timeline + Slack/Teams
+ * — before the on-call engineer has finished reading the page. The heavy lifting
+ * (run lifecycle, the agent loop, confidence gating) lives in the shared
+ * SentinelInvestigationEngine; this file only assembles incident context and
+ * posts the result to the incident's feed.
+ *
+ * Built on the fire-and-forget async pattern already used throughout
+ * IncidentService.onCreateSuccess. See Internal/Roadmap/AISentinelExecution.md.
+ */
+export default class SentinelIncidentInvestigationRunner {
+  /*
+   * Entry point called (fire-and-forget) from IncidentService.onCreateSuccess.
+   * Never throws — all failures are logged and recorded on the AIRun.
+   */
+  @CaptureSpan()
+  public static async investigateNewIncident(data: {
+    incidentId: ObjectID;
+    projectId: ObjectID;
+  }): Promise<void> {
+    const { incidentId, projectId } = data;
+
+    try {
+      if (
+        !(await SentinelInvestigationEngine.isEnabledForProject(
+          projectId,
+          "Incident",
+        ))
+      ) {
+        return;
+      }
+
+      const contextData: IncidentContextData =
+        await IncidentAIContextBuilder.buildIncidentContext({
+          incidentId,
+          includeWorkspaceMessages: false,
+        });
+
+      /*
+       * Recurrence memory: pull in past resolved incidents that share this
+       * incident's monitors/labels so Sentinel can recognise "I've seen this
+       * before" and reference the prior fix.
+       */
+      const priorCasesContext: string =
+        await SentinelMemory.getPriorSimilarIncidentsContext({
+          projectId,
+          currentIncidentId: incidentId,
+          monitorNames: (contextData.incident.monitors || [])
+            .map((m: { name?: string }) => {
+              return m.name || "";
+            })
+            .filter(Boolean),
+          labelNames: (contextData.incident.labels || [])
+            .map((l: { name?: string }) => {
+              return l.name || "";
+            })
+            .filter(Boolean),
+        });
+
+      await SentinelInvestigationEngine.investigate({
+        projectId,
+        feature: "Sentinel Incident Investigation",
+        subjectIncidentId: incidentId,
+        contextSummary:
+          this.buildIncidentSummary(contextData) + priorCasesContext,
+        postAnalysis: async (postData: {
+          analysisMarkdown: string;
+          isConfident: boolean;
+          result: ObservabilityAssistantResult;
+        }): Promise<void> => {
+          await IncidentFeedService.createIncidentFeedItem({
+            incidentId,
+            projectId,
+            incidentFeedEventType: IncidentFeedEventType.RootCause,
+            displayColor: Blue500,
+            feedInfoInMarkdown: postData.analysisMarkdown,
+            /*
+             * Quiet mode: an inconclusive investigation posts to the timeline
+             * but does NOT loudly ping the workspace / on-call.
+             */
+            workspaceNotification: {
+              sendWorkspaceNotification: postData.isConfident,
+            },
+          });
+        },
+      });
+    } catch (error) {
+      logger.error(
+        `Sentinel: unexpected error investigating incident ${incidentId.toString()}: ${error}`,
+      );
+    }
+  }
+
+  // Build a compact incident record to seed the investigation.
+  private static buildIncidentSummary(
+    contextData: IncidentContextData,
+  ): string {
+    const { incident, internalNotes } = contextData;
+
+    const lines: Array<string> = [];
+    lines.push("# Incident");
+    lines.push(`Title: ${incident.title || "N/A"}`);
+
+    if (incident.description) {
+      lines.push(`Description: ${incident.description}`);
+    }
+
+    lines.push(`Severity: ${incident.incidentSeverity?.name || "N/A"}`);
+    lines.push(
+      `Current state: ${incident.currentIncidentState?.name || "N/A"}`,
+    );
+    lines.push(
+      `Declared at: ${
+        incident.createdAt
+          ? OneUptimeDate.getDateAsFormattedString(incident.createdAt)
+          : "N/A"
+      }`,
+    );
+
+    if (incident.monitors && incident.monitors.length > 0) {
+      lines.push(
+        `Affected monitors: ${incident.monitors
+          .map((m: { name?: string }) => {
+            return m.name;
+          })
+          .filter(Boolean)
+          .join(", ")}`,
+      );
+    }
+
+    if (incident.labels && incident.labels.length > 0) {
+      lines.push(
+        `Labels: ${incident.labels
+          .map((l: { name?: string }) => {
+            return l.name;
+          })
+          .filter(Boolean)
+          .join(", ")}`,
+      );
+    }
+
+    if (incident.rootCause) {
+      lines.push(`Root cause (as recorded so far): ${incident.rootCause}`);
+    }
+
+    const recentNotes: Array<string> = (internalNotes || [])
+      .slice(-5)
+      .map((note: { note?: string }) => {
+        return note.note ? `- ${note.note}` : "";
+      })
+      .filter(Boolean);
+
+    if (recentNotes.length > 0) {
+      lines.push("");
+      lines.push("Recent internal notes:");
+      lines.push(...recentNotes);
+    }
+
+    return lines.join("\n");
+  }
+}

package/Server/Utils/AI/Sentinel/IncidentPostmortemRunner.ts

@@ -0,0 +1,105 @@
+import ObjectID from "../../../../Types/ObjectID";
+import { Blue500 } from "../../../../Types/BrandColors";
+import Incident from "../../../../Models/DatabaseModels/Incident";
+import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
+import IncidentService from "../../../Services/IncidentService";
+import IncidentFeedService from "../../../Services/IncidentFeedService";
+import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+
+/*
+ * Sentinel — auto-draft postmortem on resolve (Phase 3, "close the loop").
+ *
+ * When an incident moves to a resolved state, Sentinel drafts a postmortem from
+ * the incident timeline + telemetry (reusing the existing
+ * IncidentService.generatePostmortemFromAI) and saves it to the incident for a
+ * human to review and edit — turning a ~90-minute manual writeup into a review.
+ *
+ * It NEVER overwrites a postmortem that already exists (human work wins), is
+ * gated by the same per-project opt-in + LLM provider as investigations, and is
+ * fire-and-forget: failures are logged, never surfaced to the resolve flow.
+ */
+const MAX_FEED_PREVIEW_CHARS: number = 6000;
+
+export default class SentinelIncidentPostmortemRunner {
+  @CaptureSpan()
+  public static async draftPostmortemOnResolve(data: {
+    incidentId: ObjectID;
+    projectId: ObjectID;
+  }): Promise<void> {
+    const { incidentId, projectId } = data;
+
+    try {
+      if (
+        !(await SentinelInvestigationEngine.isEnabledForProject(
+          projectId,
+          "Incident",
+        ))
+      ) {
+        return;
+      }
+
+      const incident: Incident | null = await IncidentService.findOneById({
+        id: incidentId,
+        select: {
+          _id: true,
+          incidentNumber: true,
+          postmortemNote: true,
+        },
+        props: { isRoot: true },
+      });
+
+      if (!incident) {
+        return;
+      }
+
+      // Never clobber an existing postmortem (human or previously drafted).
+      if (
+        incident.postmortemNote &&
+        incident.postmortemNote.trim().length > 0
+      ) {
+        return;
+      }
+
+      const draft: string = await IncidentService.generatePostmortemFromAI({
+        incidentId,
+      });
+
+      if (!draft || !draft.trim()) {
+        return;
+      }
+
+      await IncidentService.updateOneById({
+        id: incidentId,
+        data: {
+          postmortemNote: draft,
+        },
+        props: { isRoot: true },
+      });
+
+      await IncidentFeedService.createIncidentFeedItem({
+        incidentId,
+        projectId,
+        incidentFeedEventType: IncidentFeedEventType.PostmortemNote,
+        displayColor: Blue500,
+        feedInfoInMarkdown: `## 🧠 Sentinel — Draft Postmortem\n\nSentinel drafted a postmortem for incident #${incident.incidentNumber} from the incident timeline and telemetry. It has been saved on the incident for you to review and edit.`,
+        moreInformationInMarkdown:
+          draft.length > MAX_FEED_PREVIEW_CHARS
+            ? `${draft.substring(0, MAX_FEED_PREVIEW_CHARS)}\n\n…(truncated — the full draft is saved on the incident.)`
+            : draft,
+        workspaceNotification: {
+          sendWorkspaceNotification: true,
+        },
+      });
+
+      logger.debug(
+        `Sentinel: drafted postmortem for incident ${incidentId.toString()}.`,
+      );
+    } catch (error) {
+      logger.error(
+        `Sentinel: failed to draft postmortem for incident ${incidentId.toString()}: ${error}`,
+      );
+    }
+  }
+}

package/Server/Utils/AI/Sentinel/README.md

@@ -0,0 +1,10 @@
+# Sentinel — autonomous investigation engine
+
+Wake-on-signal AI SRE: when an incident or alert is created (and the project has opted in), Sentinel runs a read-only, budgeted, citation-minting investigation over the project's own telemetry and posts a root-cause analysis to the subject's feed, with a glass-box `AIRun`/`AIRunEvent` trail behind a live dashboard panel.
+
+- `SentinelInvestigationEngine.ts` — shared run lifecycle, budgets, persona, quiet-mode confidence gate
+- `IncidentInvestigationRunner.ts` / `AlertInvestigationRunner.ts` — per-subject context assembly + result posting
+- `IncidentPostmortemRunner.ts` — auto-draft postmortem when an incident resolves
+- `SentinelMemory.ts` — recurrence context from past resolved incidents (retrieval-only v1)
+
+**Roadmap & status:** [Internal/Roadmap/AISentinelExecution.md](../../../../../Internal/Roadmap/AISentinelExecution.md) (tracker — update it in the same PR when you change behavior here) and [Internal/Roadmap/AISentinelVision.md](../../../../../Internal/Roadmap/AISentinelVision.md) (strategy).