@oneuptime/common 11.3.26 → 11.4.0

package/Server/Services/UserOnCallLogTimelineService.ts

@@ -15,6 +15,8 @@ import UserNotificationExecutionStatus from "../../Types/UserNotification/UserNo
 import User from "../../Models/DatabaseModels/User";
 import Model from "../../Models/DatabaseModels/UserOnCallLogTimeline";
 import AlertService from "./AlertService";
+import AlertEpisodeService from "./AlertEpisodeService";
+import IncidentEpisodeService from "./IncidentEpisodeService";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
@@ -40,6 +42,8 @@ export class Service extends DatabaseService<Model> {
           onCallDutyPolicyExecutionLogId: true,
           triggeredByIncidentId: true,
           triggeredByAlertId: true,
+          triggeredByAlertEpisodeId: true,
+          triggeredByIncidentEpisodeId: true,
           onCallDutyPolicyExecutionLogTimelineId: true,
         },
         skip: 0,
@@ -52,6 +56,19 @@ export class Service extends DatabaseService<Model> {
       for (const item of items) {
         const isIncident: boolean = Boolean(item.triggeredByIncidentId);
         const isAlert: boolean = Boolean(item.triggeredByAlertId);
+        /*
+         * Alert-episode and incident-episode pages were never handled here, so
+         * acknowledging one via call/SMS/email/push marked only this user's log
+         * done but never acknowledged the EPISODE. ExecutePendingExecutions gates
+         * every co-notified user's continuation solely on isEpisodeAcknowledged,
+         * so the other responders kept escalating and the episode stayed
+         * unacknowledged in its own state (audit F15). Handle both episode kinds
+         * the same way incident/alert are handled below.
+         */
+        const isAlertEpisode: boolean = Boolean(item.triggeredByAlertEpisodeId);
+        const isIncidentEpisode: boolean = Boolean(
+          item.triggeredByIncidentEpisodeId,
+        );
 
         // this incident is acknowledged.
 
@@ -73,7 +90,16 @@ export class Service extends DatabaseService<Model> {
           throw new BadDataException("User not found.");
         }
 
-        const statusMessage: string = `${isIncident ? "Incident" : "Alert"} acknowledged by ${user.name} (${user.email})`;
+        const entityLabel: string =
+          isIncident || isIncidentEpisode
+            ? isIncidentEpisode
+              ? "Incident Episode"
+              : "Incident"
+            : isAlertEpisode
+              ? "Alert Episode"
+              : "Alert";
+
+        const statusMessage: string = `${entityLabel} acknowledged by ${user.name} (${user.email})`;
 
         await UserOnCallLogService.updateOneById({
           id: item.userNotificationLogId!,
@@ -133,6 +159,22 @@ export class Service extends DatabaseService<Model> {
             item.userId!,
           );
         }
+
+        if (isAlertEpisode) {
+          // alert episode — stops co-notified responders from escalating.
+          await AlertEpisodeService.acknowledgeEpisode(
+            item.triggeredByAlertEpisodeId!,
+            item.userId!,
+          );
+        }
+
+        if (isIncidentEpisode) {
+          // incident episode — stops co-notified responders from escalating.
+          await IncidentEpisodeService.acknowledgeEpisode(
+            item.triggeredByIncidentEpisodeId!,
+            item.userId!,
+          );
+        }
       }
     }
 

package/Server/Types/Database/QueryUtil.ts

@@ -549,6 +549,63 @@ export default class QueryUtil {
           _id: QueryHelper.any(query[key] as Array<string>),
         };
       }
+
+      /*
+       * Text-search operator nested under a single-Entity relation, e.g.
+       * { user: { name: new Search("foo") } }. serializeQuery otherwise only
+       * visits top-level keys, so the nested operator reaches TypeORM as a raw
+       * class instance and silently degrades to an exact equality
+       * (`user.name = 'foo'` instead of `user.name ILIKE '%foo%'`) — which is
+       * why the Owner / user picker "search" appeared to do nothing once a
+       * project had more than a page of members.
+       *
+       * We deliberately touch ONLY the ILIKE-family leaf operators here and
+       * leave every other nested value (arrays, ids, plain equality, other
+       * operators) exactly as it was, so existing nested relation filters (the
+       * access-control label filters built by @CanAccessIfCanReadOn, etc.)
+       * serialize unchanged.
+       */
+      if (
+        tableColumnMetadata &&
+        tableColumnMetadata.modelType &&
+        tableColumnMetadata.type === TableColumnType.Entity &&
+        query[key] &&
+        typeof query[key] === Typeof.Object &&
+        !Array.isArray(query[key]) &&
+        (Object.getPrototypeOf(query[key]) === Object.prototype ||
+          Object.getPrototypeOf(query[key]) === null)
+      ) {
+        const relationModel: BaseModel = new tableColumnMetadata.modelType();
+        const relationQuery: Record<string, any> = query[key] as Record<
+          string,
+          any
+        >;
+        for (const relationKey in relationQuery) {
+          const relationColumnMetadata: TableColumnMetadata =
+            relationModel.getTableColumnMetadata(relationKey);
+          if (!relationColumnMetadata) {
+            continue;
+          }
+          const nestedValue: any = relationQuery[relationKey];
+          if (nestedValue instanceof Search) {
+            relationQuery[relationKey] = QueryHelper.search(
+              (nestedValue as Search<string>).toString(),
+            ) as any;
+          } else if (nestedValue instanceof StartsWith) {
+            relationQuery[relationKey] = QueryHelper.startsWith(
+              (nestedValue as StartsWith<string>).toString(),
+            ) as any;
+          } else if (nestedValue instanceof EndsWith) {
+            relationQuery[relationKey] = QueryHelper.endsWith(
+              (nestedValue as EndsWith<string>).toString(),
+            ) as any;
+          } else if (nestedValue instanceof NotContains) {
+            relationQuery[relationKey] = QueryHelper.notContains(
+              (nestedValue as NotContains<string>).toString(),
+            ) as any;
+          }
+        }
+      }
     }
 
     return query;

package/Server/Utils/AI/AlertAIContextBuilder.ts

@@ -235,7 +235,12 @@ The note should:
 4. Be detailed enough to help team members understand the situation
 5. Use technical language appropriate for the engineering team
 
-Write in markdown format for better readability. Be thorough and technical.`;
+Write in markdown format for better readability. Be thorough and technical.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided alert data. Do NOT invent causes, metrics, impact, or actions the data does not support.
+- If something a section needs is not in the data, write "Not available in the alert record" instead of guessing.
+- Never fabricate numbers or state confidence levels.`;
     }
 
     // Build user message

package/Server/Utils/AI/Chat/ChatAgentRunner.ts

@@ -908,9 +908,21 @@ export default class ChatAgentRunner {
         continue;
       }
 
+      const isUserMessage: boolean = message.role === AIChatMessageRole.User;
+
       messages.push({
-        role: message.role === AIChatMessageRole.User ? "user" : "assistant",
-        content: message.contentInMarkdown,
+        role: isUserMessage ? "user" : "assistant",
+        /*
+         * A prior assistant answer carries [C#] citation markers that pointed
+         * at that turn's tool results — citations that no longer exist in this
+         * turn. Left in the replayed history the model echoes and renumbers
+         * them, and this turn's stripFabricatedCitationMarkers then deletes the
+         * unmatched ones, leaving claims that look uncited. Strip the markers
+         * from replayed answers so only freshly minted citations ever appear.
+         */
+        content: isUserMessage
+          ? message.contentInMarkdown
+          : message.contentInMarkdown.replace(/\s?\[C\d+\]/g, ""),
       });
     }
 

package/Server/Utils/AI/Chat/ObservabilityAssistant.ts

@@ -1,6 +1,7 @@
 import DatabaseCommonInteractionProps from "../../../../Types/BaseDatabase/DatabaseCommonInteractionProps";
 import OneUptimeDate from "../../../../Types/Date";
 import ObjectID from "../../../../Types/ObjectID";
+import { JSONObject } from "../../../../Types/JSON";
 import { AIChatCitation } from "../../../../Types/AI/AIChatTypes";
 import AIService, { AILogResponse } from "../../../Services/AIService";
 import logger from "../../Logger";
@@ -32,6 +33,29 @@ export interface ObservabilityAssistantPriorTurn {
   content: string;
 }
 
+/*
+ * A single "thinking step" emitted live as the agent loop runs, so callers can
+ * narrate the investigation in real time ("Reading trace ✓ 0.4s"). This is the
+ * step-level narration the OneUptime UI renders by polling AIRunEvent rows.
+ */
+export type ObservabilityAssistantStepType =
+  | "llm_started"
+  | "llm_completed"
+  | "tool_started"
+  | "tool_completed"
+  | "tool_failed";
+
+export interface ObservabilityAssistantStep {
+  type: ObservabilityAssistantStepType;
+  toolName?: string | undefined;
+  toolArguments?: JSONObject | undefined;
+  rowCount?: number | undefined;
+  durationMs?: number | undefined;
+  citationId?: string | undefined;
+  totalTokens?: number | undefined;
+  errorMessage?: string | undefined;
+}
+
 export interface ObservabilityAssistantRequest {
   projectId: ObjectID;
   userId?: ObjectID | undefined;
@@ -44,6 +68,29 @@ export interface ObservabilityAssistantRequest {
   llmProviderId?: ObjectID | undefined;
   // Label recorded on LlmLog, e.g. "Slack ChatOps".
   feature: string;
+  /*
+   * Extra instructions appended to the base observability system prompt — used
+   * to give an autonomous run (e.g. Sentinel incident investigation) a distinct
+   * persona and task framing while keeping the same hard rules (citations, no
+   * fabrication, read-only). The base prompt's grounding rules always win.
+   */
+  systemInstructions?: string | undefined;
+  /*
+   * Optional budget overrides. Autonomous investigations get more room than an
+   * interactive chat-ops answer, which must return in a single quick message.
+   * Omitted values fall back to the interactive defaults.
+   */
+  maxLlmCalls?: number | undefined;
+  maxToolCalls?: number | undefined;
+  maxWallClockMs?: number | undefined;
+  maxOutputTokens?: number | undefined;
+  /*
+   * Optional live-narration hook, fired for each LLM/tool step as the loop runs.
+   * Used by autonomous runs to persist a per-step AIRunEvent trail so the UI can
+   * "watch it think". Best-effort — implementations must not throw; a slow or
+   * failing handler should not break the run.
+   */
+  onStep?: ((step: ObservabilityAssistantStep) => Promise<void>) | undefined;
 }
 
 export interface ObservabilityAssistantResult {
@@ -70,19 +117,50 @@ export default class ObservabilityAssistant {
   ): Promise<ObservabilityAssistantResult> {
     const startedAtMs: number = Date.now();
 
+    const maxLlmCalls: number = request.maxLlmCalls ?? MAX_LLM_CALLS;
+    const maxToolCalls: number = request.maxToolCalls ?? MAX_TOOL_CALLS;
+    const maxWallClockMs: number = request.maxWallClockMs ?? MAX_WALL_CLOCK_MS;
+    const maxOutputTokens: number =
+      request.maxOutputTokens ?? MAX_OUTPUT_TOKENS;
+
+    // Best-effort live-narration emitter — never throws into the loop.
+    const emitStep: (
+      step: ObservabilityAssistantStep,
+    ) => Promise<void> = async (
+      step: ObservabilityAssistantStep,
+    ): Promise<void> => {
+      if (!request.onStep) {
+        return;
+      }
+      try {
+        await request.onStep(step);
+      } catch (err) {
+        logger.error(`ObservabilityAssistant onStep failed: ${err}`);
+      }
+    };
+
     const toolContext: ToolContext = {
       projectId: request.projectId,
       props: request.props,
     };
 
+    let systemPromptContent: string = buildObservabilityChatSystemPrompt({
+      currentTime: OneUptimeDate.getCurrentDate(),
+      /*
+       * Slack/Teams and autonomous investigations have no approval UI, so this
+       * surface stays strictly read-only.
+       */
+      permissionMode: AIChatPermissionMode.ReadOnly,
+    });
+
+    if (request.systemInstructions) {
+      systemPromptContent += `\n\n${request.systemInstructions}`;
+    }
+
     const messages: Array<LLMMessage> = [
       {
         role: "system",
-        content: buildObservabilityChatSystemPrompt({
-          currentTime: OneUptimeDate.getCurrentDate(),
-          // Slack/Teams have no approval UI, so this surface stays read-only.
-          permissionMode: AIChatPermissionMode.ReadOnly,
-        }),
+        content: systemPromptContent,
       },
     ];
 
@@ -104,9 +182,9 @@ export default class ObservabilityAssistant {
 
     while (true) {
       const budgetExhausted: boolean =
-        llmCallCount >= MAX_LLM_CALLS - 1 ||
-        toolCallCount >= MAX_TOOL_CALLS ||
-        Date.now() - startedAtMs >= MAX_WALL_CLOCK_MS;
+        llmCallCount >= maxLlmCalls - 1 ||
+        toolCallCount >= maxToolCalls ||
+        Date.now() - startedAtMs >= maxWallClockMs;
 
       if (budgetExhausted) {
         messages.push({
@@ -116,6 +194,8 @@ export default class ObservabilityAssistant {
         });
       }
 
+      await emitStep({ type: "llm_started" });
+
       const response: AILogResponse = await AIService.executeWithLogging({
         projectId: request.projectId,
         userId: request.userId,
@@ -125,7 +205,7 @@ export default class ObservabilityAssistant {
         tools: budgetExhausted
           ? undefined
           : AIToolbox.getLlmToolDefinitions(AIChatPermissionMode.ReadOnly),
-        maxTokens: MAX_OUTPUT_TOKENS,
+        maxTokens: maxOutputTokens,
         temperature: TEMPERATURE,
         // Chat-ops content is per-user — do not persist previews to LlmLog.
         storeContentPreviews: false,
@@ -134,6 +214,11 @@ export default class ObservabilityAssistant {
       llmCallCount++;
       totalTokens += response.llmLog.totalTokens || 0;
 
+      await emitStep({
+        type: "llm_completed",
+        totalTokens: response.llmLog.totalTokens || 0,
+      });
+
       if (!providerName) {
         providerName = response.llmLog.llmProviderName;
         modelName = response.llmLog.modelName;
@@ -152,8 +237,8 @@ export default class ObservabilityAssistant {
 
         for (const toolCall of response.toolCalls) {
           const overBudget: boolean =
-            toolCallCount >= MAX_TOOL_CALLS ||
-            Date.now() - startedAtMs >= MAX_WALL_CLOCK_MS;
+            toolCallCount >= maxToolCalls ||
+            Date.now() - startedAtMs >= maxWallClockMs;
 
           if (overBudget) {
             messages.push({
@@ -176,6 +261,14 @@ export default class ObservabilityAssistant {
             continue;
           }
 
+          const toolStartedAtMs: number = Date.now();
+
+          await emitStep({
+            type: "tool_started",
+            toolName: toolCall.name,
+            toolArguments: toolCall.arguments,
+          });
+
           const outcome: ToolCallOutcome = await AIToolbox.executeTool({
             name: toolCall.name,
             args: toolCall.arguments,
@@ -183,6 +276,13 @@ export default class ObservabilityAssistant {
           });
 
           if (!outcome.success || !outcome.result) {
+            await emitStep({
+              type: "tool_failed",
+              toolName: toolCall.name,
+              durationMs: Date.now() - toolStartedAtMs,
+              errorMessage: outcome.errorMessage || outcome.textForLlm,
+            });
+
             messages.push({
               role: "tool",
               toolCallId: toolCall.id,
@@ -202,6 +302,14 @@ export default class ObservabilityAssistant {
             target: outcome.result.citationTarget,
           });
 
+          await emitStep({
+            type: "tool_completed",
+            toolName: toolCall.name,
+            durationMs: Date.now() - toolStartedAtMs,
+            rowCount: outcome.result.rowCount,
+            citationId: citationId,
+          });
+
           const escapedText: string = escapeToolResultContent(
             outcome.textForLlm,
           );

package/Server/Utils/AI/IncidentAIContextBuilder.ts

@@ -365,7 +365,12 @@ The postmortem should:
 
 Use a standard incident postmortem format with sections for: Executive Summary, Timeline, Root Cause Analysis, Impact, Action Items, and Lessons Learned.
 
-Write in a professional, clear, and concise manner. Use markdown formatting for better readability.`;
+Write in a professional, clear, and concise manner. Use markdown formatting for better readability.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided incident data. Do NOT invent root causes, timelines, metrics, customer impact, or actions the data does not support.
+- If a section needs information the data does not contain, write "Not available in the incident record" instead of guessing.
+- Never fabricate numbers or state confidence levels.`;
     }
 
     // Build user message based on whether template is provided
@@ -549,7 +554,12 @@ DO NOT include:
 - Confidential information
 - Excessive jargon
 
-Write in markdown format for better readability.`;
+Write in markdown format for better readability.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided incident data; never invent impact, causes, timing, or status.
+- If something is not known from the data, say it is still being investigated rather than guessing.
+- Never fabricate numbers or specific timings the data does not support.`;
       }
     } else if (template) {
       // Internal note with template
@@ -577,7 +587,12 @@ The note should:
 4. Be detailed enough to help team members understand the situation
 5. Use technical language appropriate for the engineering team
 
-Write in markdown format for better readability. Be thorough and technical.`;
+Write in markdown format for better readability. Be thorough and technical.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided incident data. Do NOT invent root causes, metrics, impact, or actions the data does not support.
+- If something a section needs is not in the data, write "Not available in the incident record" instead of guessing.
+- Never fabricate numbers or state confidence levels.`;
     }
 
     // Build user message

package/Server/Utils/AI/IncidentEpisodeAIContextBuilder.ts

@@ -389,7 +389,12 @@ The postmortem should:
 
 Use a standard incident postmortem format with sections for: Executive Summary, Timeline, Root Cause Analysis, Impact, Action Items, and Lessons Learned.
 
-Write in a professional, clear, and concise manner. Use markdown formatting for better readability.`;
+Write in a professional, clear, and concise manner. Use markdown formatting for better readability.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided episode data. Do NOT invent root causes, timelines, metrics, impact, or actions the data does not support.
+- If a section needs information the data does not contain, write "Not available in the incident record" instead of guessing.
+- Never fabricate numbers or state confidence levels.`;
     }
 
     // Build user message based on whether template is provided

package/Server/Utils/AI/ScheduledMaintenanceAIContextBuilder.ts

@@ -288,7 +288,12 @@ DO NOT include:
 - Confidential information
 - Excessive jargon
 
-Write in markdown format for better readability.`;
+Write in markdown format for better readability.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided maintenance data; never invent scope, impact, timing, or status.
+- If something is not known from the data, say it will be confirmed rather than guessing.
+- Never fabricate numbers or specific timings the data does not support.`;
       }
     } else if (template) {
       // Internal note with template
@@ -316,7 +321,12 @@ The note should:
 4. Be detailed enough to help team members understand the current status
 5. Use technical language appropriate for the engineering team
 
-Write in markdown format for better readability. Be thorough and technical.`;
+Write in markdown format for better readability. Be thorough and technical.
+
+Grounding rules (important):
+- Use ONLY facts that appear in the provided maintenance data. Do NOT invent scope, metrics, impact, or actions the data does not support.
+- If something a section needs is not in the data, write "Not available in the maintenance record" instead of guessing.
+- Never fabricate numbers or state confidence levels.`;
     }
 
     // Build user message

package/Server/Utils/AI/Sentinel/AlertInvestigationRunner.ts

@@ -0,0 +1,141 @@
+import ObjectID from "../../../../Types/ObjectID";
+import OneUptimeDate from "../../../../Types/Date";
+import { Blue500 } from "../../../../Types/BrandColors";
+import { AlertFeedEventType } from "../../../../Models/DatabaseModels/AlertFeed";
+import AlertFeedService from "../../../Services/AlertFeedService";
+import AlertAIContextBuilder, {
+  AlertContextData,
+} from "../AlertAIContextBuilder";
+import SentinelInvestigationEngine from "./SentinelInvestigationEngine";
+import { ObservabilityAssistantResult } from "../Chat/ObservabilityAssistant";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+
+/*
+ * Sentinel — alert investigation.
+ *
+ * Alerts are the earlier, "left of boom" signal. When a new alert is declared,
+ * Sentinel investigates it (read-only) and posts a cited root-cause analysis to
+ * the alert timeline + Slack/Teams. Symmetric to incident investigation and
+ * built on the same shared SentinelInvestigationEngine.
+ *
+ * Gated by the same per-project opt-in as incidents. Alert volume can be higher
+ * than incidents, so severity / rate gating is a deliberate follow-up.
+ */
+export default class SentinelAlertInvestigationRunner {
+  /*
+   * Entry point called (fire-and-forget) from AlertService.onCreateSuccess.
+   * Never throws — all failures are logged and recorded on the AIRun.
+   */
+  @CaptureSpan()
+  public static async investigateNewAlert(data: {
+    alertId: ObjectID;
+    projectId: ObjectID;
+  }): Promise<void> {
+    const { alertId, projectId } = data;
+
+    try {
+      if (
+        !(await SentinelInvestigationEngine.isEnabledForProject(
+          projectId,
+          "Alert",
+        ))
+      ) {
+        return;
+      }
+
+      const contextData: AlertContextData =
+        await AlertAIContextBuilder.buildAlertContext({
+          alertId,
+        });
+
+      await SentinelInvestigationEngine.investigate({
+        projectId,
+        feature: "Sentinel Alert Investigation",
+        subjectAlertId: alertId,
+        contextSummary: this.buildAlertSummary(contextData),
+        postAnalysis: async (postData: {
+          analysisMarkdown: string;
+          isConfident: boolean;
+          result: ObservabilityAssistantResult;
+        }): Promise<void> => {
+          await AlertFeedService.createAlertFeedItem({
+            alertId,
+            projectId,
+            alertFeedEventType: AlertFeedEventType.RootCause,
+            displayColor: Blue500,
+            feedInfoInMarkdown: postData.analysisMarkdown,
+            /*
+             * Quiet mode: an inconclusive investigation posts to the timeline
+             * but does NOT loudly ping the workspace / on-call.
+             */
+            workspaceNotification: {
+              sendWorkspaceNotification: postData.isConfident,
+            },
+          });
+        },
+      });
+    } catch (error) {
+      logger.error(
+        `Sentinel: unexpected error investigating alert ${alertId.toString()}: ${error}`,
+      );
+    }
+  }
+
+  // Build a compact alert record to seed the investigation.
+  private static buildAlertSummary(contextData: AlertContextData): string {
+    const { alert, internalNotes } = contextData;
+
+    const lines: Array<string> = [];
+    lines.push("# Alert");
+    lines.push(`Title: ${alert.title || "N/A"}`);
+
+    if (alert.description) {
+      lines.push(`Description: ${alert.description}`);
+    }
+
+    lines.push(`Severity: ${alert.alertSeverity?.name || "N/A"}`);
+    lines.push(`Current state: ${alert.currentAlertState?.name || "N/A"}`);
+    lines.push(
+      `Declared at: ${
+        alert.createdAt
+          ? OneUptimeDate.getDateAsFormattedString(alert.createdAt)
+          : "N/A"
+      }`,
+    );
+
+    if (alert.monitor?.name) {
+      lines.push(`Affected monitor: ${alert.monitor.name}`);
+    }
+
+    if (alert.labels && alert.labels.length > 0) {
+      lines.push(
+        `Labels: ${alert.labels
+          .map((l: { name?: string }) => {
+            return l.name;
+          })
+          .filter(Boolean)
+          .join(", ")}`,
+      );
+    }
+
+    if (alert.rootCause) {
+      lines.push(`Root cause (as recorded so far): ${alert.rootCause}`);
+    }
+
+    const recentNotes: Array<string> = (internalNotes || [])
+      .slice(-5)
+      .map((note: { note?: string }) => {
+        return note.note ? `- ${note.note}` : "";
+      })
+      .filter(Boolean);
+
+    if (recentNotes.length > 0) {
+      lines.push("");
+      lines.push("Recent internal notes:");
+      lines.push(...recentNotes);
+    }
+
+    return lines.join("\n");
+  }
+}