@oneuptime/common 14.0.1 → 14.0.2

package/Server/Utils/AI/SRE/AIInvestigationEngine.ts

@@ -10,6 +10,10 @@ import AIRunStatus from "../../../../Types/AI/AIRunStatus";
 import AIRunCodeFixRecommendation from "../../../../Types/AI/AIRunCodeFixRecommendation";
 import AIRunEventType from "../../../../Types/AI/AIRunEventType";
 import AIRunEvent from "../../../../Models/DatabaseModels/AIRunEvent";
+import {
+  LIST_CLUSTER_ACCESS_TOOL_NAME,
+  RUN_KUBECTL_TOOL_NAME,
+} from "../../../../Types/Kubernetes/KubernetesClusterAiAccessToolNames";
 import Project from "../../../../Models/DatabaseModels/Project";
 import LlmProvider from "../../../../Models/DatabaseModels/LlmProvider";
 import AIRunService from "../../../Services/AIRunService";
@@ -49,7 +53,13 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
 // Budgets — larger than an interactive chat-ops answer, small enough to stay cheap.
 const MAX_LLM_CALLS: number = 8;
 const MAX_TOOL_CALLS: number = 12;
-const MAX_WALL_CLOCK_MS: number = 150 * 1000;
+/*
+ * The wall clock an investigation run gets by default. Exported because the
+ * kubectl read toolkit plans every command's wait against this same budget
+ * (the runners hand it to the engine AND to the toolkit as an absolute
+ * deadline), and one definition is how the two can never disagree.
+ */
+export const MAX_WALL_CLOCK_MS: number = 150 * 1000;
 const MAX_OUTPUT_TOKENS: number = 2000;
 
 /*
@@ -66,6 +76,17 @@ const CODE_FIX_RECOMMENDATION_PERSIST_ATTEMPTS: number = 3;
 const PERMANENT_FAILURE_RE: RegExp =
   /no llm provider configured|llm provider type is not configured|token budget exhausted/i;
 
+/*
+ * The tools that reach a Kubernetes cluster rather than the project's
+ * telemetry: their calls are kubectl commands (or a listing of clusters),
+ * never "queries run across your own telemetry". Named from the pure
+ * constants module — the kubectl toolkit itself imports this engine.
+ */
+const CLUSTER_TOOL_NAMES: ReadonlyArray<string> = [
+  RUN_KUBECTL_TOOL_NAME,
+  LIST_CLUSTER_ACCESS_TOOL_NAME,
+];
+
 // Maps a live agent step to the AIRunEvent type persisted for the glass-box trail.
 const STEP_EVENT_TYPE: Record<ObservabilityAssistantStepType, AIRunEventType> =
   {
@@ -161,6 +182,13 @@ export interface InvestigationRequest {
    */
   // Replaces INVESTIGATION_PERSONA as the appended system instructions.
   personaOverride?: string | undefined;
+  /*
+   * Appended to the persona (default or overridden). Used to tell an
+   * investigation which clusters it may inspect with kubectl, and which it
+   * cannot and why — deterministic text from the access service, never
+   * model output.
+   */
+  additionalInstructions?: string | undefined;
   // Replaces the "A new signal has just been declared..." user preamble.
   questionOverride?: string | undefined;
   /*
@@ -274,6 +302,13 @@ export default class AIInvestigationEngine {
       eventType: AIRunEventType.RunStarted,
     });
 
+    /*
+     * Cluster tool calls this attempt started. The run's toolCallCount
+     * includes them, and the posted report must not call them telemetry
+     * queries.
+     */
+    let clusterToolCallCount: number = 0;
+
     /*
      * Live narration: persist each LLM/tool step as an AIRunEvent so the UI can
      * "watch it think" by polling the run's events. Best-effort, ordered.
@@ -281,6 +316,13 @@ export default class AIInvestigationEngine {
     const onStep: (step: ObservabilityAssistantStep) => Promise<void> = async (
       step: ObservabilityAssistantStep,
     ): Promise<void> => {
+      if (
+        step.type === "tool_started" &&
+        CLUSTER_TOOL_NAMES.includes(step.toolName || "")
+      ) {
+        clusterToolCallCount++;
+      }
+
       /*
        * A completed tool call also records the citation it minted (label +
        * deep-link target) and — via toolArguments below — the arguments it
@@ -343,7 +385,11 @@ export default class AIInvestigationEngine {
           // System run — full read access to the project's telemetry.
           props: { isRoot: true },
           feature: request.feature,
-          systemInstructions: request.personaOverride ?? INVESTIGATION_PERSONA,
+          systemInstructions: `${request.personaOverride ?? INVESTIGATION_PERSONA}${
+            request.additionalInstructions
+              ? `\n\n${request.additionalInstructions}`
+              : ""
+          }`,
           question: `${
             request.questionOverride ??
             "A new signal has just been declared and you have been woken to investigate it. Investigate now and produce your root cause analysis."
@@ -509,6 +555,7 @@ export default class AIInvestigationEngine {
       const investigationAnalysisMarkdown: string = this.buildBrandedMarkdown(
         result,
         analysis,
+        clusterToolCallCount,
       );
 
       await request.postAnalysis({
@@ -759,10 +806,25 @@ export default class AIInvestigationEngine {
     return false;
   }
 
-  // Wrap the agent's analysis with AI branding + an evidence list.
-  private static buildBrandedMarkdown(
+  /*
+   * Wrap the agent's analysis with AI branding + an evidence list. This is
+   * the text posted to the subject's timeline, so each cluster call is
+   * described as what it was — a kubectl command that succeeded or where
+   * kubectl returned an error, a listing of clusters — never in rows, and
+   * the footer counts telemetry queries and kubectl commands apart.
+   *
+   * The dashboard's report parser reads this text back (InvestigationReport
+   * keeps a copy of the format): it recognises a "— N row(s)" entry, and
+   * checks the block against the footer's "N queries run" (a block with
+   * more row entries than queries, or under "0 queries run", is read as the
+   * model's). So a run that made no cluster call posts exactly what it
+   * always has, and a run that did writes its telemetry count only when it
+   * is not zero.
+   */
+  public static buildBrandedMarkdown(
     result: ObservabilityAssistantResult,
     analysisMarkdown: string,
+    clusterToolCallCount: number = 0,
   ): string {
     let markdown: string = `## 🧠 AI — Automated Root Cause Analysis\n\n${analysisMarkdown}`;
 
@@ -771,19 +833,89 @@ export default class AIInvestigationEngine {
     if (citations.length > 0) {
       markdown += `\n\n**Evidence checked**`;
       for (const citation of citations.slice(0, 15)) {
+        const clusterOutcome: string | null =
+          AIInvestigationEngine.describeClusterCitationOutcome(citation);
+
+        if (clusterOutcome !== null) {
+          markdown += `\n- **[${citation.id}]** ${citation.label} — ${clusterOutcome}`;
+          continue;
+        }
+
         markdown += `\n- **[${citation.id}]** ${citation.label} — ${citation.rowCount} row(s)`;
       }
     }
 
-    markdown += `\n\n---\n*Investigated automatically by OneUptime AI — read-only, ${result.toolCallCount} quer${
-      result.toolCallCount === 1 ? "y" : "ies"
-    } run across your own telemetry${
+    if (clusterToolCallCount <= 0) {
+      markdown += `\n\n---\n*Investigated automatically by OneUptime AI — read-only, ${result.toolCallCount} quer${
+        result.toolCallCount === 1 ? "y" : "ies"
+      } run across your own telemetry${
+        result.modelName ? ` using ${result.modelName}` : ""
+      }. This is an AI-generated first pass; verify before acting.*`;
+
+      return markdown;
+    }
+
+    const telemetryQueryCount: number = Math.max(
+      0,
+      result.toolCallCount - clusterToolCallCount,
+    );
+    // Only commands that reached kubectl are cited, so only they are "run".
+    const kubectlCommandCount: number = citations.filter(
+      (citation: AIChatCitation): boolean => {
+        return citation.toolName === RUN_KUBECTL_TOOL_NAME;
+      },
+    ).length;
+    const counts: Array<string> = [];
+
+    if (telemetryQueryCount > 0) {
+      counts.push(
+        `${telemetryQueryCount} quer${
+          telemetryQueryCount === 1 ? "y" : "ies"
+        } run across your own telemetry`,
+      );
+    }
+
+    if (kubectlCommandCount > 0) {
+      counts.push(
+        `${kubectlCommandCount} kubectl ${
+          kubectlCommandCount === 1 ? "command" : "commands"
+        } run on your Kubernetes clusters`,
+      );
+    }
+
+    if (counts.length === 0) {
+      counts.push("no telemetry queries or kubectl commands run");
+    }
+
+    markdown += `\n\n---\n*Investigated automatically by OneUptime AI — read-only, ${counts.join(
+      " and ",
+    )}${
       result.modelName ? ` using ${result.modelName}` : ""
     }. This is an AI-generated first pass; verify before acting.*`;
 
     return markdown;
   }
 
+  /*
+   * What one cited cluster call returned, in its own terms, or null for a
+   * telemetry query (whose rowCount really is rows). A kubectl command's
+   * rowCount is 1 when kubectl completed and 0 when it ran and returned an
+   * error; a cluster listing's is the number of clusters it listed.
+   */
+  private static describeClusterCitationOutcome(
+    citation: AIChatCitation,
+  ): string | null {
+    if (citation.toolName === RUN_KUBECTL_TOOL_NAME) {
+      return citation.rowCount > 0 ? "succeeded" : "kubectl returned an error";
+    }
+
+    if (citation.toolName === LIST_CLUSTER_ACCESS_TOOL_NAME) {
+      return `${citation.rowCount} cluster(s)`;
+    }
+
+    return null;
+  }
+
   // Refresh lastHeartbeatAt so a long-running investigation isn't swept as stale.
   private static async touchHeartbeat(
     aiRunId: ObjectID,

package/Server/Utils/AI/SRE/AlertInvestigationRunner.ts

@@ -31,7 +31,16 @@ import AIConfidenceSignal, { ConfidenceSignal } from "./ConfidenceSignal";
 import FixFromIncidentTaskTrigger from "./FixFromIncidentTaskTrigger";
 import InstrumentationTaskTrigger from "./InstrumentationTaskTrigger";
 import RemediationHandoff from "./RemediationHandoff";
-import { ObservabilityAssistantResult } from "../Chat/ObservabilityAssistant";
+import {
+  ObservabilityAssistantExtraTool,
+  ObservabilityAssistantResult,
+} from "../Chat/ObservabilityAssistant";
+import { KubernetesClusterAiAccessStatus } from "../../../../Types/Kubernetes/KubernetesClusterAiAccess";
+import KubernetesClusterAiAccessService from "../../../Services/KubernetesClusterAiAccessService";
+import ClusterAccessContext from "../ClusterAccess/ClusterAccessContext";
+import KubectlInvestigationToolkit, {
+  INVESTIGATION_MAX_WALL_CLOCK_MS,
+} from "../ClusterAccess/KubectlInvestigationToolkit";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
@@ -155,6 +164,7 @@ export default class AIAlertInvestigationRunner {
     const { aiRunId, projectId, alertId, attemptCount } = data;
 
     let contextSummary: string;
+    let clusterStatuses: Array<KubernetesClusterAiAccessStatus> = [];
     try {
       const contextData: AlertContextData =
         await AlertAIContextBuilder.buildAlertContext({
@@ -162,6 +172,26 @@ export default class AIAlertInvestigationRunner {
         });
 
       contextSummary = this.buildAlertSummary(contextData);
+
+      /*
+       * Direct cluster access. Never a prerequisite: an access lookup that
+       * fails leaves the investigation on OneUptime telemetry alone, which
+       * is exactly what it did before clusters could be reached at all.
+       */
+      try {
+        clusterStatuses =
+          await KubernetesClusterAiAccessService.getStatusesForSubject({
+            projectId,
+            alertId,
+          });
+      } catch (error) {
+        logger.error(
+          `AI: could not resolve cluster access for alert ${alertId.toString()}; investigating with OneUptime data only: ${error}`,
+        );
+      }
+
+      contextSummary +=
+        ClusterAccessContext.buildContextSection(clusterStatuses);
     } catch (error) {
       /*
        * Context assembly failed — the run is claimed, so hand it to the
@@ -189,6 +219,25 @@ export default class AIAlertInvestigationRunner {
       return;
     }
 
+    /*
+     * The run's wall clock, stated once: the engine enforces it between
+     * tool calls and the kubectl toolkit plans every command's wait to end
+     * before it, so a run_kubectl issued late in the run cannot outlive the
+     * budget (the loop cannot interrupt a tool once it is running).
+     */
+    const runDeadlineAtMs: number =
+      Date.now() + INVESTIGATION_MAX_WALL_CLOCK_MS;
+
+    const kubectlToolkit: KubectlInvestigationToolkit =
+      new KubectlInvestigationToolkit({
+        projectId,
+        aiRunId,
+        clusters: clusterStatuses,
+        runDeadlineAtMs,
+      });
+    const extraTools: Array<ObservabilityAssistantExtraTool> =
+      kubectlToolkit.buildTools();
+
     await AIInvestigationEngine.executeRun({
       aiRunId,
       projectId,
@@ -198,6 +247,14 @@ export default class AIAlertInvestigationRunner {
         feature: AI_ALERT_INVESTIGATION_FEATURE,
         alertId,
         contextSummary,
+        maxWallClockMs: INVESTIGATION_MAX_WALL_CLOCK_MS,
+        ...(clusterStatuses.length > 0
+          ? {
+              additionalInstructions:
+                ClusterAccessContext.buildPersonaAddendum(clusterStatuses),
+            }
+          : {}),
+        ...(extraTools.length > 0 ? { extraTools } : {}),
         persistCodeFixRecommendation: true,
         persistAnalysisTldr: true,
         postAnalysis: async (postData: {

package/Server/Utils/AI/SRE/IncidentInvestigationRunner.ts

@@ -33,7 +33,16 @@ import AIMemory from "./AIMemory";
 import FixFromIncidentTaskTrigger from "./FixFromIncidentTaskTrigger";
 import InstrumentationTaskTrigger from "./InstrumentationTaskTrigger";
 import RemediationHandoff from "./RemediationHandoff";
-import { ObservabilityAssistantResult } from "../Chat/ObservabilityAssistant";
+import {
+  ObservabilityAssistantExtraTool,
+  ObservabilityAssistantResult,
+} from "../Chat/ObservabilityAssistant";
+import { KubernetesClusterAiAccessStatus } from "../../../../Types/Kubernetes/KubernetesClusterAiAccess";
+import KubernetesClusterAiAccessService from "../../../Services/KubernetesClusterAiAccessService";
+import ClusterAccessContext from "../ClusterAccess/ClusterAccessContext";
+import KubectlInvestigationToolkit, {
+  INVESTIGATION_MAX_WALL_CLOCK_MS,
+} from "../ClusterAccess/KubectlInvestigationToolkit";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
@@ -324,6 +333,7 @@ export default class AIIncidentInvestigationRunner {
     const { aiRunId, projectId, incidentId, attemptCount } = data;
 
     let contextSummary: string;
+    let clusterStatuses: Array<KubernetesClusterAiAccessStatus> = [];
     try {
       const contextData: IncidentContextData =
         await IncidentAIContextBuilder.buildIncidentContext({
@@ -354,6 +364,22 @@ export default class AIIncidentInvestigationRunner {
 
       contextSummary =
         this.buildIncidentSummary(contextData) + priorCasesContext;
+
+      // Direct cluster access — enrichment only, never a prerequisite.
+      try {
+        clusterStatuses =
+          await KubernetesClusterAiAccessService.getStatusesForSubject({
+            projectId,
+            incidentId,
+          });
+      } catch (error) {
+        logger.error(
+          `AI: could not resolve cluster access for incident ${incidentId.toString()}; investigating with OneUptime data only: ${error}`,
+        );
+      }
+
+      contextSummary +=
+        ClusterAccessContext.buildContextSection(clusterStatuses);
     } catch (error) {
       /*
        * Context assembly failed — the run is claimed, so hand it to the
@@ -381,6 +407,25 @@ export default class AIIncidentInvestigationRunner {
       return;
     }
 
+    /*
+     * The run's wall clock, stated once: the engine enforces it between
+     * tool calls and the kubectl toolkit plans every command's wait to end
+     * before it, so a run_kubectl issued late in the run cannot outlive the
+     * budget (the loop cannot interrupt a tool once it is running).
+     */
+    const runDeadlineAtMs: number =
+      Date.now() + INVESTIGATION_MAX_WALL_CLOCK_MS;
+
+    const kubectlToolkit: KubectlInvestigationToolkit =
+      new KubectlInvestigationToolkit({
+        projectId,
+        aiRunId,
+        clusters: clusterStatuses,
+        runDeadlineAtMs,
+      });
+    const extraTools: Array<ObservabilityAssistantExtraTool> =
+      kubectlToolkit.buildTools();
+
     await AIInvestigationEngine.executeRun({
       aiRunId,
       projectId,
@@ -390,6 +435,14 @@ export default class AIIncidentInvestigationRunner {
         feature: AI_INCIDENT_INVESTIGATION_FEATURE,
         incidentId,
         contextSummary,
+        maxWallClockMs: INVESTIGATION_MAX_WALL_CLOCK_MS,
+        ...(clusterStatuses.length > 0
+          ? {
+              additionalInstructions:
+                ClusterAccessContext.buildPersonaAddendum(clusterStatuses),
+            }
+          : {}),
+        ...(extraTools.length > 0 ? { extraTools } : {}),
         persistCodeFixRecommendation: true,
         persistAnalysisTldr: true,
         postAnalysis: async (postData: {

package/Server/Utils/AI/Toolbox/AIActionTools.ts

@@ -11,6 +11,7 @@ import BadDataException from "../../../../Types/Exception/BadDataException";
 import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
 import OneUptimeDate from "../../../../Types/Date";
 import UserNotificationEventType from "../../../../Types/UserNotification/UserNotificationEventType";
+import PublicNoteSubscriberNotificationDefault from "../../../../Types/StatusPage/PublicNoteSubscriberNotificationDefault";
 import { AIChatCitationTargetType } from "../../../../Types/AI/AIChatTypes";
 import IncidentService from "../../../Services/IncidentService";
 import IncidentSeverityService from "../../../Services/IncidentSeverityService";
@@ -337,7 +338,7 @@ export const RunRunbookTool: ObservabilityTool = {
 export const PostIncidentStatusUpdateTool: ObservabilityTool = {
   name: "post_incident_status_update",
   description:
-    "Post a customer-facing status update (public note) on an incident. It appears on the status page and, by default, notifies subscribers. Provide the incident id and the update text.",
+    "Post a customer-facing status update (public note) on an incident. It appears on the status page and, by default, notifies subscribers - unless subscribers were not notified when the incident was declared, in which case it stays quiet by default. Provide the incident id and the update text.",
   inputSchema: {
     type: "object",
     properties: {
@@ -352,7 +353,7 @@ export const PostIncidentStatusUpdateTool: ObservabilityTool = {
       notifySubscribers: {
         type: "boolean",
         description:
-          "Whether to notify status page subscribers. Defaults to true.",
+          "Whether to notify status page subscribers. Defaults to true, or to false when subscribers were not notified when the incident was declared.",
       },
     },
     required: ["incidentId", "note"],
@@ -390,12 +391,14 @@ export const PostIncidentStatusUpdateTool: ObservabilityTool = {
       );
     }
 
-    const notifySubscribers: boolean =
-      ToolArgs.getBoolean(args, "notifySubscribers") ?? true;
-
     const incident: Incident | null = await IncidentService.findOneById({
       id: incidentId,
-      select: { _id: true, incidentNumber: true, title: true },
+      select: {
+        _id: true,
+        incidentNumber: true,
+        title: true,
+        shouldStatusPageSubscribersBeNotifiedOnIncidentCreated: true,
+      },
       props: ctx.props,
     });
     if (!incident) {
@@ -404,6 +407,11 @@ export const PostIncidentStatusUpdateTool: ObservabilityTool = {
       );
     }
 
+    // An incident declared quietly stays quiet unless the caller asks.
+    const notifySubscribers: boolean =
+      ToolArgs.getBoolean(args, "notifySubscribers") ??
+      PublicNoteSubscriberNotificationDefault.shouldNotifyForIncident(incident);
+
     const publicNote: IncidentPublicNote = new IncidentPublicNote();
     publicNote.incidentId = incidentId;
     publicNote.projectId = ctx.projectId;

package/Server/Utils/AI/Toolbox/ResourceTools.ts

@@ -856,13 +856,15 @@ export const QueryResourceTelemetryTool: ObservabilityTool = {
     if (signal === "logs") {
       /*
        * The dashboard histogram includes whole boundary minutes so it can
-       * use minute projections. Describe that effective interval exactly.
+       * use minute projections: every minute that starts at or after the
+       * start's minute and before the end. Describe that effective interval
+       * exactly - a minute-aligned end is itself the exclusive edge.
        */
       const effectiveStart: Date = new Date(
         Math.floor(startTime.getTime() / 60000) * 60000,
       );
       const effectiveEndExclusive: Date = new Date(
-        (Math.floor(endTime.getTime() / 60000) + 1) * 60000,
+        Math.ceil(endTime.getTime() / 60000) * 60000,
       );
       const severity: string | undefined = ToolArgs.getString(
         args,