@oneuptime/common 14.0.8 → 14.0.10

package/Server/Utils/AI/SRE/AIInvestigationEngine.ts

@@ -14,10 +14,16 @@ import {
   LIST_CLUSTER_ACCESS_TOOL_NAME,
   RUN_KUBECTL_TOOL_NAME,
 } from "../../../../Types/Kubernetes/KubernetesClusterAiAccessToolNames";
+import {
+  LIST_INFRASTRUCTURE_ACCESS_TOOL_NAME,
+  RUN_INFRASTRUCTURE_COMMAND_TOOL_NAME,
+  isInfrastructureToolName,
+} from "../ResourceAccess/ResourceAccessToolNames";
 import Project from "../../../../Models/DatabaseModels/Project";
 import LlmProvider from "../../../../Models/DatabaseModels/LlmProvider";
 import AIRunService from "../../../Services/AIRunService";
 import AIRunEventService from "../../../Services/AIRunEventService";
+import AIService from "../../../Services/AIService";
 import ProjectService from "../../../Services/ProjectService";
 import LlmProviderService from "../../../Services/LlmProviderService";
 import AIInvestigationQueue from "./InvestigationQueue";
@@ -205,10 +211,10 @@ export interface InvestigationRequest {
 
 export default class AIInvestigationEngine {
   /*
-   * Shared gate: AI enabled, the subject's auto-investigation opt-in on, and an
-   * LLM provider configured. Incidents and alerts each have their own opt-in so
-   * they can be enabled independently. Runs before any (subject-specific)
-   * context assembly.
+   * Shared gate: AI enabled, the subject's auto-investigation opt-in on, an
+   * LLM provider configured, and AI credits to pay for it. Incidents and
+   * alerts each have their own opt-in so they can be enabled independently.
+   * Runs before any (subject-specific) context assembly.
    */
   @CaptureSpan()
   public static async isEnabledForProject(
@@ -249,16 +255,58 @@ export default class AIInvestigationEngine {
       return "automatic_investigation_disabled";
     }
 
+    return this.getProviderOrBalanceReason(projectId);
+  }
+
+  /*
+   * The half of the gate that is not a switch: a model to call, and AI
+   * credits to pay for it. Shared with the other autonomous AI work that
+   * has its own switch (the postmortem draft on resolve), so the two can
+   * never disagree about whether AI can run at all.
+   *
+   * The balance check is the same predicate as the Kubernetes cluster's
+   * ai_balance_insufficient gap (AIService.getAiBalanceBlocker): on
+   * OneUptime's own billed provider, with no credits left and auto-recharge
+   * off, every model call would be refused with "Insufficient AI balance" —
+   * so a run started now would only fail, be retried, and fail again, and
+   * nobody would see why. It fails OPEN: a balance that cannot be read
+   * never blocks, because the model call itself still enforces the balance.
+   */
+  public static async getProviderOrBalanceReason(
+    projectId: ObjectID,
+  ): Promise<"provider_missing" | "insufficient_ai_balance" | null> {
     const llmProvider: LlmProvider | null =
       await LlmProviderService.getLLMProviderForProject(projectId);
 
     if (!llmProvider) {
       logger.debug(
-        `AI: skipping investigation for project ${projectId.toString()} — no LLM provider configured.`,
+        `AI: skipping autonomous AI work for project ${projectId.toString()} — no LLM provider configured.`,
       );
       return "provider_missing";
     }
 
+    let balanceBlocker: string | null = null;
+
+    try {
+      balanceBlocker = await AIService.getAiBalanceBlocker({
+        projectId,
+        // Already resolved above — spares the predicate a second lookup.
+        llmProvider,
+      });
+    } catch (error) {
+      logger.error(
+        `AI: could not check the AI balance for project ${projectId.toString()}; not blocking on it: ${error}`,
+      );
+      balanceBlocker = null;
+    }
+
+    if (balanceBlocker) {
+      logger.debug(
+        `AI: skipping autonomous AI work for project ${projectId.toString()} — ${balanceBlocker}`,
+      );
+      return "insufficient_ai_balance";
+    }
+
     return null;
   }
 
@@ -309,6 +357,14 @@ export default class AIInvestigationEngine {
      */
     let clusterToolCallCount: number = 0;
 
+    /*
+     * The same for the tools that reach an infrastructure resource through
+     * its AI agent (run_infrastructure_command, list_infrastructure_access):
+     * commands on a host, a container engine, a hypervisor, a storage
+     * cluster or a database — never telemetry queries either.
+     */
+    let infrastructureToolCallCount: number = 0;
+
     /*
      * Live narration: persist each LLM/tool step as an AIRunEvent so the UI can
      * "watch it think" by polling the run's events. Best-effort, ordered.
@@ -323,6 +379,13 @@ export default class AIInvestigationEngine {
         clusterToolCallCount++;
       }
 
+      if (
+        step.type === "tool_started" &&
+        isInfrastructureToolName(step.toolName)
+      ) {
+        infrastructureToolCallCount++;
+      }
+
       /*
        * A completed tool call also records the citation it minted (label +
        * deep-link target) and — via toolArguments below — the arguments it
@@ -556,6 +619,7 @@ export default class AIInvestigationEngine {
         result,
         analysis,
         clusterToolCallCount,
+        infrastructureToolCallCount,
       );
 
       await request.postAnalysis({
@@ -820,11 +884,17 @@ export default class AIInvestigationEngine {
    * model's). So a run that made no cluster call posts exactly what it
    * always has, and a run that did writes its telemetry count only when it
    * is not zero.
+   *
+   * Commands on infrastructure resources (run_infrastructure_command) are
+   * counted the same way, as their own "N infrastructure commands run"
+   * part appended only when a run used those tools — so every footer of a
+   * run that did not stays byte-for-byte what it was.
    */
   public static buildBrandedMarkdown(
     result: ObservabilityAssistantResult,
     analysisMarkdown: string,
     clusterToolCallCount: number = 0,
+    infrastructureToolCallCount: number = 0,
   ): string {
     let markdown: string = `## 🧠 AI — Automated Root Cause Analysis\n\n${analysisMarkdown}`;
 
@@ -845,7 +915,7 @@ export default class AIInvestigationEngine {
       }
     }
 
-    if (clusterToolCallCount <= 0) {
+    if (clusterToolCallCount <= 0 && infrastructureToolCallCount <= 0) {
       markdown += `\n\n---\n*Investigated automatically by OneUptime AI — read-only, ${result.toolCallCount} quer${
         result.toolCallCount === 1 ? "y" : "ies"
       } run across your own telemetry${
@@ -857,7 +927,9 @@ export default class AIInvestigationEngine {
 
     const telemetryQueryCount: number = Math.max(
       0,
-      result.toolCallCount - clusterToolCallCount,
+      result.toolCallCount -
+        Math.max(0, clusterToolCallCount) -
+        Math.max(0, infrastructureToolCallCount),
     );
     // Only commands that reached kubectl are cited, so only they are "run".
     const kubectlCommandCount: number = citations.filter(
@@ -883,10 +955,38 @@ export default class AIInvestigationEngine {
       );
     }
 
+    // Only commands that reached a resource are cited, so only they are "run".
+    const infrastructureCommandCount: number = citations.filter(
+      (citation: AIChatCitation): boolean => {
+        return citation.toolName === RUN_INFRASTRUCTURE_COMMAND_TOOL_NAME;
+      },
+    ).length;
+
+    if (infrastructureCommandCount > 0) {
+      counts.push(
+        `${infrastructureCommandCount} infrastructure ${
+          infrastructureCommandCount === 1 ? "command" : "commands"
+        } run on your infrastructure`,
+      );
+    }
+
+    if (counts.length === 0 && infrastructureToolCallCount > 0) {
+      counts.push(
+        clusterToolCallCount > 0
+          ? "no telemetry queries, kubectl commands or infrastructure commands run"
+          : "no telemetry queries or infrastructure commands run",
+      );
+    }
+
     if (counts.length === 0) {
       counts.push("no telemetry queries or kubectl commands run");
     }
 
+    // Three parts read "A, B and C"; one or two read exactly as they always have.
+    if (counts.length > 2) {
+      counts.splice(0, counts.length - 1, counts.slice(0, -1).join(", "));
+    }
+
     markdown += `\n\n---\n*Investigated automatically by OneUptime AI — read-only, ${counts.join(
       " and ",
     )}${
@@ -913,6 +1013,19 @@ export default class AIInvestigationEngine {
       return `${citation.rowCount} cluster(s)`;
     }
 
+    /*
+     * A command on an infrastructure resource: 1 when it completed, 0 when
+     * it ran and the program returned an error; a listing of resources is
+     * the number it listed.
+     */
+    if (citation.toolName === RUN_INFRASTRUCTURE_COMMAND_TOOL_NAME) {
+      return citation.rowCount > 0 ? "succeeded" : "command returned an error";
+    }
+
+    if (citation.toolName === LIST_INFRASTRUCTURE_ACCESS_TOOL_NAME) {
+      return `${citation.rowCount} resource(s)`;
+    }
+
     return null;
   }
 

package/Server/Utils/AI/SRE/AlertInvestigationRunner.ts

@@ -41,6 +41,10 @@ import ClusterAccessContext from "../ClusterAccess/ClusterAccessContext";
 import KubectlInvestigationToolkit, {
   INVESTIGATION_MAX_WALL_CLOCK_MS,
 } from "../ClusterAccess/KubectlInvestigationToolkit";
+import { ResourceAiAccessStatus } from "../../../../Types/ResourceAiAgent/ResourceAiAccess";
+import ResourceAiAccessService from "../../../Services/ResourceAiAccessService";
+import ResourceAccessContext from "../ResourceAccess/ResourceAccessContext";
+import InfrastructureInvestigationToolkit from "../ResourceAccess/InfrastructureInvestigationToolkit";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
@@ -165,6 +169,7 @@ export default class AIAlertInvestigationRunner {
 
     let contextSummary: string;
     let clusterStatuses: Array<KubernetesClusterAiAccessStatus> = [];
+    let resourceStatuses: Array<ResourceAiAccessStatus> = [];
     try {
       const contextData: AlertContextData =
         await AlertAIContextBuilder.buildAlertContext({
@@ -192,6 +197,29 @@ export default class AIAlertInvestigationRunner {
 
       contextSummary +=
         ClusterAccessContext.buildContextSection(clusterStatuses);
+
+      /*
+       * Direct infrastructure access (Docker, Podman, Swarm, Proxmox,
+       * VMware, Ceph, databases, hosts) through their resource AI agents.
+       * Enrichment only, like cluster access, and on its own: losing it
+       * never fails the investigation nor takes the cluster access away.
+       */
+      try {
+        const statuses: Array<ResourceAiAccessStatus> =
+          await ResourceAiAccessService.getStatusesForSubject({
+            projectId,
+            alertId,
+          });
+        const section: string =
+          ResourceAccessContext.buildContextSection(statuses);
+
+        resourceStatuses = statuses;
+        contextSummary += section;
+      } catch (error) {
+        logger.error(
+          `AI: could not resolve infrastructure access for alert ${alertId.toString()}; investigating it with OneUptime data only: ${error}`,
+        );
+      }
     } catch (error) {
       /*
        * Context assembly failed — the run is claimed, so hand it to the
@@ -238,6 +266,48 @@ export default class AIAlertInvestigationRunner {
     const extraTools: Array<ObservabilityAssistantExtraTool> =
       kubectlToolkit.buildTools();
 
+    /*
+     * The infrastructure tools, bound to the same run and deadline. Built
+     * on their own so a failure here leaves the kubectl tools (and the
+     * run) as they are.
+     */
+    let resourceAddendum: string = "";
+
+    try {
+      const infrastructureToolkit: InfrastructureInvestigationToolkit =
+        new InfrastructureInvestigationToolkit({
+          projectId,
+          aiRunId,
+          resources: resourceStatuses,
+          runDeadlineAtMs,
+        });
+      const infrastructureTools: Array<ObservabilityAssistantExtraTool> =
+        infrastructureToolkit.buildTools();
+
+      resourceAddendum =
+        resourceStatuses.length > 0
+          ? ResourceAccessContext.buildPersonaAddendum(resourceStatuses)
+          : "";
+      extraTools.push(...infrastructureTools);
+    } catch (error) {
+      logger.error(
+        `AI: could not offer infrastructure access for alert ${alertId.toString()}; investigating it with OneUptime data only: ${error}`,
+      );
+    }
+
+    // The cluster rules first, exactly as before; the infrastructure rules after.
+    const additionalInstructions: Array<string> = [];
+
+    if (clusterStatuses.length > 0) {
+      additionalInstructions.push(
+        ClusterAccessContext.buildPersonaAddendum(clusterStatuses),
+      );
+    }
+
+    if (resourceAddendum) {
+      additionalInstructions.push(resourceAddendum);
+    }
+
     await AIInvestigationEngine.executeRun({
       aiRunId,
       projectId,
@@ -248,11 +318,8 @@ export default class AIAlertInvestigationRunner {
         alertId,
         contextSummary,
         maxWallClockMs: INVESTIGATION_MAX_WALL_CLOCK_MS,
-        ...(clusterStatuses.length > 0
-          ? {
-              additionalInstructions:
-                ClusterAccessContext.buildPersonaAddendum(clusterStatuses),
-            }
+        ...(additionalInstructions.length > 0
+          ? { additionalInstructions: additionalInstructions.join("\n\n") }
           : {}),
         ...(extraTools.length > 0 ? { extraTools } : {}),
         persistCodeFixRecommendation: true,

package/Server/Utils/AI/SRE/IncidentInvestigationRunner.ts

@@ -43,6 +43,10 @@ import ClusterAccessContext from "../ClusterAccess/ClusterAccessContext";
 import KubectlInvestigationToolkit, {
   INVESTIGATION_MAX_WALL_CLOCK_MS,
 } from "../ClusterAccess/KubectlInvestigationToolkit";
+import { ResourceAiAccessStatus } from "../../../../Types/ResourceAiAgent/ResourceAiAccess";
+import ResourceAiAccessService from "../../../Services/ResourceAiAccessService";
+import ResourceAccessContext from "../ResourceAccess/ResourceAccessContext";
+import InfrastructureInvestigationToolkit from "../ResourceAccess/InfrastructureInvestigationToolkit";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
@@ -334,6 +338,7 @@ export default class AIIncidentInvestigationRunner {
 
     let contextSummary: string;
     let clusterStatuses: Array<KubernetesClusterAiAccessStatus> = [];
+    let resourceStatuses: Array<ResourceAiAccessStatus> = [];
     try {
       const contextData: IncidentContextData =
         await IncidentAIContextBuilder.buildIncidentContext({
@@ -380,6 +385,29 @@ export default class AIIncidentInvestigationRunner {
 
       contextSummary +=
         ClusterAccessContext.buildContextSection(clusterStatuses);
+
+      /*
+       * Direct infrastructure access (Docker, Podman, Swarm, Proxmox,
+       * VMware, Ceph, databases, hosts) through their resource AI agents.
+       * Enrichment only, like cluster access, and on its own: losing it
+       * never fails the investigation nor takes the cluster access away.
+       */
+      try {
+        const statuses: Array<ResourceAiAccessStatus> =
+          await ResourceAiAccessService.getStatusesForSubject({
+            projectId,
+            incidentId,
+          });
+        const section: string =
+          ResourceAccessContext.buildContextSection(statuses);
+
+        resourceStatuses = statuses;
+        contextSummary += section;
+      } catch (error) {
+        logger.error(
+          `AI: could not resolve infrastructure access for incident ${incidentId.toString()}; investigating it with OneUptime data only: ${error}`,
+        );
+      }
     } catch (error) {
       /*
        * Context assembly failed — the run is claimed, so hand it to the
@@ -426,6 +454,48 @@ export default class AIIncidentInvestigationRunner {
     const extraTools: Array<ObservabilityAssistantExtraTool> =
       kubectlToolkit.buildTools();
 
+    /*
+     * The infrastructure tools, bound to the same run and deadline. Built
+     * on their own so a failure here leaves the kubectl tools (and the
+     * run) as they are.
+     */
+    let resourceAddendum: string = "";
+
+    try {
+      const infrastructureToolkit: InfrastructureInvestigationToolkit =
+        new InfrastructureInvestigationToolkit({
+          projectId,
+          aiRunId,
+          resources: resourceStatuses,
+          runDeadlineAtMs,
+        });
+      const infrastructureTools: Array<ObservabilityAssistantExtraTool> =
+        infrastructureToolkit.buildTools();
+
+      resourceAddendum =
+        resourceStatuses.length > 0
+          ? ResourceAccessContext.buildPersonaAddendum(resourceStatuses)
+          : "";
+      extraTools.push(...infrastructureTools);
+    } catch (error) {
+      logger.error(
+        `AI: could not offer infrastructure access for incident ${incidentId.toString()}; investigating it with OneUptime data only: ${error}`,
+      );
+    }
+
+    // The cluster rules first, exactly as before; the infrastructure rules after.
+    const additionalInstructions: Array<string> = [];
+
+    if (clusterStatuses.length > 0) {
+      additionalInstructions.push(
+        ClusterAccessContext.buildPersonaAddendum(clusterStatuses),
+      );
+    }
+
+    if (resourceAddendum) {
+      additionalInstructions.push(resourceAddendum);
+    }
+
     await AIInvestigationEngine.executeRun({
       aiRunId,
       projectId,
@@ -436,11 +506,8 @@ export default class AIIncidentInvestigationRunner {
         incidentId,
         contextSummary,
         maxWallClockMs: INVESTIGATION_MAX_WALL_CLOCK_MS,
-        ...(clusterStatuses.length > 0
-          ? {
-              additionalInstructions:
-                ClusterAccessContext.buildPersonaAddendum(clusterStatuses),
-            }
+        ...(additionalInstructions.length > 0
+          ? { additionalInstructions: additionalInstructions.join("\n\n") }
           : {}),
         ...(extraTools.length > 0 ? { extraTools } : {}),
         persistCodeFixRecommendation: true,

package/Server/Utils/AI/SRE/IncidentPostmortemRunner.ts

@@ -2,8 +2,10 @@ import ObjectID from "../../../../Types/ObjectID";
 import { Blue500 } from "../../../../Types/BrandColors";
 import Incident from "../../../../Models/DatabaseModels/Incident";
 import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
+import Project from "../../../../Models/DatabaseModels/Project";
 import IncidentService from "../../../Services/IncidentService";
 import IncidentFeedService from "../../../Services/IncidentFeedService";
+import ProjectService from "../../../Services/ProjectService";
 import AIInvestigationEngine from "./AIInvestigationEngine";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
@@ -17,12 +19,47 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  * human to review and edit — turning a ~90-minute manual writeup into a review.
  *
  * It NEVER overwrites a postmortem that already exists (human work wins), is
- * gated by the same per-project opt-in + LLM provider as investigations, and is
- * fire-and-forget: failures are logged, never surfaced to the resolve flow.
+ * gated by its own per-project opt-in (Project.enableAutomaticPostmortemDraft)
+ * plus the same AI switch, LLM provider and AI balance as investigations, and
+ * is fire-and-forget: failures are logged, never surfaced to the resolve flow.
  */
 const MAX_FEED_PREVIEW_CHARS: number = 6000;
 
 export default class AIIncidentPostmortemRunner {
+  /*
+   * The draft has its own switch, separate from automatic incident
+   * investigation. Investigations are read-only and on by default for new
+   * projects; a draft writes to the incident and posts to the project's
+   * Slack/Teams channels, so a project turns it on deliberately (opt-in:
+   * === true). The AI kill switch, the LLM provider and the AI balance gate
+   * it exactly as they gate an investigation.
+   */
+  public static async isEnabledForProject(
+    projectId: ObjectID,
+  ): Promise<boolean> {
+    const project: Project | null = await ProjectService.findOneById({
+      id: projectId,
+      select: {
+        enableAi: true,
+        enableAutomaticPostmortemDraft: true,
+      },
+      props: { isRoot: true },
+    });
+
+    if (
+      !project ||
+      project.enableAi === false ||
+      project.enableAutomaticPostmortemDraft !== true
+    ) {
+      return false;
+    }
+
+    return (
+      (await AIInvestigationEngine.getProviderOrBalanceReason(projectId)) ===
+      null
+    );
+  }
+
   @CaptureSpan()
   public static async draftPostmortemOnResolve(data: {
     incidentId: ObjectID;
@@ -31,12 +68,7 @@ export default class AIIncidentPostmortemRunner {
     const { incidentId, projectId } = data;
 
     try {
-      if (
-        !(await AIInvestigationEngine.isEnabledForProject(
-          projectId,
-          "Incident",
-        ))
-      ) {
+      if (!(await this.isEnabledForProject(projectId))) {
         return;
       }
 

package/Server/Utils/AI/SRE/InvestigationEligibility.ts

@@ -54,8 +54,8 @@ export default class InvestigationEligibility {
           ? `AI was disabled for this project when this ${kind} was created, so its automatic investigation did not start.`
           : "AI is currently disabled for this project, so new automatic investigations cannot start.",
         nextStep: recorded
-          ? "Review project AI settings for new records. Existing records are not automatically retried when AI is enabled."
-          : "Enable project AI and automatic investigations to investigate new records. Existing records are not automatically retried.",
+          ? "Review project AI settings (Project Settings → AI Features) for new records. Existing records are not automatically retried when AI is enabled."
+          : "Turn on Enable AI (Project Settings → AI Features) and automatic investigations to investigate new records. Existing records are not automatically retried.",
       },
       automatic_investigation_disabled: {
         title: recorded
@@ -77,6 +77,17 @@ export default class InvestigationEligibility {
           ? "Review the AI provider configuration for new investigations. Configuring a provider later does not automatically retry existing records."
           : "Configure an AI provider in project settings. New eligible records can then be investigated; existing records are not automatically retried.",
       },
+      insufficient_ai_balance: {
+        title: recorded
+          ? "The project was out of AI credits at creation"
+          : "The project is currently out of AI credits",
+        description: recorded
+          ? `This project had no AI credits left when this ${kind} was created, so its automatic investigation did not start.`
+          : "This project is currently out of AI credits, so new automatic investigations cannot start.",
+        nextStep: recorded
+          ? "Review AI credits under Project Settings → AI Credits, or turn on auto-recharge. Adding credits later does not retry this record."
+          : "Add AI credits under Project Settings → AI Credits, or turn on auto-recharge. Existing records are not automatically retried.",
+      },
       severity_below_threshold: {
         title: recorded
           ? "Severity was below the investigation threshold at creation"

package/Server/Utils/AI/SRE/InvestigationGrader.ts

@@ -145,7 +145,7 @@ export default class InvestigationGrader {
        * executeWithLogging throw would file a permanent error for every
        * incident such a project ever resolves. It is a silent skip instead —
        * the same posture the sibling on-resolve job takes via
-       * AIInvestigationEngine.isEnabledForProject.
+       * AIIncidentPostmortemRunner.isEnabledForProject.
        */
       if (!(await AIService.isProjectAIEnabled(projectId))) {
         logger.debug(