@oneuptime/common 14.0.9 → 14.0.10

package/Server/Utils/AI/SRE/AIInvestigationEngine.ts

@@ -14,6 +14,11 @@ import {
   LIST_CLUSTER_ACCESS_TOOL_NAME,
   RUN_KUBECTL_TOOL_NAME,
 } from "../../../../Types/Kubernetes/KubernetesClusterAiAccessToolNames";
+import {
+  LIST_INFRASTRUCTURE_ACCESS_TOOL_NAME,
+  RUN_INFRASTRUCTURE_COMMAND_TOOL_NAME,
+  isInfrastructureToolName,
+} from "../ResourceAccess/ResourceAccessToolNames";
 import Project from "../../../../Models/DatabaseModels/Project";
 import LlmProvider from "../../../../Models/DatabaseModels/LlmProvider";
 import AIRunService from "../../../Services/AIRunService";
@@ -352,6 +357,14 @@ export default class AIInvestigationEngine {
      */
     let clusterToolCallCount: number = 0;
 
+    /*
+     * The same for the tools that reach an infrastructure resource through
+     * its AI agent (run_infrastructure_command, list_infrastructure_access):
+     * commands on a host, a container engine, a hypervisor, a storage
+     * cluster or a database — never telemetry queries either.
+     */
+    let infrastructureToolCallCount: number = 0;
+
     /*
      * Live narration: persist each LLM/tool step as an AIRunEvent so the UI can
      * "watch it think" by polling the run's events. Best-effort, ordered.
@@ -366,6 +379,13 @@ export default class AIInvestigationEngine {
         clusterToolCallCount++;
       }
 
+      if (
+        step.type === "tool_started" &&
+        isInfrastructureToolName(step.toolName)
+      ) {
+        infrastructureToolCallCount++;
+      }
+
       /*
        * A completed tool call also records the citation it minted (label +
        * deep-link target) and — via toolArguments below — the arguments it
@@ -599,6 +619,7 @@ export default class AIInvestigationEngine {
         result,
         analysis,
         clusterToolCallCount,
+        infrastructureToolCallCount,
       );
 
       await request.postAnalysis({
@@ -863,11 +884,17 @@ export default class AIInvestigationEngine {
    * model's). So a run that made no cluster call posts exactly what it
    * always has, and a run that did writes its telemetry count only when it
    * is not zero.
+   *
+   * Commands on infrastructure resources (run_infrastructure_command) are
+   * counted the same way, as their own "N infrastructure commands run"
+   * part appended only when a run used those tools — so every footer of a
+   * run that did not stays byte-for-byte what it was.
    */
   public static buildBrandedMarkdown(
     result: ObservabilityAssistantResult,
     analysisMarkdown: string,
     clusterToolCallCount: number = 0,
+    infrastructureToolCallCount: number = 0,
   ): string {
     let markdown: string = `## 🧠 AI — Automated Root Cause Analysis\n\n${analysisMarkdown}`;
 
@@ -888,7 +915,7 @@ export default class AIInvestigationEngine {
       }
     }
 
-    if (clusterToolCallCount <= 0) {
+    if (clusterToolCallCount <= 0 && infrastructureToolCallCount <= 0) {
       markdown += `\n\n---\n*Investigated automatically by OneUptime AI — read-only, ${result.toolCallCount} quer${
         result.toolCallCount === 1 ? "y" : "ies"
       } run across your own telemetry${
@@ -900,7 +927,9 @@ export default class AIInvestigationEngine {
 
     const telemetryQueryCount: number = Math.max(
       0,
-      result.toolCallCount - clusterToolCallCount,
+      result.toolCallCount -
+        Math.max(0, clusterToolCallCount) -
+        Math.max(0, infrastructureToolCallCount),
     );
     // Only commands that reached kubectl are cited, so only they are "run".
     const kubectlCommandCount: number = citations.filter(
@@ -926,10 +955,38 @@ export default class AIInvestigationEngine {
       );
     }
 
+    // Only commands that reached a resource are cited, so only they are "run".
+    const infrastructureCommandCount: number = citations.filter(
+      (citation: AIChatCitation): boolean => {
+        return citation.toolName === RUN_INFRASTRUCTURE_COMMAND_TOOL_NAME;
+      },
+    ).length;
+
+    if (infrastructureCommandCount > 0) {
+      counts.push(
+        `${infrastructureCommandCount} infrastructure ${
+          infrastructureCommandCount === 1 ? "command" : "commands"
+        } run on your infrastructure`,
+      );
+    }
+
+    if (counts.length === 0 && infrastructureToolCallCount > 0) {
+      counts.push(
+        clusterToolCallCount > 0
+          ? "no telemetry queries, kubectl commands or infrastructure commands run"
+          : "no telemetry queries or infrastructure commands run",
+      );
+    }
+
     if (counts.length === 0) {
       counts.push("no telemetry queries or kubectl commands run");
     }
 
+    // Three parts read "A, B and C"; one or two read exactly as they always have.
+    if (counts.length > 2) {
+      counts.splice(0, counts.length - 1, counts.slice(0, -1).join(", "));
+    }
+
     markdown += `\n\n---\n*Investigated automatically by OneUptime AI — read-only, ${counts.join(
       " and ",
     )}${
@@ -956,6 +1013,19 @@ export default class AIInvestigationEngine {
       return `${citation.rowCount} cluster(s)`;
     }
 
+    /*
+     * A command on an infrastructure resource: 1 when it completed, 0 when
+     * it ran and the program returned an error; a listing of resources is
+     * the number it listed.
+     */
+    if (citation.toolName === RUN_INFRASTRUCTURE_COMMAND_TOOL_NAME) {
+      return citation.rowCount > 0 ? "succeeded" : "command returned an error";
+    }
+
+    if (citation.toolName === LIST_INFRASTRUCTURE_ACCESS_TOOL_NAME) {
+      return `${citation.rowCount} resource(s)`;
+    }
+
     return null;
   }
 

package/Server/Utils/AI/SRE/AlertInvestigationRunner.ts

@@ -41,6 +41,10 @@ import ClusterAccessContext from "../ClusterAccess/ClusterAccessContext";
 import KubectlInvestigationToolkit, {
   INVESTIGATION_MAX_WALL_CLOCK_MS,
 } from "../ClusterAccess/KubectlInvestigationToolkit";
+import { ResourceAiAccessStatus } from "../../../../Types/ResourceAiAgent/ResourceAiAccess";
+import ResourceAiAccessService from "../../../Services/ResourceAiAccessService";
+import ResourceAccessContext from "../ResourceAccess/ResourceAccessContext";
+import InfrastructureInvestigationToolkit from "../ResourceAccess/InfrastructureInvestigationToolkit";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
@@ -165,6 +169,7 @@ export default class AIAlertInvestigationRunner {
 
     let contextSummary: string;
     let clusterStatuses: Array<KubernetesClusterAiAccessStatus> = [];
+    let resourceStatuses: Array<ResourceAiAccessStatus> = [];
     try {
       const contextData: AlertContextData =
         await AlertAIContextBuilder.buildAlertContext({
@@ -192,6 +197,29 @@ export default class AIAlertInvestigationRunner {
 
       contextSummary +=
         ClusterAccessContext.buildContextSection(clusterStatuses);
+
+      /*
+       * Direct infrastructure access (Docker, Podman, Swarm, Proxmox,
+       * VMware, Ceph, databases, hosts) through their resource AI agents.
+       * Enrichment only, like cluster access, and on its own: losing it
+       * never fails the investigation nor takes the cluster access away.
+       */
+      try {
+        const statuses: Array<ResourceAiAccessStatus> =
+          await ResourceAiAccessService.getStatusesForSubject({
+            projectId,
+            alertId,
+          });
+        const section: string =
+          ResourceAccessContext.buildContextSection(statuses);
+
+        resourceStatuses = statuses;
+        contextSummary += section;
+      } catch (error) {
+        logger.error(
+          `AI: could not resolve infrastructure access for alert ${alertId.toString()}; investigating it with OneUptime data only: ${error}`,
+        );
+      }
     } catch (error) {
       /*
        * Context assembly failed — the run is claimed, so hand it to the
@@ -238,6 +266,48 @@ export default class AIAlertInvestigationRunner {
     const extraTools: Array<ObservabilityAssistantExtraTool> =
       kubectlToolkit.buildTools();
 
+    /*
+     * The infrastructure tools, bound to the same run and deadline. Built
+     * on their own so a failure here leaves the kubectl tools (and the
+     * run) as they are.
+     */
+    let resourceAddendum: string = "";
+
+    try {
+      const infrastructureToolkit: InfrastructureInvestigationToolkit =
+        new InfrastructureInvestigationToolkit({
+          projectId,
+          aiRunId,
+          resources: resourceStatuses,
+          runDeadlineAtMs,
+        });
+      const infrastructureTools: Array<ObservabilityAssistantExtraTool> =
+        infrastructureToolkit.buildTools();
+
+      resourceAddendum =
+        resourceStatuses.length > 0
+          ? ResourceAccessContext.buildPersonaAddendum(resourceStatuses)
+          : "";
+      extraTools.push(...infrastructureTools);
+    } catch (error) {
+      logger.error(
+        `AI: could not offer infrastructure access for alert ${alertId.toString()}; investigating it with OneUptime data only: ${error}`,
+      );
+    }
+
+    // The cluster rules first, exactly as before; the infrastructure rules after.
+    const additionalInstructions: Array<string> = [];
+
+    if (clusterStatuses.length > 0) {
+      additionalInstructions.push(
+        ClusterAccessContext.buildPersonaAddendum(clusterStatuses),
+      );
+    }
+
+    if (resourceAddendum) {
+      additionalInstructions.push(resourceAddendum);
+    }
+
     await AIInvestigationEngine.executeRun({
       aiRunId,
       projectId,
@@ -248,11 +318,8 @@ export default class AIAlertInvestigationRunner {
         alertId,
         contextSummary,
         maxWallClockMs: INVESTIGATION_MAX_WALL_CLOCK_MS,
-        ...(clusterStatuses.length > 0
-          ? {
-              additionalInstructions:
-                ClusterAccessContext.buildPersonaAddendum(clusterStatuses),
-            }
+        ...(additionalInstructions.length > 0
+          ? { additionalInstructions: additionalInstructions.join("\n\n") }
           : {}),
         ...(extraTools.length > 0 ? { extraTools } : {}),
         persistCodeFixRecommendation: true,

package/Server/Utils/AI/SRE/IncidentInvestigationRunner.ts

@@ -43,6 +43,10 @@ import ClusterAccessContext from "../ClusterAccess/ClusterAccessContext";
 import KubectlInvestigationToolkit, {
   INVESTIGATION_MAX_WALL_CLOCK_MS,
 } from "../ClusterAccess/KubectlInvestigationToolkit";
+import { ResourceAiAccessStatus } from "../../../../Types/ResourceAiAgent/ResourceAiAccess";
+import ResourceAiAccessService from "../../../Services/ResourceAiAccessService";
+import ResourceAccessContext from "../ResourceAccess/ResourceAccessContext";
+import InfrastructureInvestigationToolkit from "../ResourceAccess/InfrastructureInvestigationToolkit";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
 
@@ -334,6 +338,7 @@ export default class AIIncidentInvestigationRunner {
 
     let contextSummary: string;
     let clusterStatuses: Array<KubernetesClusterAiAccessStatus> = [];
+    let resourceStatuses: Array<ResourceAiAccessStatus> = [];
     try {
       const contextData: IncidentContextData =
         await IncidentAIContextBuilder.buildIncidentContext({
@@ -380,6 +385,29 @@ export default class AIIncidentInvestigationRunner {
 
       contextSummary +=
         ClusterAccessContext.buildContextSection(clusterStatuses);
+
+      /*
+       * Direct infrastructure access (Docker, Podman, Swarm, Proxmox,
+       * VMware, Ceph, databases, hosts) through their resource AI agents.
+       * Enrichment only, like cluster access, and on its own: losing it
+       * never fails the investigation nor takes the cluster access away.
+       */
+      try {
+        const statuses: Array<ResourceAiAccessStatus> =
+          await ResourceAiAccessService.getStatusesForSubject({
+            projectId,
+            incidentId,
+          });
+        const section: string =
+          ResourceAccessContext.buildContextSection(statuses);
+
+        resourceStatuses = statuses;
+        contextSummary += section;
+      } catch (error) {
+        logger.error(
+          `AI: could not resolve infrastructure access for incident ${incidentId.toString()}; investigating it with OneUptime data only: ${error}`,
+        );
+      }
     } catch (error) {
       /*
        * Context assembly failed — the run is claimed, so hand it to the
@@ -426,6 +454,48 @@ export default class AIIncidentInvestigationRunner {
     const extraTools: Array<ObservabilityAssistantExtraTool> =
       kubectlToolkit.buildTools();
 
+    /*
+     * The infrastructure tools, bound to the same run and deadline. Built
+     * on their own so a failure here leaves the kubectl tools (and the
+     * run) as they are.
+     */
+    let resourceAddendum: string = "";
+
+    try {
+      const infrastructureToolkit: InfrastructureInvestigationToolkit =
+        new InfrastructureInvestigationToolkit({
+          projectId,
+          aiRunId,
+          resources: resourceStatuses,
+          runDeadlineAtMs,
+        });
+      const infrastructureTools: Array<ObservabilityAssistantExtraTool> =
+        infrastructureToolkit.buildTools();
+
+      resourceAddendum =
+        resourceStatuses.length > 0
+          ? ResourceAccessContext.buildPersonaAddendum(resourceStatuses)
+          : "";
+      extraTools.push(...infrastructureTools);
+    } catch (error) {
+      logger.error(
+        `AI: could not offer infrastructure access for incident ${incidentId.toString()}; investigating it with OneUptime data only: ${error}`,
+      );
+    }
+
+    // The cluster rules first, exactly as before; the infrastructure rules after.
+    const additionalInstructions: Array<string> = [];
+
+    if (clusterStatuses.length > 0) {
+      additionalInstructions.push(
+        ClusterAccessContext.buildPersonaAddendum(clusterStatuses),
+      );
+    }
+
+    if (resourceAddendum) {
+      additionalInstructions.push(resourceAddendum);
+    }
+
     await AIInvestigationEngine.executeRun({
       aiRunId,
       projectId,
@@ -436,11 +506,8 @@ export default class AIIncidentInvestigationRunner {
         incidentId,
         contextSummary,
         maxWallClockMs: INVESTIGATION_MAX_WALL_CLOCK_MS,
-        ...(clusterStatuses.length > 0
-          ? {
-              additionalInstructions:
-                ClusterAccessContext.buildPersonaAddendum(clusterStatuses),
-            }
+        ...(additionalInstructions.length > 0
+          ? { additionalInstructions: additionalInstructions.join("\n\n") }
           : {}),
         ...(extraTools.length > 0 ? { extraTools } : {}),
         persistCodeFixRecommendation: true,