@oneuptime/common 13.0.6 → 13.0.7

package/Server/Utils/AI/SRE/IncidentInvestigationRunner.ts

@@ -1,3 +1,8 @@
+import {
+  InvestigationNotStartedCode,
+  InvestigationGateDetails,
+} from "../../../../Types/AI/InvestigationNotStartedReason";
+import InvestigationEligibility from "./InvestigationEligibility";
 import ObjectID from "../../../../Types/ObjectID";
 import OneUptimeDate from "../../../../Types/Date";
 import AIRunType from "../../../../Types/AI/AIRunType";
@@ -17,7 +22,10 @@ import IncidentService from "../../../Services/IncidentService";
 import IncidentAIContextBuilder, {
   IncidentContextData,
 } from "../IncidentAIContextBuilder";
-import { AI_INCIDENT_INVESTIGATION_FEATURE } from "../../../Services/AIService";
+import {
+  AI_INCIDENT_INVESTIGATION_FEATURE,
+  AutonomousBudgetStatus,
+} from "../../../Services/AIService";
 import AIInvestigationEngine from "./AIInvestigationEngine";
 import AIInvestigationQueue from "./InvestigationQueue";
 import AIConfidenceSignal, { ConfidenceSignal } from "./ConfidenceSignal";
@@ -50,6 +58,8 @@ const MAX_INCIDENT_DEDUPE_WINDOW_MINUTES: number = 24 * 60;
 
 export interface IncidentGateDecision {
   investigate: boolean;
+  notStartedCode?: InvestigationNotStartedCode | undefined;
+  notStartedDetails?: InvestigationGateDetails | undefined;
   reason: string;
   monitorId?: ObjectID | undefined;
 }
@@ -74,12 +84,10 @@ export default class AIIncidentInvestigationRunner {
     const { incidentId, projectId } = data;
 
     try {
-      if (
-        !(await AIInvestigationEngine.isEnabledForProject(
-          projectId,
-          "Incident",
-        ))
-      ) {
+      const disabled: InvestigationNotStartedCode | null =
+        await AIInvestigationEngine.getDisabledReason(projectId, "Incident");
+      if (disabled) {
+        await InvestigationEligibility.recordSkipped(data, disabled);
         return false;
       }
 
@@ -89,6 +97,12 @@ export default class AIIncidentInvestigationRunner {
       });
 
       if (!gate.investigate) {
+        await InvestigationEligibility.recordSkipped(
+          data,
+          gate.notStartedCode || "eligibility_check_failed",
+          undefined,
+          gate.notStartedDetails,
+        );
         logger.debug(
           `AI: skipping investigation for incident ${incidentId.toString()} — ${gate.reason}.`,
         );
@@ -100,11 +114,21 @@ export default class AIIncidentInvestigationRunner {
           projectId,
           subjectIncidentId: incidentId,
           subjectMonitorId: gate.monitorId,
+          onNotEnqueued: async (
+            code: InvestigationNotStartedCode,
+            budget?: AutonomousBudgetStatus,
+          ): Promise<void> => {
+            await InvestigationEligibility.recordSkipped(data, code, budget);
+          },
         },
       );
 
       return enqueuedRunId !== null;
     } catch (error) {
+      await InvestigationEligibility.recordSkipped(
+        data,
+        "eligibility_check_failed",
+      );
       logger.error(
         `AI: unexpected error enqueueing investigation for incident ${incidentId.toString()}: ${error}`,
       );
@@ -140,13 +164,18 @@ export default class AIIncidentInvestigationRunner {
         },
         incidentSeverity: {
           order: true,
+          name: true,
         },
       },
       props: { isRoot: true },
     });
 
     if (!incident) {
-      return { investigate: false, reason: "incident not found" };
+      return {
+        investigate: false,
+        reason: "incident not found",
+        notStartedCode: "eligibility_check_failed",
+      };
     }
 
     const monitorIds: Array<ObjectID> = (incident.monitors || [])
@@ -175,18 +204,24 @@ export default class AIIncidentInvestigationRunner {
      * whose severity order cannot be determined passes, as does every
      * incident when no floor is configured.
      */
-    const floorOrder: number | null = await this.getSeverityFloorOrder(
+    const floorSeverity: IncidentSeverity | null = await this.getSeverityFloor(
       project?.incidentInvestigationMinimumSeverityId,
     );
+    const floorOrder: number | undefined = floorSeverity?.order;
     const incidentOrder: number | undefined = incident.incidentSeverity?.order;
 
     if (
-      floorOrder !== null &&
+      floorOrder !== undefined &&
       incidentOrder !== undefined &&
       incidentOrder > floorOrder
     ) {
       return {
         investigate: false,
+        notStartedCode: "severity_below_threshold",
+        notStartedDetails: {
+          severityName: incident.incidentSeverity?.name,
+          minimumSeverityName: floorSeverity?.name,
+        },
         reason: `severity order ${incidentOrder} is below the investigation floor (order ${floorOrder})`,
         monitorId: primaryMonitorId,
       };
@@ -227,6 +262,8 @@ export default class AIIncidentInvestigationRunner {
       if (recentRunCount > 0) {
         return {
           investigate: false,
+          notStartedCode: "monitor_cooldown",
+          notStartedDetails: { cooldownWindowMinutes: dedupeWindowMinutes },
           reason: `a monitor affected by this incident was already investigated within the last ${dedupeWindowMinutes} minutes`,
           monitorId: primaryMonitorId,
         };
@@ -246,9 +283,9 @@ export default class AIIncidentInvestigationRunner {
    * Returns null when no floor applies — which, unlike the alert lane, is the
    * default: no configured minimum means investigate everything.
    */
-  private static async getSeverityFloorOrder(
+  private static async getSeverityFloor(
     minimumSeverityId: ObjectID | undefined,
-  ): Promise<number | null> {
+  ): Promise<IncidentSeverity | null> {
     if (!minimumSeverityId) {
       return null;
     }
@@ -256,7 +293,7 @@ export default class AIIncidentInvestigationRunner {
     const floorSeverity: IncidentSeverity | null =
       await IncidentSeverityService.findOneById({
         id: minimumSeverityId,
-        select: { order: true },
+        select: { order: true, name: true },
         props: { isRoot: true },
       });
 
@@ -268,7 +305,7 @@ export default class AIIncidentInvestigationRunner {
       return null;
     }
 
-    return floorSeverity.order;
+    return floorSeverity;
   }
 
   /*

package/Server/Utils/AI/SRE/InvestigationEligibility.ts

@@ -0,0 +1,292 @@
+import Alert from "../../../../Models/DatabaseModels/Alert";
+import Incident from "../../../../Models/DatabaseModels/Incident";
+import InvestigationNotStartedReason, {
+  InvestigationNotStartedCode,
+  InvestigationGateDetails,
+} from "../../../../Types/AI/InvestigationNotStartedReason";
+import ObjectID from "../../../../Types/ObjectID";
+import AIService, { AutonomousBudgetStatus } from "../../../Services/AIService";
+import AlertService from "../../../Services/AlertService";
+import IncidentService from "../../../Services/IncidentService";
+import logger from "../../Logger";
+import AIInvestigationEngine from "./AIInvestigationEngine";
+import AIAlertInvestigationRunner from "./AlertInvestigationRunner";
+import AIIncidentInvestigationRunner from "./IncidentInvestigationRunner";
+
+export interface InvestigationSubject {
+  projectId: ObjectID;
+  incidentId?: ObjectID | undefined;
+  alertId?: ObjectID | undefined;
+}
+
+export default class InvestigationEligibility {
+  public static reason(
+    code: InvestigationNotStartedCode,
+    subject: InvestigationSubject,
+    budget?: AutonomousBudgetStatus,
+    details?: InvestigationGateDetails,
+  ): InvestigationNotStartedReason {
+    return this.buildReason("recorded", code, subject, budget, details);
+  }
+
+  private static buildReason(
+    source: "recorded" | "current_configuration",
+    code: InvestigationNotStartedCode,
+    subject: InvestigationSubject,
+    budget?: AutonomousBudgetStatus,
+    details?: InvestigationGateDetails,
+  ): InvestigationNotStartedReason {
+    const kind: string = subject.alertId ? "alert" : "incident";
+    const recorded: boolean = source === "recorded";
+    const cooldownWindow: string =
+      details?.cooldownWindowMinutes !== undefined
+        ? `${details.cooldownWindowMinutes}-minute`
+        : "configured";
+    const reasons: Record<
+      InvestigationNotStartedCode,
+      { title: string; description: string; nextStep: string }
+    > = {
+      ai_disabled: {
+        title: recorded
+          ? "Project AI was turned off at creation"
+          : "Project AI is currently turned off",
+        description: recorded
+          ? `AI was disabled for this project when this ${kind} was created, so its automatic investigation did not start.`
+          : "AI is currently disabled for this project, so new automatic investigations cannot start.",
+        nextStep: recorded
+          ? "Review project AI settings for new records. Existing records are not automatically retried when AI is enabled."
+          : "Enable project AI and automatic investigations to investigate new records. Existing records are not automatically retried.",
+      },
+      automatic_investigation_disabled: {
+        title: recorded
+          ? `Automatic ${kind} investigation was off at creation`
+          : `Automatic ${kind} investigation is currently turned off`,
+        description: `${recorded ? `Automatic investigation was not enabled for ${kind}s when this ${kind} was created.` : `Automatic investigation is not currently enabled for ${kind}s.`} Alerts and incidents have separate investigation settings.`,
+        nextStep: recorded
+          ? `Review automatic ${kind} investigation settings for new ${kind}s. Enabling investigation later does not automatically retry this ${kind}.`
+          : `Enable automatic ${kind} investigation for new ${kind}s. Existing records are not automatically retried.`,
+      },
+      provider_missing: {
+        title: recorded
+          ? "No AI provider was available at creation"
+          : "No AI provider is currently available",
+        description: recorded
+          ? `Automatic investigation was enabled when this ${kind} was created, but no configured AI provider was available to run it.`
+          : "Automatic investigation is currently enabled, but this project has no configured AI provider available to run new investigations.",
+        nextStep: recorded
+          ? "Review the AI provider configuration for new investigations. Configuring a provider later does not automatically retry existing records."
+          : "Configure an AI provider in project settings. New eligible records can then be investigated; existing records are not automatically retried.",
+      },
+      severity_below_threshold: {
+        title: recorded
+          ? "Severity was below the investigation threshold at creation"
+          : "Severity is below the current investigation threshold",
+        description: `${recorded ? `When this ${kind} was created, its` : `This ${kind}'s`}${details?.severityName ? ` ${details.severityName}` : ""} severity ${recorded ? "did not meet" : "does not meet"} ${details?.minimumSeverityName ? `the minimum ${details.minimumSeverityName} investigation severity` : "the configured investigation threshold"}.${kind === "alert" ? " By default, only the two highest alert severity tiers are investigated." : ""}`,
+        nextStep:
+          "Review the minimum investigation severity in AI settings. Changes apply to new records and do not retry this record.",
+      },
+      monitor_cooldown: {
+        title: recorded
+          ? "Skipped during the monitor cooldown"
+          : "A monitor is currently within its investigation cooldown",
+        description: recorded
+          ? `Another ${kind} investigation was already queued or started for an affected monitor within the ${cooldownWindow} investigation cooldown window. This repeat ${kind} did not start another investigation.`
+          : `An investigation for an affected monitor was queued or started within the current ${cooldownWindow} cooldown window, which prevents another automatic ${kind} investigation right now.`,
+        nextStep:
+          "Review recent investigations for the affected monitor, or adjust the investigation cooldown for new records. An earlier run may still be queued, running, or may have failed.",
+      },
+      daily_budget_exhausted: {
+        title:
+          budget?.limitInTokens === 0
+            ? recorded
+              ? "Automatic investigation was paused by its token limit"
+              : "Automatic investigation is currently paused by its token limit"
+            : recorded
+              ? "Daily AI token budget had been reached"
+              : "Daily AI token budget is currently exhausted",
+        description: budget
+          ? `AI work for ${kind}s ${recorded ? "had used" : "has used"} ${budget.usedTokensToday.toLocaleString("en-US")} of its ${budget.limitInTokens?.toLocaleString("en-US")} token daily allowance ${recorded ? "when checked at creation. No investigation was queued." : "today. The current allowance does not permit a new investigation."}`
+          : `The daily autonomous token allowance for ${kind}s ${recorded ? "had been reached when this record was created, so no investigation was queued." : "is currently exhausted, so new investigations cannot start."}`,
+        nextStep:
+          "Review the daily token limit in AI settings. Usage resets at midnight UTC; skipped records are not automatically retried after the reset.",
+      },
+      budget_check_failed: {
+        title: recorded
+          ? "The AI budget could not be checked"
+          : "The current AI budget cannot be checked",
+        description: recorded
+          ? "The system could not verify the available AI budget. It did not queue an investigation to avoid exceeding the configured spending limit."
+          : "The available AI budget cannot currently be verified, so eligibility for a new investigation cannot be confirmed.",
+        nextStep:
+          "Ask a project administrator to check service health and AI settings. This record was not queued for automatic retry.",
+      },
+      enqueue_failed: {
+        title: recorded
+          ? "The investigation could not be queued"
+          : "Investigation queue availability could not be verified",
+        description: recorded
+          ? "The system could not save an investigation to the queue, so no investigation started for this record."
+          : "The service could not confirm whether a new investigation can currently be saved to the queue.",
+        nextStep:
+          "Ask a project administrator to check service health and server logs. This record was not queued for automatic retry.",
+      },
+      eligibility_check_failed: {
+        title: recorded
+          ? "Investigation eligibility could not be verified"
+          : "Current investigation eligibility cannot be verified",
+        description: recorded
+          ? "The system could not complete the checks required to start an automatic investigation."
+          : "The current checks required to start an automatic investigation could not be completed.",
+        nextStep:
+          "Ask a project administrator to check service health and AI settings. No automatic retry is scheduled for this record.",
+      },
+      no_run_recorded: {
+        title: "No investigation was recorded",
+        description: `This ${kind} passes the current automatic investigation checks, but no investigation or original skip decision was found. Settings may have changed since it was created, or automatic investigation may not have started when this record was created.`,
+        nextStep:
+          "Automatic investigations are triggered when a record is created. Enabling investigation later does not backfill existing records; check the AI logs and service health if new eligible records are also missing investigations.",
+      },
+    };
+    return {
+      code,
+      ...reasons[code],
+      description: recorded
+        ? reasons[code].description
+        : `${reasons[code].description} These are the current conditions; the original decision was not recorded, so they do not confirm why this record was skipped at creation.`,
+      source,
+      evaluatedAt: new Date().toISOString(),
+    };
+  }
+
+  /** Best effort, without overriding the first decision or failing creation. */
+  public static async recordSkipped(
+    subject: InvestigationSubject,
+    code: InvestigationNotStartedCode,
+    budget?: AutonomousBudgetStatus,
+    details?: InvestigationGateDetails,
+  ): Promise<void> {
+    try {
+      const decision: InvestigationNotStartedReason = this.reason(
+        code,
+        subject,
+        budget,
+        details,
+      );
+      if (subject.alertId) {
+        await AlertService.updateColumnsByIdWithoutHooks({
+          id: subject.alertId,
+          data: { aiInvestigationDecision: decision },
+          expectedData: {
+            projectId: subject.projectId,
+            aiInvestigationDecision: null,
+          },
+          skipUpdateDateColumn: true,
+        });
+      } else if (subject.incidentId) {
+        await IncidentService.updateColumnsByIdWithoutHooks({
+          id: subject.incidentId,
+          data: { aiInvestigationDecision: decision },
+          expectedData: {
+            projectId: subject.projectId,
+            aiInvestigationDecision: null,
+          },
+          skipUpdateDateColumn: true,
+        });
+      }
+    } catch (error) {
+      logger.error(`AI: could not record investigation decision: ${error}`);
+    }
+  }
+
+  /** Read-only; the caller must first authorize access to the subject. */
+  public static async getNotStartedReason(
+    subject: InvestigationSubject,
+  ): Promise<InvestigationNotStartedReason> {
+    try {
+      const row: Alert | Incident | null = subject.alertId
+        ? await AlertService.findOneBy({
+            query: {
+              _id: subject.alertId.toString(),
+              projectId: subject.projectId,
+            },
+            select: { aiInvestigationDecision: true },
+            props: { isRoot: true },
+          })
+        : await IncidentService.findOneBy({
+            query: {
+              _id: subject.incidentId!.toString(),
+              projectId: subject.projectId,
+            },
+            select: { aiInvestigationDecision: true },
+            props: { isRoot: true },
+          });
+      if (row?.aiInvestigationDecision) {
+        return row.aiInvestigationDecision;
+      }
+
+      const disabled: InvestigationNotStartedCode | null =
+        await AIInvestigationEngine.getDisabledReason(
+          subject.projectId,
+          subject.alertId ? "Alert" : "Incident",
+        );
+      if (disabled) {
+        return this.currentReason(disabled, subject);
+      }
+
+      const gate: {
+        investigate: boolean;
+        notStartedCode?: InvestigationNotStartedCode | undefined;
+        notStartedDetails?: InvestigationGateDetails | undefined;
+      } = subject.alertId
+        ? await AIAlertInvestigationRunner.shouldInvestigateAlert({
+            projectId: subject.projectId,
+            alertId: subject.alertId,
+          })
+        : await AIIncidentInvestigationRunner.shouldInvestigateIncident({
+            projectId: subject.projectId,
+            incidentId: subject.incidentId!,
+          });
+      if (!gate.investigate) {
+        return this.currentReason(
+          gate.notStartedCode || "eligibility_check_failed",
+          subject,
+          undefined,
+          gate.notStartedDetails,
+        );
+      }
+
+      const budget: AutonomousBudgetStatus =
+        await AIService.getAutonomousDailyBudgetStatus(subject.projectId, {
+          incidentId: subject.incidentId,
+          alertId: subject.alertId,
+        });
+      if (budget.exhausted) {
+        return this.currentReason("daily_budget_exhausted", subject, budget);
+      }
+      return { ...this.reason("no_run_recorded", subject), source: "unknown" };
+    } catch (error) {
+      logger.error(`AI: could not explain missing investigation: ${error}`);
+      return {
+        ...this.reason("eligibility_check_failed", subject),
+        description:
+          "No investigation was found. The original decision or current eligibility checks could not be loaded.",
+        source: "unknown",
+      };
+    }
+  }
+
+  private static currentReason(
+    code: InvestigationNotStartedCode,
+    subject: InvestigationSubject,
+    budget?: AutonomousBudgetStatus,
+    details?: InvestigationGateDetails,
+  ): InvestigationNotStartedReason {
+    return this.buildReason(
+      "current_configuration",
+      code,
+      subject,
+      budget,
+      details,
+    );
+  }
+}

package/Server/Utils/AI/SRE/InvestigationQueue.ts

@@ -1,3 +1,4 @@
+import { InvestigationNotStartedCode } from "../../../../Types/AI/InvestigationNotStartedReason";
 import ObjectID from "../../../../Types/ObjectID";
 import OneUptimeDate from "../../../../Types/Date";
 import Query from "../../../../Types/BaseDatabase/Query";
@@ -127,6 +128,12 @@ export default class AIInvestigationQueue {
     subjectAlertId?: ObjectID | undefined;
     subjectMonitorId?: ObjectID | undefined;
     subjectAIInsightId?: ObjectID | undefined;
+    onNotEnqueued?:
+      | ((
+          code: InvestigationNotStartedCode,
+          budget?: AutonomousBudgetStatus,
+        ) => Promise<void>)
+      | undefined;
     /*
      * When set, the run is a remediation run (not an Investigation): by
      * default a read-only RemediationPlan that picks a runbook for this
@@ -146,6 +153,21 @@ export default class AIInvestigationQueue {
   }): Promise<ObjectID | null> {
     const { projectId } = data;
 
+    // Diagnostic persistence must never change queue admission or remediation.
+    const recordSkip: (
+      code: InvestigationNotStartedCode,
+      budget?: AutonomousBudgetStatus,
+    ) => Promise<void> = async (
+      code: InvestigationNotStartedCode,
+      budget?: AutonomousBudgetStatus,
+    ): Promise<void> => {
+      try {
+        await data.onNotEnqueued?.(code, budget);
+      } catch (error) {
+        logger.error(`AI: could not record queue admission decision: ${error}`);
+      }
+    };
+
     if (data.subjectIncidentId && data.subjectAlertId) {
       logger.error(
         `AI: not enqueueing investigation for project ${projectId.toString()} — a run cannot belong to both an incident and an alert.`,
@@ -166,12 +188,14 @@ export default class AIInvestigationQueue {
         });
 
       if (budget.exhausted) {
+        await recordSkip("daily_budget_exhausted", budget);
         logger.debug(
           `AI: not enqueueing investigation for project ${projectId.toString()} — daily autonomous token budget exhausted (${budget.usedTokensToday} of ${budget.limitInTokens} tokens used today).`,
         );
         return null;
       }
     } catch (error) {
+      await recordSkip("budget_check_failed");
       logger.error(
         `AI: budget check failed, not enqueueing investigation: ${error}`,
       );
@@ -240,6 +264,7 @@ export default class AIInvestigationQueue {
             )
           : await createRun();
     } catch (error) {
+      await recordSkip("enqueue_failed");
       logger.error(`AI: failed to enqueue investigation run: ${error}`);
       return null;
     }

package/Server/Utils/AI/Toolbox/Index.ts

@@ -16,6 +16,11 @@ import {
 import { QueryIncidentsTool, SearchIncidentsTool } from "./IncidentTools";
 import { QueryAlertsTool } from "./AlertTools";
 import { QueryMonitorsTool } from "./MonitorTools";
+import { QueryRumApplicationsTool, QueryRumWebVitalsTool } from "./RumTools";
+import {
+  QueryTelemetryResourcesTool,
+  QueryResourceTelemetryTool,
+} from "./ResourceTools";
 import { QueryScheduledMaintenanceTool } from "./ScheduledMaintenanceTools";
 import {
   GetOnCallStatusTool,
@@ -100,6 +105,10 @@ export default class AIToolbox {
     GetAlertTimelineTool,
     QueryAlertsTool,
     QueryMonitorsTool,
+    QueryRumApplicationsTool,
+    QueryRumWebVitalsTool,
+    QueryTelemetryResourcesTool,
+    QueryResourceTelemetryTool,
     QueryScheduledMaintenanceTool,
     /*
      * Platform reads: the operational surface an on-call product exists to
@@ -246,6 +255,7 @@ export default class AIToolbox {
   public static hasPermissionForTool(
     tool: ObservabilityTool,
     ctx: ToolContext,
+    args?: JSONObject,
   ): boolean {
     if (ctx.props.isRoot || ctx.props.isMasterAdmin) {
       return true;
@@ -270,15 +280,6 @@ export default class AIToolbox {
         return userPermission.permission;
       });
 
-    if (
-      PermissionHelper.doesPermissionsIntersect(
-        blockedPermissions,
-        tool.requiredPermissions,
-      )
-    ) {
-      return false;
-    }
-
     const userPermissions: Array<Permission> =
       DatabaseCommonInteractionPropsUtil.getUserPermissions(
         ctx.props,
@@ -287,9 +288,22 @@ export default class AIToolbox {
         return userPermission.permission;
       });
 
-    return PermissionHelper.doesPermissionsIntersect(
-      userPermissions,
-      tool.requiredPermissions,
+    const groups: Array<Array<Permission>> =
+      args !== undefined && tool.getRequiredPermissionGroups
+        ? tool.getRequiredPermissionGroups(args)
+        : [tool.requiredPermissions];
+    return (
+      groups.length > 0 &&
+      groups.every((group: Array<Permission>): boolean => {
+        return (
+          group.length > 0 &&
+          !PermissionHelper.doesPermissionsIntersect(
+            blockedPermissions,
+            group,
+          ) &&
+          PermissionHelper.doesPermissionsIntersect(userPermissions, group)
+        );
+      })
     );
   }
 
@@ -321,7 +335,23 @@ export default class AIToolbox {
       };
     }
 
-    if (!this.hasPermissionForTool(tool, data.ctx)) {
+    /*
+     * Tool JSON is untrusted at runtime even though callers are typed. Check
+     * the envelope before argument-dependent permission functions read it.
+     */
+    if (
+      !data.args ||
+      typeof data.args !== "object" ||
+      Array.isArray(data.args)
+    ) {
+      return {
+        success: false,
+        textForLlm: `Error: arguments for ${data.name} must be a JSON object. Adjust the arguments and try again.`,
+        errorMessage: `Invalid arguments for tool: ${data.name}; expected a JSON object`,
+      };
+    }
+
+    if (!this.hasPermissionForTool(tool, data.ctx, data.args)) {
       return {
         success: false,
         textForLlm: `Error: the current user does not have permission to use ${data.name}. Answer with the data you already have, and tell the user which permission is missing.`,