@oneuptime/common 14.0.7 → 14.0.9

package/Server/Utils/AI/Remediation/RemediationExecutionRunner.ts

@@ -107,6 +107,26 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  * suggestion settles what happened instead of re-executing anything.
  */
 
+/*
+ * A cluster round: remediation a cluster's AI agent page asked for (its
+ * Fixes mode), not an auto-remediation rule — it names a cluster and no
+ * rule. The project's "Enable AI command execution" opt-in gates rule
+ * rounds only; a cluster round's consent lives on the cluster (see
+ * RemediationExecutionRunner.checkProjectGates). Keyed on the suggestion
+ * row, never on the plan's step types: a rule can compose an all-kubectl
+ * plan, and it was composed under the opt-in, so it stays under it. Shared
+ * with the approve route so execution and approval can never disagree about
+ * which rounds need the opt-in.
+ */
+export function isClusterRemediationRound(suggestion: {
+  kubernetesClusterId?: ObjectID | undefined;
+  autoRemediationRuleId?: ObjectID | undefined;
+}): boolean {
+  return (
+    Boolean(suggestion.kubernetesClusterId) && !suggestion.autoRemediationRuleId
+  );
+}
+
 const MAX_SIGNAL_TITLE_CHARS: number = 500;
 const MAX_SIGNAL_DESCRIPTION_CHARS: number = 4000;
 const MAX_POSTED_ANALYSIS_CHARS: number = 6000;
@@ -507,25 +527,16 @@ export default class RemediationExecutionRunner {
         return;
       }
 
-      // Gates that must still hold at execution time, not just at rule match.
-      const gateFailure: string | null =
-        await this.checkProjectGates(projectId);
-      if (gateFailure) {
-        await this.settleNoneApplicable({
-          suggestion,
-          rationaleMarkdown: gateFailure,
-        });
-        await this.completeRunQuietly(aiRunId);
-        return;
-      }
-
       /*
        * A cluster-level round whose cluster was deleted while it waited in
        * the queue: the delete nulled kubernetesClusterId (ON DELETE SET
        * NULL), and a cluster round never had a rule — so without this it
        * would fall into the rule lane below and be closed as "the rule
        * behind this suggestion was deleted", naming a rule that never
-       * existed. Its server-written name is what is left to tell.
+       * existed. Its server-written name is what is left to tell. Checked
+       * before the project gates: nothing is left to run, and with no
+       * cluster on the row it no longer reads as a cluster round, so the
+       * gates would otherwise blame an opt-in it never needed.
        */
       const deletedClusterRound: { clusterName: string } | null =
         !suggestion.kubernetesClusterId && !suggestion.autoRemediationRuleId
@@ -541,6 +552,20 @@ export default class RemediationExecutionRunner {
         return;
       }
 
+      // Gates that must still hold at execution time, not just at rule match.
+      const gateFailure: string | null = await this.checkProjectGates({
+        projectId,
+        isClusterRound: isClusterRemediationRound(suggestion),
+      });
+      if (gateFailure) {
+        await this.settleNoneApplicable({
+          suggestion,
+          rationaleMarkdown: gateFailure,
+        });
+        await this.completeRunQuietly(aiRunId);
+        return;
+      }
+
       if (suggestion.kubernetesClusterId) {
         /*
          * Cluster-level remediation: the cluster's AI page plays the rule's
@@ -565,7 +590,7 @@ export default class RemediationExecutionRunner {
             suggestion,
             rationaleMarkdown: `OneUptime AI can no longer remediate cluster "${
               clusterTarget?.clusterName || "(deleted)"
-            }"${firstGap ? `: ${firstGap}` : ""}. Nothing was run or proposed. Review the cluster's AI page.`,
+            }"${firstGap ? `: ${firstGap}` : ""}. Nothing was run or proposed. Review the cluster's AI agent page (AI → Agent).`,
           });
           await this.completeRunQuietly(aiRunId);
           return;
@@ -1119,7 +1144,7 @@ export default class RemediationExecutionRunner {
     if (live.withdrawnReason) {
       await this.settleNoneApplicable({
         suggestion,
-        rationaleMarkdown: `${live.withdrawnReason} Review the cluster's AI page.\n\n${data.rationaleMarkdown}`,
+        rationaleMarkdown: `${live.withdrawnReason} Review the cluster's AI agent page (AI → Agent).\n\n${data.rationaleMarkdown}`,
       });
       return;
     }
@@ -1386,12 +1411,26 @@ export default class RemediationExecutionRunner {
       });
   }
 
-  // Null when all gates pass; otherwise the rationale for settling quietly.
-  private static async checkProjectGates(
-    projectId: ObjectID,
-  ): Promise<string | null> {
+  /*
+   * The project switches a round must still pass when it runs. Null when
+   * they do; otherwise the rationale for settling quietly.
+   *
+   * Enable AI and Enable auto-remediation are kill switches for every
+   * round. The "Enable AI command execution" opt-in gates rule rounds
+   * (Bash/SSH on Runners, and kubectl a rule composes) but not a cluster
+   * round (isClusterRemediationRound): its consent is the cluster's own
+   * Fixes mode plus, for the in-cluster Kubernetes AI agent, the chart's
+   * write RBAC. A cluster reached through an advanced Runner with a
+   * Kubernetes credential has no chart RBAC behind it, so its status keeps
+   * the project_ai_command_execution_disabled gap while the opt-in is off —
+   * and the cluster round stops at isRemediationReady instead.
+   */
+  public static async checkProjectGates(data: {
+    projectId: ObjectID;
+    isClusterRound: boolean;
+  }): Promise<string | null> {
     const project: Project | null = await ProjectService.findOneById({
-      id: projectId,
+      id: data.projectId,
       select: {
         enableAi: true,
         enableAutoRemediation: true,
@@ -1405,15 +1444,19 @@ export default class RemediationExecutionRunner {
       project.enableAi === false ||
       project.enableAutoRemediation === false
     ) {
-      return "AI or auto-remediation was disabled for this project before the run started — nothing was run or proposed.";
+      return "AI or auto-remediation was disabled for this project before the run started (Project Settings → AI Features) — nothing was run or proposed.";
+    }
+
+    if (data.isClusterRound) {
+      return null;
     }
 
     /*
-     * Opt-in semantics (=== true): AI command execution never runs in a
-     * project that has not explicitly turned it on.
+     * Opt-in semantics (=== true): a rule's AI command execution never runs
+     * in a project that has not explicitly turned it on.
      */
     if (project.enableAiCommandExecution !== true) {
-      return "AI command execution is not enabled for this project (Project Settings → AI) — nothing was run or proposed.";
+      return "AI command execution is not enabled for this project (Project Settings → AI Features) — nothing was run or proposed.";
     }
 
     return null;

package/Server/Utils/AI/SRE/AIInvestigationEngine.ts

@@ -18,6 +18,7 @@ import Project from "../../../../Models/DatabaseModels/Project";
 import LlmProvider from "../../../../Models/DatabaseModels/LlmProvider";
 import AIRunService from "../../../Services/AIRunService";
 import AIRunEventService from "../../../Services/AIRunEventService";
+import AIService from "../../../Services/AIService";
 import ProjectService from "../../../Services/ProjectService";
 import LlmProviderService from "../../../Services/LlmProviderService";
 import AIInvestigationQueue from "./InvestigationQueue";
@@ -205,10 +206,10 @@ export interface InvestigationRequest {
 
 export default class AIInvestigationEngine {
   /*
-   * Shared gate: AI enabled, the subject's auto-investigation opt-in on, and an
-   * LLM provider configured. Incidents and alerts each have their own opt-in so
-   * they can be enabled independently. Runs before any (subject-specific)
-   * context assembly.
+   * Shared gate: AI enabled, the subject's auto-investigation opt-in on, an
+   * LLM provider configured, and AI credits to pay for it. Incidents and
+   * alerts each have their own opt-in so they can be enabled independently.
+   * Runs before any (subject-specific) context assembly.
    */
   @CaptureSpan()
   public static async isEnabledForProject(
@@ -249,16 +250,58 @@ export default class AIInvestigationEngine {
       return "automatic_investigation_disabled";
     }
 
+    return this.getProviderOrBalanceReason(projectId);
+  }
+
+  /*
+   * The half of the gate that is not a switch: a model to call, and AI
+   * credits to pay for it. Shared with the other autonomous AI work that
+   * has its own switch (the postmortem draft on resolve), so the two can
+   * never disagree about whether AI can run at all.
+   *
+   * The balance check is the same predicate as the Kubernetes cluster's
+   * ai_balance_insufficient gap (AIService.getAiBalanceBlocker): on
+   * OneUptime's own billed provider, with no credits left and auto-recharge
+   * off, every model call would be refused with "Insufficient AI balance" —
+   * so a run started now would only fail, be retried, and fail again, and
+   * nobody would see why. It fails OPEN: a balance that cannot be read
+   * never blocks, because the model call itself still enforces the balance.
+   */
+  public static async getProviderOrBalanceReason(
+    projectId: ObjectID,
+  ): Promise<"provider_missing" | "insufficient_ai_balance" | null> {
     const llmProvider: LlmProvider | null =
       await LlmProviderService.getLLMProviderForProject(projectId);
 
     if (!llmProvider) {
       logger.debug(
-        `AI: skipping investigation for project ${projectId.toString()} — no LLM provider configured.`,
+        `AI: skipping autonomous AI work for project ${projectId.toString()} — no LLM provider configured.`,
       );
       return "provider_missing";
     }
 
+    let balanceBlocker: string | null = null;
+
+    try {
+      balanceBlocker = await AIService.getAiBalanceBlocker({
+        projectId,
+        // Already resolved above — spares the predicate a second lookup.
+        llmProvider,
+      });
+    } catch (error) {
+      logger.error(
+        `AI: could not check the AI balance for project ${projectId.toString()}; not blocking on it: ${error}`,
+      );
+      balanceBlocker = null;
+    }
+
+    if (balanceBlocker) {
+      logger.debug(
+        `AI: skipping autonomous AI work for project ${projectId.toString()} — ${balanceBlocker}`,
+      );
+      return "insufficient_ai_balance";
+    }
+
     return null;
   }
 

package/Server/Utils/AI/SRE/IncidentPostmortemRunner.ts

@@ -2,8 +2,10 @@ import ObjectID from "../../../../Types/ObjectID";
 import { Blue500 } from "../../../../Types/BrandColors";
 import Incident from "../../../../Models/DatabaseModels/Incident";
 import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
+import Project from "../../../../Models/DatabaseModels/Project";
 import IncidentService from "../../../Services/IncidentService";
 import IncidentFeedService from "../../../Services/IncidentFeedService";
+import ProjectService from "../../../Services/ProjectService";
 import AIInvestigationEngine from "./AIInvestigationEngine";
 import logger from "../../Logger";
 import CaptureSpan from "../../Telemetry/CaptureSpan";
@@ -17,12 +19,47 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
  * human to review and edit — turning a ~90-minute manual writeup into a review.
  *
  * It NEVER overwrites a postmortem that already exists (human work wins), is
- * gated by the same per-project opt-in + LLM provider as investigations, and is
- * fire-and-forget: failures are logged, never surfaced to the resolve flow.
+ * gated by its own per-project opt-in (Project.enableAutomaticPostmortemDraft)
+ * plus the same AI switch, LLM provider and AI balance as investigations, and
+ * is fire-and-forget: failures are logged, never surfaced to the resolve flow.
  */
 const MAX_FEED_PREVIEW_CHARS: number = 6000;
 
 export default class AIIncidentPostmortemRunner {
+  /*
+   * The draft has its own switch, separate from automatic incident
+   * investigation. Investigations are read-only and on by default for new
+   * projects; a draft writes to the incident and posts to the project's
+   * Slack/Teams channels, so a project turns it on deliberately (opt-in:
+   * === true). The AI kill switch, the LLM provider and the AI balance gate
+   * it exactly as they gate an investigation.
+   */
+  public static async isEnabledForProject(
+    projectId: ObjectID,
+  ): Promise<boolean> {
+    const project: Project | null = await ProjectService.findOneById({
+      id: projectId,
+      select: {
+        enableAi: true,
+        enableAutomaticPostmortemDraft: true,
+      },
+      props: { isRoot: true },
+    });
+
+    if (
+      !project ||
+      project.enableAi === false ||
+      project.enableAutomaticPostmortemDraft !== true
+    ) {
+      return false;
+    }
+
+    return (
+      (await AIInvestigationEngine.getProviderOrBalanceReason(projectId)) ===
+      null
+    );
+  }
+
   @CaptureSpan()
   public static async draftPostmortemOnResolve(data: {
     incidentId: ObjectID;
@@ -31,12 +68,7 @@ export default class AIIncidentPostmortemRunner {
     const { incidentId, projectId } = data;
 
     try {
-      if (
-        !(await AIInvestigationEngine.isEnabledForProject(
-          projectId,
-          "Incident",
-        ))
-      ) {
+      if (!(await this.isEnabledForProject(projectId))) {
         return;
       }
 

package/Server/Utils/AI/SRE/InvestigationEligibility.ts

@@ -54,8 +54,8 @@ export default class InvestigationEligibility {
           ? `AI was disabled for this project when this ${kind} was created, so its automatic investigation did not start.`
           : "AI is currently disabled for this project, so new automatic investigations cannot start.",
         nextStep: recorded
-          ? "Review project AI settings for new records. Existing records are not automatically retried when AI is enabled."
-          : "Enable project AI and automatic investigations to investigate new records. Existing records are not automatically retried.",
+          ? "Review project AI settings (Project Settings → AI Features) for new records. Existing records are not automatically retried when AI is enabled."
+          : "Turn on Enable AI (Project Settings → AI Features) and automatic investigations to investigate new records. Existing records are not automatically retried.",
       },
       automatic_investigation_disabled: {
         title: recorded
@@ -77,6 +77,17 @@ export default class InvestigationEligibility {
           ? "Review the AI provider configuration for new investigations. Configuring a provider later does not automatically retry existing records."
           : "Configure an AI provider in project settings. New eligible records can then be investigated; existing records are not automatically retried.",
       },
+      insufficient_ai_balance: {
+        title: recorded
+          ? "The project was out of AI credits at creation"
+          : "The project is currently out of AI credits",
+        description: recorded
+          ? `This project had no AI credits left when this ${kind} was created, so its automatic investigation did not start.`
+          : "This project is currently out of AI credits, so new automatic investigations cannot start.",
+        nextStep: recorded
+          ? "Review AI credits under Project Settings → AI Credits, or turn on auto-recharge. Adding credits later does not retry this record."
+          : "Add AI credits under Project Settings → AI Credits, or turn on auto-recharge. Existing records are not automatically retried.",
+      },
       severity_below_threshold: {
         title: recorded
           ? "Severity was below the investigation threshold at creation"

package/Server/Utils/AI/SRE/InvestigationGrader.ts

@@ -145,7 +145,7 @@ export default class InvestigationGrader {
        * executeWithLogging throw would file a permanent error for every
        * incident such a project ever resolves. It is a silent skip instead —
        * the same posture the sibling on-resolve job takes via
-       * AIInvestigationEngine.isEnabledForProject.
+       * AIIncidentPostmortemRunner.isEnabledForProject.
        */
       if (!(await AIService.isProjectAIEnabled(projectId))) {
         logger.debug(

package/Server/Utils/AutoRemediation/CommandPlanExecutor.ts

@@ -27,6 +27,7 @@ import KubectlPolicy, {
   KubectlPolicyResult,
 } from "../../../Utils/AiRemediation/KubectlPolicy";
 import {
+  KUBERNETES_AI_AGENT_DISPLAY_NAME,
   KubectlCommandTier,
   KubernetesAiAccessGap,
   KubernetesAiRemediationMode,
@@ -1023,7 +1024,11 @@ export default class CommandPlanExecutor {
     }
 
     if (!status.runner || status.runner.id !== command.runnerId) {
-      return `cluster "${status.clusterName}" is no longer reached through Runner "${command.runnerNameSnapshot}", which this plan was composed for`;
+      return `cluster "${status.clusterName}" is no longer reached through ${
+        command.runnerNameSnapshot === KUBERNETES_AI_AGENT_DISPLAY_NAME
+          ? "the Kubernetes AI agent"
+          : `Runner "${command.runnerNameSnapshot}"`
+      }, which this plan was composed for`;
     }
 
     if (