@oneuptime/common 14.0.7 → 14.0.9

package/Tests/Server/Services/KubernetesClusterAiAccessGapNextSteps.test.ts

@@ -1,12 +1,21 @@
 import KubernetesClusterAiAccessService, {
+  AI_AGENT_INSTALL_COMMAND,
+  AI_AGENT_NOT_CONNECTED_NEXT_STEP,
+  CLUSTER_AI_AGENT_PAGE,
+  CREDENTIAL_RUNNER_REMEDIATION_WRITE_ACCESS_NEXT_STEP,
   KubernetesClusterAiAccessProjectGates,
+  LEGACY_RUNNER_REMEDIATION_WRITE_ACCESS_NEXT_STEP,
   REMEDIATION_WRITE_ACCESS_NEXT_STEP,
+  getAiAgentLogsCommand,
+  getRemediationWriteAccessNextStep,
 } from "../../../Server/Services/KubernetesClusterAiAccessService";
+import KubernetesAiAgentService from "../../../Server/Services/KubernetesAiAgentService";
 import KubernetesClusterService from "../../../Server/Services/KubernetesClusterService";
 import RunbookCredentialService from "../../../Server/Services/RunbookCredentialService";
 import RunbookSecretService from "../../../Server/Services/RunbookSecretService";
 import RunnerService from "../../../Server/Services/RunnerService";
 import logger from "../../../Server/Utils/Logger";
+import KubernetesAiAgent from "../../../Models/DatabaseModels/KubernetesAiAgent";
 import KubernetesCluster from "../../../Models/DatabaseModels/KubernetesCluster";
 import Runner, {
   RunnerConnectionStatus,
@@ -20,20 +29,21 @@ import {
   KubernetesAiRemediationMode,
   KubernetesClusterAiAccessStatus,
 } from "../../../Types/Kubernetes/KubernetesClusterAiAccess";
-import fs from "fs";
-import path from "path";
 import { afterEach, beforeEach, describe, expect, it } from "@jest/globals";
 
 /*
  * Contract under test — the next step of every readiness gap is written for
- * wherever it is shown. The cluster's AI page shows it, and so does the
- * investigation panel on an incident or alert page (verbatim, for a viewer
- * who may read the cluster). So a next step never says "this page" or
- * "here", which on an incident page would point at the incident; one that
- * sends the operator to the AI page names it: "the cluster's AI page".
+ * wherever it is shown. The cluster's AI agent page shows it, and so does
+ * the investigation panel on an incident or alert page (verbatim, for a
+ * viewer who may read the cluster). So a next step never says "this page"
+ * or "here", which on an incident page would point at the incident; one
+ * that sends the operator to the cluster's page names it: "the cluster's AI
+ * agent page (AI → Agent)" — never the retired "cluster's AI page".
  *
  * Every gap is produced by the real status computation, one scenario per
- * branch, so a next step added or reworded later is checked too.
+ * branch (the Kubernetes AI agent, the previous in-cluster Runner, an
+ * advanced Runner, nothing at all), so a next step added or reworded later
+ * is checked too.
  */
 
 const PROJECT_ID: ObjectID = new ObjectID(
@@ -48,12 +58,14 @@ const RUNNER_ID: ObjectID = new ObjectID(
 const CREDENTIAL_ID: ObjectID = new ObjectID(
   "55555555-5555-4555-8555-555555555555",
 );
+const AGENT_ID: ObjectID = new ObjectID("66666666-6666-4666-8666-666666666666");
 
 const READY_GATES: KubernetesClusterAiAccessProjectGates = {
   isAiEnabled: true,
   isAutoRemediationEnabled: true,
   isAiCommandExecutionEnabled: true,
   hasLlmProvider: true,
+  aiBalanceBlocker: null,
 };
 
 const CLOSED_GATES: KubernetesClusterAiAccessProjectGates = {
@@ -61,14 +73,14 @@ const CLOSED_GATES: KubernetesClusterAiAccessProjectGates = {
   isAutoRemediationEnabled: false,
   isAiCommandExecutionEnabled: false,
   hasLlmProvider: false,
+  aiBalanceBlocker: "This project's AI credit balance is used up.",
 };
 
 // "this page" / "on this page" / "here" read as the incident on an incident page.
 const PAGE_RELATIVE_WORDING: RegExp = /\bthis page\b|\bhere\b/i;
 
-// Any mention of an AI page must say whose.
-const AI_PAGE_MENTION: RegExp = /\bAI page\b/;
-const NAMED_AI_PAGE: string = "the cluster's AI page";
+// Any mention of the cluster's page must say whose, and which one.
+const AI_PAGE_MENTION: RegExp = /\bAI (agent )?page\b/;
 
 function cluster(overrides: Record<string, unknown> = {}): KubernetesCluster {
   return {
@@ -85,6 +97,24 @@ function cluster(overrides: Record<string, unknown> = {}): KubernetesCluster {
   } as unknown as KubernetesCluster;
 }
 
+function agent(overrides: Record<string, unknown> = {}): KubernetesAiAgent {
+  return {
+    id: AGENT_ID,
+    _id: AGENT_ID.toString(),
+    projectId: PROJECT_ID,
+    kubernetesClusterId: CLUSTER_ID,
+    connectionStatus: "connected",
+    lastAliveAt: OneUptimeDate.getCurrentDate(),
+    posture: {
+      inCluster: true,
+      allowWrites: true,
+      clusterIdentifier: "prod-us",
+      podNamespace: "oneuptime-agent",
+    },
+    ...overrides,
+  } as unknown as KubernetesAiAgent;
+}
+
 function agentRunner(overrides: Record<string, unknown> = {}): Runner {
   return {
     id: RUNNER_ID,
@@ -117,33 +147,71 @@ function dashboardRunner(overrides: Record<string, unknown> = {}): Runner {
 interface Scenario {
   label: string;
   cluster: KubernetesCluster;
-  // What the Runner lookup (bound id, or this cluster's agent by name) answers.
+  // What the bound Runner lookup answers.
   runner: Runner | null;
+  // The cluster's Kubernetes AI agent row.
+  agent: KubernetesAiAgent | null;
   gates?: KubernetesClusterAiAccessProjectGates | undefined;
 }
 
 const SCENARIOS: Array<Scenario> = [
   {
-    label: "no Runner bound and none registered",
+    label: "no agent and no Runner",
     cluster: cluster({ aiAccessRunnerId: undefined }),
     runner: null,
+    agent: null,
   },
   {
-    label: "no Runner bound, this cluster's agent registered",
+    label: "the agent stopped reporting",
     cluster: cluster({ aiAccessRunnerId: undefined }),
-    runner: agentRunner(),
+    runner: null,
+    agent: agent({ lastAliveAt: OneUptimeDate.getSomeMinutesAgo(30) }),
+  },
+  {
+    label: "the agent signed off",
+    cluster: cluster({ aiAccessRunnerId: undefined }),
+    runner: null,
+    agent: agent({ connectionStatus: "disconnected" }),
+  },
+  {
+    label: "the agent reports another cluster",
+    cluster: cluster({ aiAccessRunnerId: undefined }),
+    runner: null,
+    agent: agent({
+      posture: { inCluster: true, allowWrites: true, clusterIdentifier: "x" },
+    }),
+  },
+  {
+    label: "a read-only agent asked to remediate",
+    cluster: cluster({ aiAccessRunnerId: undefined }),
+    runner: null,
+    agent: agent({
+      posture: {
+        inCluster: true,
+        allowWrites: false,
+        clusterIdentifier: "prod-us",
+      },
+    }),
   },
   {
     label: "the bound Runner was just deleted",
     cluster: cluster(),
     runner: null,
+    agent: null,
   },
   {
-    label: "the in-cluster Runner signed off",
+    label: "the previous in-cluster Runner signed off",
     cluster: cluster(),
     runner: agentRunner({
       connectionStatus: RunnerConnectionStatus.Disconnected,
     }),
+    agent: null,
+  },
+  {
+    label: "the previous in-cluster Runner is offline",
+    cluster: cluster(),
+    runner: agentRunner({ lastAlive: OneUptimeDate.getSomeMinutesAgo(30) }),
+    agent: null,
   },
   {
     label: "a dashboard Runner signed off",
@@ -151,70 +219,42 @@ const SCENARIOS: Array<Scenario> = [
     runner: dashboardRunner({
       connectionStatus: RunnerConnectionStatus.Disconnected,
     }),
+    agent: null,
   },
   {
     label: "a Runner that never connected",
     cluster: cluster(),
     runner: dashboardRunner({ lastAlive: undefined }),
-  },
-  {
-    label: "an offline in-cluster Runner",
-    cluster: cluster(),
-    runner: agentRunner({ lastAlive: OneUptimeDate.getSomeMinutesAgo(30) }),
-  },
-  {
-    label: "an offline in-cluster Runner that holds more than the defaults",
-    cluster: cluster(),
-    runner: agentRunner({
-      lastAlive: OneUptimeDate.getSomeMinutesAgo(30),
-      canRunRunbooks: true,
-    }),
+    agent: null,
   },
   {
     label: "a Runner without the AI-commands capability",
     cluster: cluster(),
     runner: agentRunner({ canRunAiCommands: false }),
-  },
-  {
-    label: "another cluster's agent Runner with a credential",
-    cluster: cluster({ aiAccessCredentialId: CREDENTIAL_ID }),
-    runner: agentRunner({
-      name: "kubernetes-agent/staging",
-      hostInfo: {
-        kubernetes: {
-          inCluster: true,
-          allowWrites: true,
-          clusterIdentifier: "staging",
-        },
-      },
-    }),
+    agent: null,
   },
   {
     label: "an external Runner whose credential is gone",
     cluster: cluster({ aiAccessCredentialId: CREDENTIAL_ID }),
     runner: dashboardRunner(),
+    agent: null,
   },
   {
-    label: "another cluster's agent Runner without a credential",
+    label: "an external Runner without a credential",
     cluster: cluster(),
-    runner: agentRunner({
-      name: "kubernetes-agent/staging",
-      hostInfo: {
-        kubernetes: {
-          inCluster: true,
-          allowWrites: true,
-          clusterIdentifier: "staging",
-        },
-      },
-    }),
+    runner: dashboardRunner(),
+    agent: null,
   },
   {
-    label: "an external Runner without a credential",
+    label: "a pod Runner that never said which cluster it runs in",
     cluster: cluster(),
-    runner: dashboardRunner(),
+    runner: dashboardRunner({
+      hostInfo: { kubernetes: { inCluster: true, allowWrites: false } },
+    }),
+    agent: null,
   },
   {
-    label: "a read-only in-cluster Runner asked to remediate",
+    label: "a read-only previous in-cluster Runner asked to remediate",
     cluster: cluster(),
     runner: agentRunner({
       hostInfo: {
@@ -225,27 +265,30 @@ const SCENARIOS: Array<Scenario> = [
         },
       },
     }),
+    agent: null,
   },
   {
-    label: "both switches off, every project gate closed",
+    label: "both switches off, every project gate closed, an advanced Runner",
     cluster: cluster({
       isAiInvestigationEnabled: false,
       aiRemediationMode: KubernetesAiRemediationMode.Disabled,
     }),
-    runner: agentRunner(),
+    runner: dashboardRunner(),
+    agent: null,
     gates: CLOSED_GATES,
   },
 ];
 
 // Every gap the status computation can produce today.
 const PRODUCED_GAP_CODES: Array<KubernetesAiAccessGapCode> = [
-  "no_runner_bound",
+  "ai_agent_not_connected",
+  "ai_agent_offline",
+  "ai_balance_insufficient",
   "runner_missing",
   "runner_offline",
   "runner_ai_commands_disabled",
   "runner_cluster_mismatch",
   "credential_missing",
-  "credential_on_agent_runner",
   "investigation_disabled",
   "remediation_disabled",
   "remediation_write_access_missing",
@@ -255,6 +298,18 @@ const PRODUCED_GAP_CODES: Array<KubernetesAiAccessGapCode> = [
   "llm_provider_missing",
 ];
 
+/*
+ * Codes kept in the union for compatibility that the status no longer
+ * produces: the AI agent replaced "bind a Runner" (no_runner_bound), a
+ * kubernetes-agent row is never an advanced Runner any more
+ * (credential_on_agent_runner), and last_access_check_failed never was.
+ */
+const RETIRED_GAP_CODES: Array<KubernetesAiAccessGapCode> = [
+  "no_runner_bound",
+  "credential_on_agent_runner",
+  "last_access_check_failed",
+];
+
 describe("KubernetesClusterAiAccessService gap next steps", () => {
   beforeEach(() => {
     jest.spyOn(logger, "error").mockImplementation((): void => {
@@ -280,6 +335,9 @@ describe("KubernetesClusterAiAccessService gap next steps", () => {
     scenario: Scenario,
   ): Promise<Array<KubernetesAiAccessGap>> {
     jest.spyOn(RunnerService, "findOneBy").mockResolvedValue(scenario.runner);
+    jest
+      .spyOn(KubernetesAiAgentService, "findForCluster")
+      .mockResolvedValue(scenario.agent);
 
     const status: KubernetesClusterAiAccessStatus =
       await KubernetesClusterAiAccessService.getStatusForClusterModel({
@@ -322,6 +380,12 @@ describe("KubernetesClusterAiAccessService gap next steps", () => {
     expect([...codes].sort()).toEqual([...PRODUCED_GAP_CODES].sort());
   });
 
+  it("never produces a retired code", async () => {
+    for (const entry of await everyGap()) {
+      expect(RETIRED_GAP_CODES).not.toContain(entry.gap.code);
+    }
+  });
+
   it('no next step says "this page" or "here"', async () => {
     const offenders: Array<string> = (await everyGap())
       .filter((entry: { scenario: string; gap: KubernetesAiAccessGap }) => {
@@ -334,88 +398,186 @@ describe("KubernetesClusterAiAccessService gap next steps", () => {
     expect(offenders).toEqual([]);
   });
 
-  it("a next step that sends the operator to the AI page names it as the cluster's", async () => {
+  it("a next step that sends the operator to the cluster's page names it as the AI agent page (AI → Agent)", async () => {
     const entries: Array<{ scenario: string; gap: KubernetesAiAccessGap }> = (
       await everyGap()
     ).filter((entry: { scenario: string; gap: KubernetesAiAccessGap }) => {
       return AI_PAGE_MENTION.test(entry.gap.nextStep);
     });
 
-    // Most cluster-level steps are taken on the AI page.
+    // Most cluster-level steps are taken on the AI agent page.
     expect(entries.length).toBeGreaterThanOrEqual(8);
 
     for (const entry of entries) {
       expect(`${entry.gap.code}: ${entry.gap.nextStep}`).toContain(
-        NAMED_AI_PAGE,
+        CLUSTER_AI_AGENT_PAGE,
       );
-      expect(entry.gap.nextStep).not.toContain("this cluster's AI page");
+      expect(entry.gap.nextStep).not.toContain("the cluster's AI page");
+      expect(entry.gap.nextStep).not.toContain("this cluster's AI");
     }
   });
 
-  it("every step taken on the AI page names it", async () => {
-    const onTheAiPage: Array<KubernetesAiAccessGapCode> = [
-      "no_runner_bound",
+  it("every step taken on the AI agent page names it", async () => {
+    const onTheAiAgentPage: Array<KubernetesAiAccessGapCode> = [
       "runner_missing",
-      "credential_on_agent_runner",
       "runner_cluster_mismatch",
+      "credential_missing",
       "investigation_disabled",
       "remediation_disabled",
       "remediation_write_access_missing",
     ];
 
     for (const entry of await everyGap()) {
-      if (onTheAiPage.includes(entry.gap.code)) {
+      /*
+       * A Runner outside the chart is given writes on its own host, not on
+       * the page (see the read-only gap's tests below).
+       */
+      const isRunnerHostStep: boolean =
+        entry.gap.nextStep ===
+        CREDENTIAL_RUNNER_REMEDIATION_WRITE_ACCESS_NEXT_STEP;
+
+      if (onTheAiAgentPage.includes(entry.gap.code) && !isRunnerHostStep) {
         expect(`${entry.scenario}: ${entry.gap.nextStep}`).toContain(
-          NAMED_AI_PAGE,
+          "the cluster's AI agent page (AI → Agent)",
         );
       }
     }
   });
 
-  /*
-   * The dashboard tells "installed but not selected" apart from "no Runner
-   * can reach this cluster" only by a quoted kubernetes-agent Runner name in
-   * the no_runner_bound next step (AI.tsx getAiAccessConnectCardMode), so
-   * the reworded step must still quote it.
-   */
-  it("still quotes the agent Runner's name in the 'installed but not selected' step", async () => {
-    const gaps: Array<KubernetesAiAccessGap> = await gapsOf(SCENARIOS[1]!);
-
-    expect(gaps[0]!.code).toBe("no_runner_bound");
-    expect(gaps[0]!.nextStep).toContain('"kubernetes-agent/prod-us"');
-    expect(gaps[0]!.nextStep).not.toContain("aiAccess.enabled=true");
+  it("the project-level steps name Project Settings → AI Features, never AI Credits for a switch", async () => {
+    const gaps: Array<KubernetesAiAccessGap> = await gapsOf(
+      SCENARIOS[SCENARIOS.length - 1]!,
+    );
+
+    const stepOf: (code: KubernetesAiAccessGapCode) => string = (
+      code: KubernetesAiAccessGapCode,
+    ): string => {
+      return (
+        gaps.find((gap: KubernetesAiAccessGap) => {
+          return gap.code === code;
+        })?.nextStep || ""
+      );
+    };
+
+    expect(stepOf("project_ai_disabled")).toBe(
+      "Enable AI under Project Settings → AI Features.",
+    );
+    expect(stepOf("project_auto_remediation_disabled")).toBe(
+      "Enable auto-remediation under Project Settings → AI Features.",
+    );
+    expect(stepOf("project_ai_command_execution_disabled")).toBe(
+      'Turn on "Enable AI Command Execution" under Project Settings → AI Features.',
+    );
+    // Credits are still bought where credits live.
+    expect(stepOf("ai_balance_insufficient")).toBe(
+      "Add AI credits under Project Settings → AI Credits (or enable auto-recharge).",
+    );
+  });
+
+  it("no next step still tells anyone to turn aiAccess on", async () => {
+    for (const entry of await everyGap()) {
+      expect(`${entry.scenario}: ${entry.gap.nextStep}`).not.toContain(
+        "aiAccess.enabled",
+      );
+    }
+  });
+});
+
+describe("the shared install and logs commands", () => {
+  it("prints the install command exactly as every surface prints it", () => {
+    expect(AI_AGENT_INSTALL_COMMAND).toBe(
+      [
+        "helm repo update",
+        "helm upgrade kubernetes-agent oneuptime/kubernetes-agent \\",
+        "  --namespace oneuptime-agent --reuse-values \\",
+        "  --set aiAgent.enabled=true",
+      ].join("\n"),
+    );
   });
 
-  it("negative control: the 'no Runner registered' step quotes no Runner name", async () => {
-    const gaps: Array<KubernetesAiAccessGap> = await gapsOf(SCENARIOS[0]!);
+  it("the not-connected step carries the install command verbatim and the logs command", () => {
+    expect(AI_AGENT_NOT_CONNECTED_NEXT_STEP).toContain(
+      AI_AGENT_INSTALL_COMMAND,
+    );
+    expect(AI_AGENT_NOT_CONNECTED_NEXT_STEP).toContain(
+      "kubectl logs -n <namespace> -l component=ai-agent --tail=100",
+    );
+    expect(AI_AGENT_NOT_CONNECTED_NEXT_STEP).toContain(
+      "use your own release name and namespace",
+    );
+  });
 
-    expect(gaps[0]!.code).toBe("no_runner_bound");
-    expect(gaps[0]!.nextStep).not.toContain('"kubernetes-agent/');
+  it("the logs command names the agent's own namespace when it is known", () => {
+    expect(getAiAgentLogsCommand("oneuptime-agent")).toBe(
+      "kubectl logs -n oneuptime-agent -l component=ai-agent --tail=100",
+    );
+    expect(getAiAgentLogsCommand("  monitoring  ")).toBe(
+      "kubectl logs -n monitoring -l component=ai-agent --tail=100",
+    );
+    expect(getAiAgentLogsCommand(undefined)).toBe(
+      "kubectl logs -n <namespace> -l component=ai-agent --tail=100",
+    );
+    expect(getAiAgentLogsCommand("   ")).toBe(
+      "kubectl logs -n <namespace> -l component=ai-agent --tail=100",
+    );
   });
 });
 
 /*
- * The read-only gap names a section of the cluster's AI page. If the page
- * renames that card, this gap would send the operator looking for a title
- * that is not there.
+ * The read-only gap says what grants writes where the executor is
+ * configured: the AI agent's chart values, the upgrade to the AI agent for
+ * the previous in-cluster Runner, a Runner's own environment otherwise.
+ * The complete command lives on the AI agent page, so the step names it.
  */
-describe("the read-only gap names a section the AI page has", () => {
-  const AI_PAGE_SOURCE: string = path.join(
-    __dirname,
-    "../../../../App/FeatureSet/Dashboard/src/Pages/Kubernetes/View/AI.tsx",
-  );
-  const SECTION_TITLE_REGEX: RegExp = /"Let AI apply fixes \(write access\)"/;
-
-  it("quotes the write-access card's title", () => {
-    const quoted: RegExpMatchArray | null =
-      REMEDIATION_WRITE_ACCESS_NEXT_STEP.match(SECTION_TITLE_REGEX);
-
-    expect(quoted).not.toBeNull();
+describe("the read-only gap's next step, by who runs kubectl", () => {
+  it("the AI agent: aiAgent.remediation.* and the AI agent page", () => {
+    expect(getRemediationWriteAccessNextStep("ai_agent")).toBe(
+      REMEDIATION_WRITE_ACCESS_NEXT_STEP,
+    );
+    expect(REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(
+      "--set aiAgent.remediation.enabled=true",
+    );
+    expect(REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(
+      "aiAgent.remediation.namespaces",
+    );
+    expect(REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(
+      "aiAgent.remediation.nodeOperations=false",
+    );
+    expect(REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain("cluster-wide");
+    expect(REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(CLUSTER_AI_AGENT_PAGE);
+    expect(REMEDIATION_WRITE_ACCESS_NEXT_STEP).not.toContain("aiAccess");
   });
 
-  it("which is the title of a card on the AI page", () => {
-    const source: string = fs.readFileSync(AI_PAGE_SOURCE, "utf8");
+  it("the previous in-cluster Runner: upgrade the chart to the AI agent, then aiAgent.remediation.enabled", () => {
+    expect(getRemediationWriteAccessNextStep("legacy_runner")).toBe(
+      LEGACY_RUNNER_REMEDIATION_WRITE_ACCESS_NEXT_STEP,
+    );
+    expect(LEGACY_RUNNER_REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(
+      "Upgrade the Kubernetes agent chart",
+    );
+    expect(LEGACY_RUNNER_REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(
+      "the Kubernetes AI agent replaces this Runner",
+    );
+    expect(LEGACY_RUNNER_REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(
+      "--set aiAgent.remediation.enabled=true",
+    );
+    expect(LEGACY_RUNNER_REMEDIATION_WRITE_ACCESS_NEXT_STEP).not.toContain(
+      "aiAccess",
+    );
+  });
 
-    expect(source).toContain('title="Let AI apply fixes (write access)"');
+  it("a Runner with a credential: its own environment", () => {
+    expect(getRemediationWriteAccessNextStep("credential_runner")).toBe(
+      CREDENTIAL_RUNNER_REMEDIATION_WRITE_ACCESS_NEXT_STEP,
+    );
+    expect(CREDENTIAL_RUNNER_REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(
+      "ONEUPTIME_KUBECTL_ALLOW_WRITES=true",
+    );
+    expect(CREDENTIAL_RUNNER_REMEDIATION_WRITE_ACCESS_NEXT_STEP).toContain(
+      "ONEUPTIME_KUBECTL_WRITE_NAMESPACES",
+    );
+    expect(CREDENTIAL_RUNNER_REMEDIATION_WRITE_ACCESS_NEXT_STEP).not.toContain(
+      "helm",
+    );
   });
 });