@oneuptime/common 14.0.9 → 14.0.11

package/Tests/Server/Services/ResourceServiceDeleteCleansUpAi.test.ts

@@ -0,0 +1,319 @@
+import AutoRemediationSuggestionService from "../../../Server/Services/AutoRemediationSuggestionService";
+import CephClusterService from "../../../Server/Services/CephClusterService";
+import DatabaseServerService from "../../../Server/Services/DatabaseServerService";
+import DatabaseService from "../../../Server/Services/DatabaseService";
+import DockerHostService from "../../../Server/Services/DockerHostService";
+import DockerSwarmClusterService from "../../../Server/Services/DockerSwarmClusterService";
+import HostService from "../../../Server/Services/HostService";
+import PodmanHostService from "../../../Server/Services/PodmanHostService";
+import ProxmoxClusterService from "../../../Server/Services/ProxmoxClusterService";
+import ResourceAiAgentService from "../../../Server/Services/ResourceAiAgentService";
+import VMwareVCenterService from "../../../Server/Services/VMwareVCenterService";
+import DeleteBy from "../../../Server/Types/Database/DeleteBy";
+import { OnDelete } from "../../../Server/Types/Database/Hooks";
+import logger from "../../../Server/Utils/Logger";
+import AutoRemediationSuggestion from "../../../Models/DatabaseModels/AutoRemediationSuggestion";
+import BaseModel from "../../../Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
+import AutoRemediationSuggestionStatus from "../../../Types/AutoRemediation/AutoRemediationSuggestionStatus";
+import DatabaseCommonInteractionProps from "../../../Types/BaseDatabase/DatabaseCommonInteractionProps";
+import ObjectID from "../../../Types/ObjectID";
+import AiResourceType, {
+  ALL_AI_RESOURCE_TYPES,
+} from "../../../Types/ResourceAiAgent/AiResourceType";
+import { getJestSpyOn } from "../../Spy";
+import { afterEach, beforeEach, describe, expect, it } from "@jest/globals";
+
+/*
+ * Deleting any of the eight infrastructure resources a resource AI agent
+ * serves cleans up its OneUptime AI state, through each service's own
+ * delete hooks (the shared ResourceAiDeleteCleanup; its rules are pinned in
+ * Tests/Server/Utils/AI/ResourceAccess/ResourceAiDeleteCleanup.test.ts):
+ *
+ * - onBeforeDelete reads the resource's in-flight rounds of ITS type,
+ *   scoped to the resources the delete matches in the caller's project;
+ * - onDeleteSuccess settles them only for the resources actually deleted,
+ *   with a note naming the resource, then deletes those resources' AI
+ *   agents;
+ * - a failed read, settle or agent delete is logged and never fails the
+ *   delete.
+ *
+ * A table over all eight services proves each one is wired, with its own
+ * resource type.
+ */
+
+const PROJECT_ID: ObjectID = new ObjectID(
+  "11111111-1111-4111-8111-111111111111",
+);
+const RESOURCE_ID: ObjectID = new ObjectID(
+  "22222222-2222-4222-8222-222222222222",
+);
+const OTHER_RESOURCE_ID: ObjectID = new ObjectID(
+  "33333333-3333-4333-8333-333333333333",
+);
+const USER_ID: ObjectID = new ObjectID("44444444-4444-4444-8444-444444444444");
+const ROUND_ID: ObjectID = new ObjectID("55555555-5555-4555-8555-555555555555");
+const OTHER_ROUND_ID: ObjectID = new ObjectID(
+  "66666666-6666-4666-8666-666666666666",
+);
+
+type Hooks = {
+  onBeforeDelete: (
+    deleteBy: DeleteBy<BaseModel>,
+  ) => Promise<OnDelete<BaseModel>>;
+  onDeleteSuccess: (
+    onDelete: OnDelete<BaseModel>,
+    deletedItemIds: Array<ObjectID>,
+  ) => Promise<OnDelete<BaseModel>>;
+};
+
+interface ServiceWiring {
+  resourceType: AiResourceType;
+  service: DatabaseService<BaseModel>;
+  // How the note names the resource.
+  resourceInNote: string;
+}
+
+function asService(service: unknown): DatabaseService<BaseModel> {
+  return service as DatabaseService<BaseModel>;
+}
+
+const WIRING: Array<ServiceWiring> = [
+  {
+    resourceType: AiResourceType.DockerHost,
+    service: asService(DockerHostService),
+    resourceInNote: 'the Docker host "web-1"',
+  },
+  {
+    resourceType: AiResourceType.PodmanHost,
+    service: asService(PodmanHostService),
+    resourceInNote: 'the Podman host "web-1"',
+  },
+  {
+    resourceType: AiResourceType.DockerSwarmCluster,
+    service: asService(DockerSwarmClusterService),
+    resourceInNote: 'the Docker Swarm cluster "web-1"',
+  },
+  {
+    resourceType: AiResourceType.ProxmoxCluster,
+    service: asService(ProxmoxClusterService),
+    resourceInNote: 'the Proxmox cluster "web-1"',
+  },
+  {
+    resourceType: AiResourceType.VMwareVCenter,
+    service: asService(VMwareVCenterService),
+    resourceInNote: 'the VMware vCenter "web-1"',
+  },
+  {
+    resourceType: AiResourceType.CephCluster,
+    service: asService(CephClusterService),
+    resourceInNote: 'the Ceph cluster "web-1"',
+  },
+  {
+    resourceType: AiResourceType.DatabaseServer,
+    service: asService(DatabaseServerService),
+    resourceInNote: 'the database server "web-1"',
+  },
+  {
+    resourceType: AiResourceType.Host,
+    service: asService(HostService),
+    resourceInNote: 'the host "web-1"',
+  },
+];
+
+function hooksOf(service: DatabaseService<BaseModel>): Hooks {
+  return service as unknown as Hooks;
+}
+
+function userProps(): DatabaseCommonInteractionProps {
+  return {
+    userId: USER_ID,
+    tenantId: PROJECT_ID,
+  } as DatabaseCommonInteractionProps;
+}
+
+function deleteBy(): DeleteBy<BaseModel> {
+  return {
+    query: { _id: RESOURCE_ID.toString() },
+    limit: 1,
+    skip: 0,
+    props: userProps(),
+  } as unknown as DeleteBy<BaseModel>;
+}
+
+function round(
+  id: ObjectID,
+  resourceType: AiResourceType,
+  resourceId: ObjectID,
+): AutoRemediationSuggestion {
+  return {
+    id,
+    _id: id.toString(),
+    status: AutoRemediationSuggestionStatus.Planning,
+    resourceType,
+    resourceId,
+  } as unknown as AutoRemediationSuggestion;
+}
+
+interface Transition {
+  suggestionId: ObjectID;
+  fromStatus: AutoRemediationSuggestionStatus;
+  set: Record<string, unknown>;
+}
+
+describe("resource services' delete hooks clean up OneUptime AI state", () => {
+  it("covers every resource type exactly once", () => {
+    expect(
+      WIRING.map((wiring: ServiceWiring): AiResourceType => {
+        return wiring.resourceType;
+      }).sort(),
+    ).toEqual([...ALL_AI_RESOURCE_TYPES].sort());
+  });
+
+  describe.each(WIRING)("$resourceType", (wiring: ServiceWiring) => {
+    let resourceFind: jest.SpyInstance;
+    let suggestionFind: jest.SpyInstance;
+    let transition: jest.SpyInstance;
+    let deleteAgents: jest.SpyInstance;
+    let loggedError: jest.SpyInstance;
+
+    beforeEach(() => {
+      loggedError = jest.spyOn(logger, "error").mockImplementation((): void => {
+        return undefined;
+      });
+      resourceFind = getJestSpyOn(wiring.service, "findBy").mockResolvedValue([
+        { id: RESOURCE_ID, _id: RESOURCE_ID.toString(), name: "web-1" },
+        {
+          id: OTHER_RESOURCE_ID,
+          _id: OTHER_RESOURCE_ID.toString(),
+          name: "web-2",
+        },
+      ]);
+      suggestionFind = jest
+        .spyOn(AutoRemediationSuggestionService, "findBy")
+        .mockResolvedValue([
+          round(ROUND_ID, wiring.resourceType, RESOURCE_ID),
+          round(OTHER_ROUND_ID, wiring.resourceType, OTHER_RESOURCE_ID),
+        ]);
+      transition = jest
+        .spyOn(AutoRemediationSuggestionService, "attemptStatusTransition")
+        .mockResolvedValue(1);
+      deleteAgents = jest
+        .spyOn(ResourceAiAgentService, "deleteAgentsForResources")
+        .mockResolvedValue(1);
+    });
+
+    afterEach(() => {
+      jest.restoreAllMocks();
+    });
+
+    async function deleteResource(
+      deletedItemIds: Array<ObjectID>,
+    ): Promise<OnDelete<BaseModel>> {
+      const hooks: Hooks = hooksOf(wiring.service);
+      const onDelete: OnDelete<BaseModel> =
+        await hooks.onBeforeDelete(deleteBy());
+
+      return await hooks.onDeleteSuccess(onDelete, deletedItemIds);
+    }
+
+    it("overrides both delete hooks on the service itself", () => {
+      const prototype: Record<string, unknown> = Object.getPrototypeOf(
+        wiring.service,
+      ) as Record<string, unknown>;
+
+      expect(
+        Object.prototype.hasOwnProperty.call(prototype, "onBeforeDelete"),
+      ).toBe(true);
+      expect(
+        Object.prototype.hasOwnProperty.call(prototype, "onDeleteSuccess"),
+      ).toBe(true);
+    });
+
+    it("reads the in-flight rounds of its own type before the delete, scoped to the caller's project", async () => {
+      const onDelete: OnDelete<BaseModel> = await hooksOf(
+        wiring.service,
+      ).onBeforeDelete(deleteBy());
+
+      // The delete itself is passed on untouched.
+      expect(onDelete.deleteBy.query).toEqual({ _id: RESOURCE_ID.toString() });
+
+      const resourceQuery: Record<string, unknown> = (
+        resourceFind.mock.calls[0]![0] as { query: Record<string, unknown> }
+      ).query;
+      expect(resourceQuery["_id"]).toBe(RESOURCE_ID.toString());
+      expect(resourceQuery["projectId"]).toBe(PROJECT_ID);
+
+      const suggestionQuery: Record<string, unknown> = (
+        suggestionFind.mock.calls[0]![0] as { query: Record<string, unknown> }
+      ).query;
+      expect(suggestionQuery["resourceType"]).toBe(wiring.resourceType);
+
+      expect(transition).not.toHaveBeenCalled();
+      expect(deleteAgents).not.toHaveBeenCalled();
+    });
+
+    it("settles only the deleted resource's rounds, naming it, then deletes its agents", async () => {
+      const onDelete: OnDelete<BaseModel> = await deleteResource([RESOURCE_ID]);
+
+      const settled: Array<Transition> = transition.mock.calls.map(
+        (call: Array<unknown>): Transition => {
+          return call[0] as Transition;
+        },
+      );
+      expect(
+        settled.map((call: Transition): string => {
+          return call.suggestionId.toString();
+        }),
+      ).toEqual([ROUND_ID.toString()]);
+      expect(settled[0]!.fromStatus).toBe(
+        AutoRemediationSuggestionStatus.Planning,
+      );
+      expect(settled[0]!.set["status"]).toBe(
+        AutoRemediationSuggestionStatus.Dismissed,
+      );
+      expect(settled[0]!.set["dismissedByUserId"]).toBe(USER_ID.toString());
+      expect(settled[0]!.set["rationaleMarkdown"]).toBe(
+        `**Dismissed: ${wiring.resourceInNote} this remediation was for was deleted.** None of its commands ran, and none will.`,
+      );
+
+      expect(deleteAgents).toHaveBeenCalledTimes(1);
+      expect(deleteAgents).toHaveBeenCalledWith({
+        resourceType: wiring.resourceType,
+        resourceIds: [RESOURCE_ID],
+      });
+
+      // The hook hands the delete back as it got it.
+      expect(onDelete.deleteBy.query).toEqual({ _id: RESOURCE_ID.toString() });
+    });
+
+    it("negative control: a delete that removed nothing settles and deletes nothing", async () => {
+      await deleteResource([]);
+
+      expect(transition).not.toHaveBeenCalled();
+      expect(deleteAgents).not.toHaveBeenCalled();
+    });
+
+    it("never fails the delete: failed reads, settles and agent deletes are only logged", async () => {
+      suggestionFind.mockRejectedValue(new Error("db down"));
+      deleteAgents.mockRejectedValue(new Error("db down"));
+
+      await expect(deleteResource([RESOURCE_ID])).resolves.toBeDefined();
+      expect(loggedError).toHaveBeenCalledTimes(2);
+
+      loggedError.mockClear();
+      suggestionFind.mockResolvedValue([
+        round(ROUND_ID, wiring.resourceType, RESOURCE_ID),
+      ]);
+      transition.mockRejectedValue(new Error("db down"));
+      deleteAgents.mockResolvedValue(1);
+
+      await expect(deleteResource([RESOURCE_ID])).resolves.toBeDefined();
+      expect(loggedError).toHaveBeenCalledTimes(1);
+      // The agents still went.
+      expect(deleteAgents).toHaveBeenLastCalledWith({
+        resourceType: wiring.resourceType,
+        resourceIds: [RESOURCE_ID],
+      });
+    });
+  });
+});

package/Tests/Server/Services/RuleRun/LabelRuleEnginesGroupBRun.test.ts

@@ -7,6 +7,9 @@ import IncidentEpisodeLabelRuleEngineService from "../../../../Server/Services/I
 import IncidentEpisodeLabelRuleService from "../../../../Server/Services/IncidentEpisodeLabelRuleService";
 import IncidentEpisodeService from "../../../../Server/Services/IncidentEpisodeService";
 import LabelService from "../../../../Server/Services/LabelService";
+import MessageQueueLabelRuleEngineService from "../../../../Server/Services/MessageQueueLabelRuleEngineService";
+import MessageQueueLabelRuleService from "../../../../Server/Services/MessageQueueLabelRuleService";
+import MessageQueueService from "../../../../Server/Services/MessageQueueService";
 import PodmanHostFeedService from "../../../../Server/Services/PodmanHostFeedService";
 import PodmanHostLabelRuleEngineService from "../../../../Server/Services/PodmanHostLabelRuleEngineService";
 import PodmanHostLabelRuleService from "../../../../Server/Services/PodmanHostLabelRuleService";
@@ -53,6 +56,8 @@ import BaseModel from "../../../../Models/DatabaseModels/DatabaseBaseModel/Datab
 import IncidentEpisode from "../../../../Models/DatabaseModels/IncidentEpisode";
 import IncidentEpisodeLabelRule from "../../../../Models/DatabaseModels/IncidentEpisodeLabelRule";
 import Label from "../../../../Models/DatabaseModels/Label";
+import MessageQueue from "../../../../Models/DatabaseModels/MessageQueue";
+import MessageQueueLabelRule from "../../../../Models/DatabaseModels/MessageQueueLabelRule";
 import PodmanHost from "../../../../Models/DatabaseModels/PodmanHost";
 import PodmanHostLabelRule from "../../../../Models/DatabaseModels/PodmanHostLabelRule";
 import ProxmoxCluster from "../../../../Models/DatabaseModels/ProxmoxCluster";
@@ -182,6 +187,11 @@ interface LabelEngineCase {
   severity: SeverityFields | null;
   expectedRuleSelect: Record<string, unknown>;
   expectedResourceSelect: Record<string, unknown>;
+  /*
+   * What a re-reading engine reads back, when it matches on more than
+   * the name, description and labels (a queue's messaging system).
+   */
+  reReadSelect?: Record<string, unknown> | undefined;
   // Mocks the feed write (and anything it reads) and returns the write spy.
   mockFeed: (() => jest.SpyInstance) | null;
   feedResourceIdKey: string | null;
@@ -537,6 +547,44 @@ const cases: Array<LabelEngineCase> = [
     mockFeed: null,
     feedResourceIdKey: null,
   }),
+  labelEngineCase<MessageQueue, MessageQueueLabelRule>({
+    name: "MessageQueueLabelRuleEngineService",
+    engine: MessageQueueLabelRuleEngineService,
+    applyOnCreate: (resource: MessageQueue): Promise<void> => {
+      return MessageQueueLabelRuleEngineService.applyRulesToMessageQueue(
+        resource,
+      );
+    },
+    ruleService: MessageQueueLabelRuleService as unknown as RuleReadService,
+    resourceService: MessageQueueService as unknown as ResourceService,
+    resourceModel: MessageQueue,
+    textField: "name",
+    patternField: "messageQueueNamePattern",
+    labelMatchField: "messageQueueLabels",
+    matchesHandedResource: false,
+    syncsInMemoryLabels: true,
+    severity: null,
+    expectedRuleSelect: {
+      _id: true,
+      name: true,
+      criteria: true,
+      messageQueueLabels: { _id: true },
+      messageQueueNamePattern: true,
+      messageQueueDescriptionPattern: true,
+      messageQueueSystemPattern: true,
+      labelsToAdd: { _id: true },
+    },
+    expectedResourceSelect: IDS_ONLY_RESOURCE_SELECT,
+    // The system pattern matches on the queue's messaging system too.
+    reReadSelect: {
+      name: true,
+      description: true,
+      messagingSystem: true,
+      labels: { _id: true },
+    },
+    mockFeed: null,
+    feedResourceIdKey: null,
+  }),
   labelEngineCase<AlertEpisode, AlertEpisodeLabelRule>({
     name: "AlertEpisodeLabelRuleEngineService",
     engine: AlertEpisodeLabelRuleEngineService,
@@ -844,7 +892,7 @@ describe("label rule engines (group B) - applying a rule to existing resources",
       cases.map((c: LabelEngineCase) => {
         return c.name;
       }),
-    ).toHaveLength(12);
+    ).toHaveLength(13);
   });
 
   describe.each(cases)("$name", (c: LabelEngineCase) => {
@@ -1168,11 +1216,13 @@ describe("label rule engines (group B) - applying a rule to existing resources",
       expect(writes).toHaveLength(1);
       const reReadArgs: any = reRead.mock.calls[0]![0];
       expect(reReadArgs.id.toString()).toBe(RESOURCE_ID.toString());
-      expect(reReadArgs.select).toEqual({
-        name: true,
-        description: true,
-        labels: { _id: true },
-      });
+      expect(reReadArgs.select).toEqual(
+        c.reReadSelect || {
+          name: true,
+          description: true,
+          labels: { _id: true },
+        },
+      );
     });
 
     /*

package/Tests/Server/Services/RuleRun/OwnerRuleEnginesGroupBRun.test.ts

@@ -10,6 +10,11 @@ import IncidentEpisodeOwnerRuleService from "../../../../Server/Services/Inciden
 import IncidentEpisodeOwnerTeamService from "../../../../Server/Services/IncidentEpisodeOwnerTeamService";
 import IncidentEpisodeOwnerUserService from "../../../../Server/Services/IncidentEpisodeOwnerUserService";
 import IncidentEpisodeService from "../../../../Server/Services/IncidentEpisodeService";
+import MessageQueueOwnerRuleEngineService from "../../../../Server/Services/MessageQueueOwnerRuleEngineService";
+import MessageQueueOwnerRuleService from "../../../../Server/Services/MessageQueueOwnerRuleService";
+import MessageQueueOwnerTeamService from "../../../../Server/Services/MessageQueueOwnerTeamService";
+import MessageQueueOwnerUserService from "../../../../Server/Services/MessageQueueOwnerUserService";
+import MessageQueueService from "../../../../Server/Services/MessageQueueService";
 import PodmanHostFeedService from "../../../../Server/Services/PodmanHostFeedService";
 import PodmanHostOwnerRuleEngineService from "../../../../Server/Services/PodmanHostOwnerRuleEngineService";
 import PodmanHostOwnerRuleService from "../../../../Server/Services/PodmanHostOwnerRuleService";
@@ -72,6 +77,8 @@ import AlertEpisodeOwnerUser from "../../../../Models/DatabaseModels/AlertEpisod
 import BaseModel from "../../../../Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
 import IncidentEpisodeOwnerTeam from "../../../../Models/DatabaseModels/IncidentEpisodeOwnerTeam";
 import IncidentEpisodeOwnerUser from "../../../../Models/DatabaseModels/IncidentEpisodeOwnerUser";
+import MessageQueueOwnerTeam from "../../../../Models/DatabaseModels/MessageQueueOwnerTeam";
+import MessageQueueOwnerUser from "../../../../Models/DatabaseModels/MessageQueueOwnerUser";
 import PodmanHostOwnerTeam from "../../../../Models/DatabaseModels/PodmanHostOwnerTeam";
 import PodmanHostOwnerUser from "../../../../Models/DatabaseModels/PodmanHostOwnerUser";
 import ProxmoxClusterOwnerTeam from "../../../../Models/DatabaseModels/ProxmoxClusterOwnerTeam";
@@ -105,7 +112,7 @@ import { afterEach, beforeEach, describe, expect, it } from "@jest/globals";
 /*
  * Contract under test - "Run now" for the owner rule engines of the Podman,
  * Proxmox, RUM, runbook, serverless, service, status page, vCenter, workflow,
- * alert episode and incident episode families.
+ * queue, alert episode and incident episode families.
  *
  * Each engine applies ONE rule to a resource that already exists. What a run
  * can get wrong that the create hook rarely does: an existing resource almost
@@ -423,6 +430,28 @@ const ENGINE_CASES: Array<EngineCase> = [
     mockFeed: noFeed,
     prepare: noop,
   },
+  {
+    name: "queue",
+    engine: MessageQueueOwnerRuleEngineService,
+    createHook: "applyRulesToMessageQueue",
+    ruleService: MessageQueueOwnerRuleService,
+    rereadService: MessageQueueService,
+    ownerUserService: MessageQueueOwnerUserService,
+    ownerTeamService: MessageQueueOwnerTeamService,
+    ownerUserModel: MessageQueueOwnerUser,
+    ownerTeamModel: MessageQueueOwnerTeam,
+    resourceIdColumn: "messageQueueId",
+    namePatternField: "messageQueueNamePattern",
+    hasNotificationFlag: true,
+    addOwnersService: null,
+    mockFeed: noFeed,
+    prepare: (): void => {
+      // The owners a rule adds are recorded as automatic, in raw SQL.
+      spy(MessageQueueService, "recordAutomaticAssignments").mockResolvedValue(
+        undefined,
+      );
+    },
+  },
   {
     name: "alert episode",
     engine: AlertEpisodeOwnerRuleEngineService,

package/Tests/Server/Services/RunnerJobEnqueueKubectl.test.ts

@@ -27,6 +27,10 @@ import { AI_COMMAND_STEP_TYPES } from "../../../Types/AutoRemediation/AiRemediat
 import BadDataException from "../../../Types/Exception/BadDataException";
 import ObjectID from "../../../Types/ObjectID";
 import PositiveNumber from "../../../Types/PositiveNumber";
+import {
+  fakeRunnerJobCountBy,
+  fakeRunnerJobRows,
+} from "../TestingUtils/Services/FakeRunnerJobCount";
 import { afterEach, beforeEach, describe, expect, it } from "@jest/globals";
 
 /*
@@ -596,6 +600,107 @@ describe("RunnerJobService.enqueueAiKubectlCommand", () => {
     expect(remediationQuery["aiRunId"]).toBeUndefined();
   });
 
+  /*
+   * Resource commands (Docker, Podman, Swarm, Proxmox, VMware, Ceph,
+   * database servers, hosts) land in the same RunnerJob table under the
+   * same two AI origins, and are braked on their own ResourceCommand rows.
+   * A busy hour on that infrastructure must never spend the cluster lane's
+   * budget: counted against an in-memory table, the kubectl brakes count
+   * only the rows they always counted.
+   */
+  describe("the hourly brakes never count resource-command rows", () => {
+    const resourceRow: Record<string, unknown> = {
+      projectId: PROJECT_ID,
+      stepType: RunbookStepType.ResourceCommand,
+      aiRunId: RUN_ID,
+    };
+
+    it("runs an investigation's kubectl after a full hour of resource investigation reads", async () => {
+      countBySpy.mockImplementation(
+        fakeRunnerJobCountBy(
+          fakeRunnerJobRows(
+            MAX_AI_INVESTIGATION_COMMAND_JOBS_PER_PROJECT_PER_HOUR,
+            { ...resourceRow, origin: RunnerJobOrigin.AiInvestigation },
+          ),
+        ) as never,
+      );
+
+      await RunnerJobService.enqueueAiKubectlCommand(args());
+
+      expect(createdRows).toHaveLength(1);
+    });
+
+    it("runs a kubectl fix after a full hour of resource fixes", async () => {
+      countBySpy.mockImplementation(
+        fakeRunnerJobCountBy(
+          fakeRunnerJobRows(MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR, {
+            ...resourceRow,
+            origin: RunnerJobOrigin.AiRemediation,
+            autoRemediationSuggestionId: SUGGESTION_ID,
+          }),
+        ) as never,
+      );
+
+      await RunnerJobService.enqueueAiKubectlCommand(
+        args({
+          origin: RunnerJobOrigin.AiRemediation,
+          autoRemediationSuggestionId: SUGGESTION_ID,
+          command: "kubectl scale deployment/web --replicas=3 -n web",
+        }),
+      );
+
+      expect(createdRows).toHaveLength(1);
+    });
+
+    it("negative control: a full hour of kubectl reads still stops the next one", async () => {
+      countBySpy.mockImplementation(
+        fakeRunnerJobCountBy(
+          fakeRunnerJobRows(
+            MAX_AI_INVESTIGATION_COMMAND_JOBS_PER_PROJECT_PER_HOUR,
+            {
+              ...resourceRow,
+              stepType: RunbookStepType.Kubectl,
+              origin: RunnerJobOrigin.AiInvestigation,
+            },
+          ),
+        ) as never,
+      );
+
+      await expect(
+        RunnerJobService.enqueueAiKubectlCommand(args()),
+      ).rejects.toThrow(/AI investigation commands in the last hour/);
+      expect(createdRows).toHaveLength(0);
+    });
+
+    it("negative control: kubectl fixes still share one budget with Bash and SSH fixes", async () => {
+      countBySpy.mockImplementation(
+        fakeRunnerJobCountBy([
+          ...fakeRunnerJobRows(MAX_AI_COMMAND_JOBS_PER_PROJECT_PER_HOUR - 1, {
+            ...resourceRow,
+            stepType: RunbookStepType.Bash,
+            origin: RunnerJobOrigin.AiRemediation,
+          }),
+          ...fakeRunnerJobRows(1, {
+            ...resourceRow,
+            stepType: RunbookStepType.SSH,
+            origin: RunnerJobOrigin.AiRemediation,
+          }),
+        ]) as never,
+      );
+
+      await expect(
+        RunnerJobService.enqueueAiKubectlCommand(
+          args({
+            origin: RunnerJobOrigin.AiRemediation,
+            autoRemediationSuggestionId: SUGGESTION_ID,
+            command: "kubectl scale deployment/web --replicas=3 -n web",
+          }),
+        ),
+      ).rejects.toThrow(/AI remediation commands in the last hour/);
+      expect(createdRows).toHaveLength(0);
+    });
+  });
+
   describe("the dashboard's access test", () => {
     it("enqueues without an AI run when flagged as an access test", async () => {
       const job: RunnerJob = await RunnerJobService.enqueueAiKubectlCommand(