@oneuptime/common 14.0.8 → 14.0.10

package/build/dist/Server/API/ResourceAiAccessAPI.js

@@ -0,0 +1,1119 @@
+import UserMiddleware from "../Middleware/UserAuthorization";
+import CommonAPI from "./CommonAPI";
+import Express from "../Utils/Express";
+import Response from "../Utils/Response";
+import OneUptimeDate from "../../Types/Date";
+import BadDataException from "../../Types/Exception/BadDataException";
+import NotAuthorizedException from "../../Types/Exception/NotAuthorizedException";
+import ServiceUnavailableException from "../../Types/Exception/ServiceUnavailableException";
+import TooManyRequestsException from "../../Types/Exception/TooManyRequestsException";
+import ObjectID from "../../Types/ObjectID";
+import RunnerJobOrigin from "../../Types/Runbook/RunnerJobOrigin";
+import RunnerJobStatus from "../../Types/Runbook/RunnerJobStatus";
+import RunbookStepType from "../../Types/Runbook/RunbookStepType";
+import AIRunType from "../../Types/AI/AIRunType";
+import SortOrder from "../../Types/BaseDatabase/SortOrder";
+import { RESOURCE_AI_ACCESS_INSIGHTS_PATH, RESOURCE_AI_ACCESS_RESET_AGENT_PATH, RESOURCE_AI_ACCESS_STATUS_PATH, RESOURCE_AI_ACCESS_TEST_PATH, RESOURCE_AI_INSIGHTS_COMMAND_WINDOW_IN_DAYS, RESOURCE_AI_INSIGHTS_LIMIT, RESOURCE_AI_INSIGHTS_RATIONALE_MAX_LENGTH, } from "../../Types/AI/ResourceAiAccessApi";
+import { RESOURCE_AI_ACCESS_ADMIN_PERMISSIONS, getResourceAgentName, getResourceAiAgentResetRefusal, getResourceSentenceName, } from "../../Types/AI/ResourceAiAccessPermissions";
+import AiResourceType, { AI_RESOURCE_TYPE_INFO, parseAiResourceType, } from "../../Types/ResourceAiAgent/AiResourceType";
+import { DEFAULT_RESOURCE_COMMAND_TIMEOUT_MS, } from "../../Types/ResourceAiAgent/ResourceAiAccess";
+import AIRun from "../../Models/DatabaseModels/AIRun";
+import GlobalCache from "../Infrastructure/GlobalCache";
+import Semaphore from "../Infrastructure/Semaphore";
+import AIRunService from "../Services/AIRunService";
+import AlertService from "../Services/AlertService";
+import AutoRemediationSuggestionService from "../Services/AutoRemediationSuggestionService";
+import CephClusterService from "../Services/CephClusterService";
+import DatabaseServerService from "../Services/DatabaseServerService";
+import DockerHostService from "../Services/DockerHostService";
+import DockerSwarmClusterService from "../Services/DockerSwarmClusterService";
+import HostService from "../Services/HostService";
+import IncidentService from "../Services/IncidentService";
+import PodmanHostService from "../Services/PodmanHostService";
+import ProxmoxClusterService from "../Services/ProxmoxClusterService";
+import ResourceAiAccessService from "../Services/ResourceAiAccessService";
+import ResourceAiAgentService from "../Services/ResourceAiAgentService";
+import RunnerJobService from "../Services/RunnerJobService";
+import VMwareVCenterService from "../Services/VMwareVCenterService";
+import QueryHelper from "../Types/Database/QueryHelper";
+import logger from "../Utils/Logger";
+import { holdsAnyUnblockedPermission } from "../Utils/Runbook/RunbookExecutePermission";
+import ResourceCommandJobRunner, { RESOURCE_COMMAND_CLAIM_TIMEOUT_MS, } from "../Utils/AI/ResourceAccess/ResourceCommandJobRunner";
+const router = Express.getRouter();
+export const RESOURCE_AI_ACCESS_KINDS = {
+    [AiResourceType.DockerHost]: {
+        service: DockerHostService,
+        subjectRelation: "dockerHosts",
+    },
+    [AiResourceType.PodmanHost]: {
+        service: PodmanHostService,
+        subjectRelation: "podmanHosts",
+    },
+    [AiResourceType.DockerSwarmCluster]: {
+        service: DockerSwarmClusterService,
+        subjectRelation: "dockerSwarmClusters",
+    },
+    [AiResourceType.ProxmoxCluster]: {
+        service: ProxmoxClusterService,
+        subjectRelation: "proxmoxClusters",
+    },
+    [AiResourceType.VMwareVCenter]: {
+        service: VMwareVCenterService,
+        subjectRelation: "vmwareVCenters",
+    },
+    [AiResourceType.CephCluster]: {
+        service: CephClusterService,
+        subjectRelation: "cephClusters",
+    },
+    [AiResourceType.DatabaseServer]: {
+        service: DatabaseServerService,
+        subjectRelation: "databaseServers",
+    },
+    [AiResourceType.Host]: {
+        service: HostService,
+        subjectRelation: "hosts",
+    },
+};
+async function getLoggedInProps(req) {
+    const props = await CommonAPI.getDatabaseCommonInteractionProps(req);
+    CommonAPI.assertAuthenticatedUser(props);
+    return Object.assign(Object.assign({}, props), { isMultiTenantRequest: false });
+}
+/*
+ * The resource type a request names: an AiResourceType value (any case) or
+ * an agent alias ("docker", "db", ...). Anything else is refused before any
+ * lookup.
+ */
+export function readResourceType(value) {
+    const resourceType = parseAiResourceType(value);
+    if (!resourceType) {
+        throw new BadDataException(`resourceType is required and must be one of ${Object.values(AiResourceType).join(", ")}.`);
+    }
+    return resourceType;
+}
+/*
+ * Access check under the USER's permissions inside the tenant. Null is
+ * "does not exist OR not yours", reported identically so the route never
+ * leaks whether an id exists in another project.
+ */
+async function findAccessibleResource(data) {
+    const body = (data.req.body || {});
+    const resourceType = readResourceType(body["resourceType"]);
+    const resourceIdString = body["resourceId"];
+    if (!resourceIdString ||
+        typeof resourceIdString !== "string" ||
+        !ObjectID.isValidUUID(resourceIdString)) {
+        throw new BadDataException("resourceId is required.");
+    }
+    /*
+     * Read under the user's own props — so the resource's read ACL (and any
+     * label-scoped block) applies exactly as on a CRUD read — and scoped to
+     * the tenant in the query itself.
+     */
+    const resource = (await RESOURCE_AI_ACCESS_KINDS[resourceType].service.findOneBy({
+        query: {
+            _id: resourceIdString,
+            projectId: data.tenantId,
+        },
+        select: { _id: true, projectId: true, name: true },
+        props: data.props,
+    }));
+    const record = resource;
+    const projectId = record ? record["projectId"] : undefined;
+    /*
+     * A row from another project is reported exactly like a missing one
+     * (belt and braces: the query is already tenant-scoped), so the route
+     * never confirms that an id exists somewhere else.
+     */
+    if (!resource ||
+        !resource.id ||
+        !projectId ||
+        String(projectId) !== data.tenantId.toString()) {
+        throw new BadDataException(`${AI_RESOURCE_TYPE_INFO[resourceType].displayName} not found (or you do not have access to it).`);
+    }
+    return {
+        resourceType,
+        id: resource.id,
+        name: typeof record["name"] === "string" && record["name"]
+            ? record["name"]
+            : resource.id.toString(),
+    };
+}
+/*
+ * The access test is read-only, but it spends the agent's time and prints
+ * what the agent can see, so it is for the people who may EDIT this
+ * resource — decided for this row exactly as a CRUD update of it would be
+ * (ResourceAiAccessService.assertCallerMayChangeResource): the update ACL,
+ * label-scoped allow and block rows against the row's own labels, a
+ * table-wide block row, and the caller's Owned scope. Holding an edit
+ * permission somewhere in the project is not enough: an edit grant limited
+ * to "staging" does not reach the "prod" host, and a block row for "prod"
+ * refuses it. Master admins are not gated.
+ */
+async function assertCanEditResource(data) {
+    try {
+        await ResourceAiAccessService.assertCallerMayChangeResource({
+            props: data.props,
+            projectId: data.projectId,
+            resourceType: data.resource.resourceType,
+            resourceId: data.resource.id,
+        });
+    }
+    catch (error) {
+        if (error instanceof NotAuthorizedException) {
+            throw new NotAuthorizedException(`You need permission to edit this ${getResourceSentenceName(data.resource.resourceType)} to run its AI access test.`);
+        }
+        throw error;
+    }
+}
+/*
+ * Resetting the agent locks out whatever holds its key, and the real agent
+ * registers afresh: the same people who may loosen AI access may do it.
+ */
+function assertCanResetAiAgent(data) {
+    if (!holdsAnyUnblockedPermission({
+        props: data.props,
+        projectId: data.projectId,
+        allowed: RESOURCE_AI_ACCESS_ADMIN_PERMISSIONS,
+    })) {
+        throw new NotAuthorizedException(getResourceAiAgentResetRefusal(data.resourceType));
+    }
+}
+async function getStatus(data) {
+    const status = await ResourceAiAccessService.getStatusForResource({
+        projectId: data.projectId,
+        resourceType: data.resource.resourceType,
+        resourceId: data.resource.id,
+    });
+    if (!status) {
+        throw new BadDataException(`${AI_RESOURCE_TYPE_INFO[data.resource.resourceType].displayName} not found.`);
+    }
+    return status;
+}
+/*
+ * The access test's own limits. It enqueues real agent jobs and holds the
+ * request open while they run, so it is bounded where it is triggered and
+ * never counted against the project-wide investigation brake, so no amount
+ * of testing can starve real incident investigations:
+ *
+ * - one test at a time per resource: an atomic reservation (Redis SET NX,
+ *   released when the test ends and expiring on its own after the longest
+ *   a test can run), so concurrent requests cannot all read "nothing
+ *   running" and all start. The per-resource counts below are read inside
+ *   it, so they are exact;
+ * - a few per minute and a cumulative ceiling per hour per resource, and a
+ *   few per minute per user across resources — the per-user count is read
+ *   and the test's first command run under a per-user lock, so a user's
+ *   concurrent requests on different resources are counted one after the
+ *   other. Counted on the RunnerJob rows the test creates, which every API
+ *   node shares.
+ *
+ * The numbers are a Kubernetes cluster's; the step ids, the reservation
+ * and the lock are this route's own, so testing a resource never counts
+ * against a cluster's tests or the other way round.
+ */
+export const MAX_RESOURCE_AI_ACCESS_TESTS_PER_RESOURCE_PER_MINUTE = 3;
+export const MAX_RESOURCE_AI_ACCESS_TESTS_PER_RESOURCE_PER_HOUR = 30;
+export const MAX_RESOURCE_AI_ACCESS_TESTS_PER_USER_PER_MINUTE = 6;
+/*
+ * A test job older than this is past both of its windows (claim plus
+ * execution, twice over), so a row a crashed request left Pending never
+ * blocks the next test for good — and neither does a reservation a crashed
+ * request never released.
+ */
+const RESOURCE_AI_ACCESS_TEST_MAX_DURATION_MINUTES = 4;
+export const RESOURCE_AI_ACCESS_TEST_RESERVATION_NAMESPACE = "resource-ai-access-test";
+export const RESOURCE_AI_ACCESS_TEST_USER_LOCK_NAMESPACE = "resource-ai-access-test-user";
+/*
+ * Held from the per-user count until the test's first command has finished.
+ * The lock refreshes itself while it is held; the timeout is how long it
+ * outlives a request that died holding it.
+ */
+const RESOURCE_AI_ACCESS_TEST_USER_LOCK_TIMEOUT_MS = 30000;
+const RESOURCE_AI_ACCESS_TEST_USER_LOCK_ACQUIRE_TIMEOUT_MS = 15000;
+/*
+ * Gaps that block both capabilities but not the access test's transport:
+ * they are about the project's AI (switched off, no provider, no credits),
+ * and the test runs the agent's commands, never a model. An agent that is
+ * online but could not reach its resource at its last probe is tested too:
+ * the test's own output is exactly what says why, and a success clears it.
+ */
+const NON_TRANSPORT_GAP_CODES = [
+    "ai_disabled_for_project",
+    "llm_provider_missing",
+    "ai_balance_insufficient",
+    "ai_agent_unreachable_resource",
+];
+/*
+ * Every access-test job's stepId starts with this; the first command's
+ * with the "-1-" form plus the user who ran it, so a test (not a command)
+ * can be counted per resource and per user from the rows alone. Distinct
+ * from a Kubernetes cluster's "ai-access-test-" prefix, which the cluster
+ * route counts with startsWith.
+ */
+export const RESOURCE_AI_ACCESS_TEST_STEP_ID_PREFIX = "resource-ai-access-test-";
+export function getResourceAiAccessTestStepId(data) {
+    return `${RESOURCE_AI_ACCESS_TEST_STEP_ID_PREFIX}${data.commandNumber}-${data.userId.toString()}`;
+}
+function getReservationKey(resource) {
+    return `${resource.resourceType}:${resource.id.toString()}`;
+}
+function getAlreadyRunningMessage(resourceType) {
+    return `An AI access test is already running for this ${getResourceSentenceName(resourceType)}. Wait for it to finish, then run it again.`;
+}
+/*
+ * Take this resource's one access-test slot, atomically, and return the
+ * token that releases it — or refuse: 429 while another test holds it, 503
+ * when the reservation cannot be checked at all (failing closed: the
+ * reservation is what makes "one at a time" true under concurrency).
+ */
+async function reserveResourceForAccessTest(resource) {
+    const token = ObjectID.generate().toString();
+    let isReserved = false;
+    try {
+        isReserved = await GlobalCache.setStringIfNotExists(RESOURCE_AI_ACCESS_TEST_RESERVATION_NAMESPACE, getReservationKey(resource), token, { expiresInSeconds: RESOURCE_AI_ACCESS_TEST_MAX_DURATION_MINUTES * 60 });
+    }
+    catch (error) {
+        logger.error(`ResourceAiAccessAPI: could not reserve the AI access test of ${getReservationKey(resource)}: ${error}`);
+        throw new ServiceUnavailableException("The AI access test could not start right now. Try again in a moment.");
+    }
+    if (!isReserved) {
+        throw new TooManyRequestsException(getAlreadyRunningMessage(resource.resourceType));
+    }
+    return token;
+}
+// Best effort: the reservation expires on its own if this fails.
+async function releaseResourceReservation(data) {
+    try {
+        await GlobalCache.deleteKeyIfValue(RESOURCE_AI_ACCESS_TEST_RESERVATION_NAMESPACE, getReservationKey(data.resource), data.token);
+    }
+    catch (error) {
+        logger.error(`ResourceAiAccessAPI: could not release the AI access test reservation of ${getReservationKey(data.resource)}; it expires on its own: ${error}`);
+    }
+}
+/*
+ * The per-resource limits, read while this request holds the resource's
+ * reservation — so no other test of this resource can be counting (or
+ * creating rows) at the same time.
+ */
+async function assertResourceAccessTestMayRun(data) {
+    const resourceQuery = {
+        projectId: data.projectId,
+        resourceType: data.resource.resourceType,
+        resourceId: data.resource.id,
+    };
+    const sentenceName = getResourceSentenceName(data.resource.resourceType);
+    const inFlight = (await RunnerJobService.countBy({
+        query: Object.assign(Object.assign({}, resourceQuery), { stepId: QueryHelper.startsWith(RESOURCE_AI_ACCESS_TEST_STEP_ID_PREFIX), status: QueryHelper.any([
+                RunnerJobStatus.Pending,
+                RunnerJobStatus.Claimed,
+                RunnerJobStatus.Running,
+            ]), createdAt: QueryHelper.greaterThan(OneUptimeDate.getSomeMinutesAgo(RESOURCE_AI_ACCESS_TEST_MAX_DURATION_MINUTES)) }),
+        props: { isRoot: true },
+    })).toNumber();
+    if (inFlight > 0) {
+        throw new TooManyRequestsException(getAlreadyRunningMessage(data.resource.resourceType));
+    }
+    const testsForResource = (await RunnerJobService.countBy({
+        query: Object.assign(Object.assign({}, resourceQuery), { stepId: QueryHelper.startsWith(`${RESOURCE_AI_ACCESS_TEST_STEP_ID_PREFIX}1-`), createdAt: QueryHelper.greaterThan(OneUptimeDate.getSomeMinutesAgo(1)) }),
+        props: { isRoot: true },
+    })).toNumber();
+    if (testsForResource >= MAX_RESOURCE_AI_ACCESS_TESTS_PER_RESOURCE_PER_MINUTE) {
+        throw new TooManyRequestsException(`This ${sentenceName}'s AI access was tested ${testsForResource} times in the last minute, which is its limit (${MAX_RESOURCE_AI_ACCESS_TESTS_PER_RESOURCE_PER_MINUTE}). Try again in a minute.`);
+    }
+    /*
+     * The cumulative ceiling: access-test jobs are kept out of the project's
+     * investigation brake, so without this nothing would bound them over an
+     * hour.
+     */
+    const testsForResourceThisHour = (await RunnerJobService.countBy({
+        query: Object.assign(Object.assign({}, resourceQuery), { stepId: QueryHelper.startsWith(`${RESOURCE_AI_ACCESS_TEST_STEP_ID_PREFIX}1-`), createdAt: QueryHelper.greaterThan(OneUptimeDate.getSomeHoursAgo(1)) }),
+        props: { isRoot: true },
+    })).toNumber();
+    if (testsForResourceThisHour >=
+        MAX_RESOURCE_AI_ACCESS_TESTS_PER_RESOURCE_PER_HOUR) {
+        throw new TooManyRequestsException(`This ${sentenceName}'s AI access was tested ${testsForResourceThisHour} times in the last hour, which is its limit (${MAX_RESOURCE_AI_ACCESS_TESTS_PER_RESOURCE_PER_HOUR}). Try again later.`);
+    }
+}
+/*
+ * The per-user limit spans resources, so the resource reservation does not
+ * serialize it: the count is read and the test's first command run under a
+ * per-user lock, so a user's concurrent tests on different resources cannot
+ * all read the same count. The lock is held until that first command has
+ * finished, because ResourceCommandJobRunner.run enqueues and waits in one
+ * call and the job it creates is what the next count must see.
+ */
+async function lockUserForAccessTestStart(userId) {
+    try {
+        return await Semaphore.lock({
+            key: userId.toString(),
+            namespace: RESOURCE_AI_ACCESS_TEST_USER_LOCK_NAMESPACE,
+            lockTimeout: RESOURCE_AI_ACCESS_TEST_USER_LOCK_TIMEOUT_MS,
+            acquireTimeout: RESOURCE_AI_ACCESS_TEST_USER_LOCK_ACQUIRE_TIMEOUT_MS,
+        });
+    }
+    catch (error) {
+        logger.error(`ResourceAiAccessAPI: could not take the AI access test lock of user ${userId.toString()}: ${error}`);
+        throw new TooManyRequestsException("Another AI access test of yours is starting right now. Try again in a moment.");
+    }
+}
+async function releaseUserLock(mutex) {
+    try {
+        await Semaphore.release(mutex);
+    }
+    catch (error) {
+        logger.error(`ResourceAiAccessAPI: could not release an AI access test user lock; it expires on its own: ${error}`);
+    }
+}
+async function assertUserMayStartAccessTest(data) {
+    const testsByUser = (await RunnerJobService.countBy({
+        query: {
+            projectId: data.projectId,
+            stepId: QueryHelper.startsWith(getResourceAiAccessTestStepId({
+                commandNumber: 1,
+                userId: data.userId,
+            })),
+            createdAt: QueryHelper.greaterThan(OneUptimeDate.getSomeMinutesAgo(1)),
+        },
+        props: { isRoot: true },
+    })).toNumber();
+    if (testsByUser >= MAX_RESOURCE_AI_ACCESS_TESTS_PER_USER_PER_MINUTE) {
+        throw new TooManyRequestsException(`You ran ${testsByUser} AI access tests in the last minute, which is the limit (${MAX_RESOURCE_AI_ACCESS_TESTS_PER_USER_PER_MINUTE}). Try again in a minute.`);
+    }
+}
+/*
+ * One access-test command, run the way every AI resource command is
+ * (ResourceCommandJobRunner.run): enqueued through the resource-command
+ * chokepoint as an access test — the policy, the read-only rule and the
+ * resource's current agent still apply; the investigation switch and the
+ * project's investigation brake do not — then waited on, read and recorded
+ * exactly like an investigation's command (only a success or an ACCESS
+ * failure becomes the resource's "Last verified" / "Last error").
+ *
+ * Origin AiInvestigation with no AI run: an access test has none to keep
+ * alive.
+ */
+async function runAccessTestCommand(data) {
+    return ResourceCommandJobRunner.run({
+        projectId: data.projectId,
+        origin: RunnerJobOrigin.AiInvestigation,
+        resourceType: data.resource.resourceType,
+        resourceId: data.resource.id,
+        targetResourceAiAgentId: data.agentId,
+        command: data.command,
+        stepId: data.stepId,
+        timeoutInMs: DEFAULT_RESOURCE_COMMAND_TIMEOUT_MS,
+        claimTimeoutInMs: RESOURCE_COMMAND_CLAIM_TIMEOUT_MS,
+        isAccessTest: true,
+    });
+}
+/*
+ * Start the test: under this user's lock, check the per-user limit and run
+ * the first command, so the next concurrent test of this user counts its
+ * job (see lockUserForAccessTestStart). The limit refusal is thrown (the
+ * route answers 429); a refusal from the chokepoint, or a wait that broke,
+ * is returned as the first command's failure.
+ */
+async function startAccessTest(data) {
+    const userLock = await lockUserForAccessTestStart(data.userId);
+    try {
+        await assertUserMayStartAccessTest({
+            projectId: data.projectId,
+            userId: data.userId,
+        });
+        try {
+            const outcome = await runAccessTestCommand({
+                projectId: data.projectId,
+                resource: data.resource,
+                agentId: data.agentId,
+                command: data.command,
+                stepId: getResourceAiAccessTestStepId({
+                    commandNumber: 1,
+                    userId: data.userId,
+                }),
+            });
+            return { outcome };
+        }
+        catch (error) {
+            return { error };
+        }
+    }
+    finally {
+        await releaseUserLock(userLock);
+    }
+}
+function describeFailure(error) {
+    if (error === undefined || error === null) {
+        return "The command could not be started.";
+    }
+    return error instanceof Error ? error.message : String(error);
+}
+/*
+ * The gap that stops the access test before anything is enqueued: one that
+ * blocks both capabilities and is about the transport — no agent, or an
+ * offline one — never one about the project's AI.
+ */
+export function getAccessTestTransportGap(status) {
+    return status.gaps.find((gap) => {
+        return (gap.blocksInvestigation &&
+            gap.blocksRemediation &&
+            !NON_TRANSPORT_GAP_CODES.includes(gap.code));
+    });
+}
+router.post(RESOURCE_AI_ACCESS_STATUS_PATH, UserMiddleware.getUserMiddleware, async (req, res, next) => {
+    try {
+        const props = await getLoggedInProps(req);
+        const tenantId = CommonAPI.assertTenantScoped(props);
+        const resource = await findAccessibleResource({
+            req,
+            props,
+            tenantId,
+        });
+        const status = await getStatus({
+            projectId: tenantId,
+            resource,
+        });
+        Response.sendJsonObjectResponse(req, res, status);
+        return;
+    }
+    catch (err) {
+        next(err);
+        return;
+    }
+});
+router.post(RESOURCE_AI_ACCESS_TEST_PATH, UserMiddleware.getUserMiddleware, async (req, res, next) => {
+    var _a, _b;
+    try {
+        const props = await getLoggedInProps(req);
+        const tenantId = CommonAPI.assertTenantScoped(props);
+        const resource = await findAccessibleResource({
+            req,
+            props,
+            tenantId,
+        });
+        await assertCanEditResource({
+            props,
+            projectId: tenantId,
+            resource,
+        });
+        const agentName = getResourceAgentName(resource.resourceType);
+        // One test at a time per resource: an atomic reservation, held to the end.
+        const reservation = await reserveResourceForAccessTest(resource);
+        try {
+            await assertResourceAccessTestMayRun({
+                projectId: tenantId,
+                resource,
+            });
+            const status = await getStatus({
+                projectId: tenantId,
+                resource,
+            });
+            /*
+             * The test needs a reachable agent, not the investigation switch:
+             * an operator checks access BEFORE turning AI on. Only gaps that
+             * block the transport itself stop it — never one about the
+             * project's AI (the test runs the agent's commands, not a model).
+             */
+            const transportGap = getAccessTestTransportGap(status);
+            if (transportGap || !status.agent || !status.agent.isOnline) {
+                Response.sendJsonObjectResponse(req, res, {
+                    ok: false,
+                    message: transportGap
+                        ? `${transportGap.title}. ${transportGap.nextStep}`
+                        : status.agent
+                            ? `The ${agentName} is offline. Check that its container is running and can reach your OneUptime URL.`
+                            : `Nothing can reach this ${getResourceSentenceName(resource.resourceType)} yet: install the ${agentName}.`,
+                    results: [],
+                    status: status,
+                });
+                return;
+            }
+            const agentId = new ObjectID(status.agent.agentId);
+            const commands = AI_RESOURCE_TYPE_INFO[resource.resourceType].testCommands;
+            const results = [];
+            let allSucceeded = true;
+            const start = await startAccessTest({
+                projectId: tenantId,
+                resource,
+                userId: props.userId,
+                agentId,
+                command: commands[0],
+            });
+            for (let index = 0; index < commands.length; index++) {
+                const command = commands[index];
+                let failure = undefined;
+                let outcome = undefined;
+                if (index === 0) {
+                    outcome = start.outcome;
+                    failure = start.error;
+                }
+                else {
+                    try {
+                        outcome = await runAccessTestCommand({
+                            projectId: tenantId,
+                            resource,
+                            agentId,
+                            command,
+                            stepId: getResourceAiAccessTestStepId({
+                                commandNumber: index + 1,
+                                userId: props.userId,
+                            }),
+                        });
+                    }
+                    catch (error) {
+                        failure = error;
+                    }
+                }
+                if (!outcome) {
+                    allSucceeded = false;
+                    results.push({
+                        command,
+                        succeeded: false,
+                        exitCode: null,
+                        output: "",
+                        errorMessage: describeFailure(failure),
+                    });
+                    break;
+                }
+                allSucceeded = allSucceeded && outcome.succeeded;
+                results.push({
+                    command: outcome.displayCommand || command,
+                    succeeded: outcome.succeeded,
+                    exitCode: (_a = outcome.exitCode) !== null && _a !== void 0 ? _a : null,
+                    output: outcome.output,
+                    errorMessage: (_b = outcome.errorMessage) !== null && _b !== void 0 ? _b : null,
+                });
+                if (!outcome.succeeded) {
+                    break;
+                }
+            }
+            const refreshed = await ResourceAiAccessService.getStatusForResource({
+                projectId: tenantId,
+                resourceType: resource.resourceType,
+                resourceId: resource.id,
+            });
+            Response.sendJsonObjectResponse(req, res, {
+                ok: allSucceeded,
+                message: allSucceeded
+                    ? `OneUptime AI can run commands on "${resource.name}" through the ${agentName}.`
+                    : `The ${agentName} could not run the test commands successfully — see the command output below.`,
+                results: results,
+                status: (refreshed || status),
+            });
+            return;
+        }
+        finally {
+            await releaseResourceReservation({
+                resource,
+                token: reservation,
+            });
+        }
+    }
+    catch (err) {
+        next(err);
+        return;
+    }
+});
+router.post(RESOURCE_AI_ACCESS_RESET_AGENT_PATH, UserMiddleware.getUserMiddleware, async (req, res, next) => {
+    try {
+        const props = await getLoggedInProps(req);
+        const tenantId = CommonAPI.assertTenantScoped(props);
+        const resource = await findAccessibleResource({
+            req,
+            props,
+            tenantId,
+        });
+        assertCanResetAiAgent({
+            props,
+            projectId: tenantId,
+            resourceType: resource.resourceType,
+        });
+        const agentName = getResourceAgentName(resource.resourceType);
+        const agent = await ResourceAiAgentService.findAgentForResource({
+            projectId: tenantId,
+            resourceType: resource.resourceType,
+            resourceId: resource.id,
+        });
+        if (!agent) {
+            throw new BadDataException(`This ${getResourceSentenceName(resource.resourceType)} has no ${agentName} to reset.`);
+        }
+        await ResourceAiAgentService.resetAgent({
+            projectId: tenantId,
+            resourceType: resource.resourceType,
+            resourceId: resource.id,
+            userId: props.userId,
+        });
+        const status = await ResourceAiAccessService.getStatusForResource({
+            projectId: tenantId,
+            resourceType: resource.resourceType,
+            resourceId: resource.id,
+        });
+        Response.sendJsonObjectResponse(req, res, Object.assign({ ok: true, message: `The ${agentName} was reset. It reconnects on its own within a few minutes.` }, (status ? { status: status } : {})));
+        return;
+    }
+    catch (err) {
+        next(err);
+        return;
+    }
+});
+/*
+ * How many of the resource's newest agent jobs the insights read to find
+ * the AI runs and suggestions that ran commands on it. Each AI run runs at
+ * most a handful, so this reaches well past the newest 25 of either.
+ */
+const INSIGHTS_JOB_SCAN_LIMIT = 500;
+// How many incidents and alerts linked to the resource the insights read.
+const INSIGHTS_LINKED_SUBJECT_LIMIT = 200;
+function toIsoString(date) {
+    return date ? OneUptimeDate.toString(date) : undefined;
+}
+function newestFirst(rows) {
+    return rows.sort((a, b) => {
+        return ((b.createdAt ? new Date(b.createdAt).getTime() : 0) -
+            (a.createdAt ? new Date(a.createdAt).getTime() : 0));
+    });
+}
+/*
+ * The union of several newest-first reads: one row per id, newest first,
+ * at most `limit` of them (all of them without one).
+ */
+function mergeNewest(groups, limit) {
+    var _a;
+    const byId = new Map();
+    for (const rows of groups) {
+        for (const row of rows) {
+            const id = (_a = row.id) === null || _a === void 0 ? void 0 : _a.toString();
+            if (id && !byId.has(id)) {
+                byId.set(id, row);
+            }
+        }
+    }
+    return newestFirst(Array.from(byId.values())).slice(0, limit);
+}
+function getIds(rows) {
+    return rows
+        .map((row) => {
+        return row.id || undefined;
+    })
+        .filter((id) => {
+        return Boolean(id);
+    });
+}
+function getUniqueIds(ids) {
+    const byId = new Map();
+    for (const id of ids) {
+        if (id) {
+            byId.set(id.toString(), id);
+        }
+    }
+    return Array.from(byId.values());
+}
+/*
+ * A read made under the caller's own props, which the permission layer
+ * refuses outright for a role that cannot read the table at all (a
+ * ReadDockerHost-only role reading Incident): that caller gets nothing from
+ * it, the same as a caller whose labels or private-incident membership
+ * leave no row readable.
+ */
+async function readIfPermitted(read) {
+    try {
+        return await read();
+    }
+    catch (error) {
+        if (error instanceof NotAuthorizedException) {
+            return [];
+        }
+        throw error;
+    }
+}
+/*
+ * What OneUptime AI investigated and changed on one resource, as summaries
+ * (ResourceAiInsights). The caller has already been checked for read
+ * access to the resource (findAccessibleResource), which is a wider
+ * audience than the incidents, alerts, AI runs and suggestions summarised
+ * here, so what is said about THOSE follows the caller's own read access to
+ * them:
+ *
+ * - incidents and alerts are read under the caller's props (tenant, labels,
+ *   private incidents), both to find the ones linked to the resource and to
+ *   name a run's subject; an investigation whose incident or alert the
+ *   caller cannot read is left out altogether;
+ * - an investigation's TL;DR is shown with its readable incident or alert —
+ *   the incident's and alert's own AI investigation panel shows the whole
+ *   analysis to anyone who may read the subject — and, for a run with
+ *   neither (an AI insight's triage), only to a caller who may read AIRun;
+ * - a fix's rationale is read under the caller's props, so only a caller who
+ *   may read that AutoRemediationSuggestion (its ACL and its incident's or
+ *   alert's privacy) gets it.
+ *
+ * The rest — which AI runs and suggestions touched the resource, their
+ * status and dates, the command counts — is about the resource itself and
+ * is read as root: AI runs, suggestions and agent jobs have narrower read
+ * ACLs of their own (an investigation run is private to its author), so
+ * reading them under the caller's props would empty the page for exactly
+ * the people it is for. Nothing here returns command output, prompts or
+ * command plans.
+ *
+ * - investigations: AI investigations that ran a command on this resource
+ *   (their RunnerJob rows name it), UNION investigations of incidents and
+ *   alerts linked to it — which also covers one whose access was not set
+ *   up, so it never ran a command. Newest first, at most
+ *   RESOURCE_AI_INSIGHTS_LIMIT.
+ * - fixes: suggestions the resource's own AI remediation setting produced
+ *   (AutoRemediationSuggestion.resourceType / resourceId), UNION any
+ *   suggestion whose commands ran on this resource (a rule's round, too).
+ *   Newest first, same limit.
+ * - commandCounts: commands in the last
+ *   RESOURCE_AI_INSIGHTS_COMMAND_WINDOW_IN_DAYS days. The AI agent page's
+ *   connection tests have no AI run behind them and are not counted.
+ */
+export async function getResourceAiInsights(data) {
+    const { projectId, resourceType, resourceId, props } = data;
+    const limit = RESOURCE_AI_INSIGHTS_LIMIT;
+    const subjectRelation = RESOURCE_AI_ACCESS_KINDS[resourceType].subjectRelation;
+    const jobs = await RunnerJobService.findBy({
+        query: {
+            projectId,
+            resourceType,
+            resourceId,
+            stepType: RunbookStepType.ResourceCommand,
+        },
+        select: { _id: true, aiRunId: true, autoRemediationSuggestionId: true },
+        sort: { createdAt: SortOrder.Descending },
+        limit: INSIGHTS_JOB_SCAN_LIMIT,
+        skip: 0,
+        props: { isRoot: true },
+    });
+    const runIdsFromJobs = new Map();
+    const suggestionIdsFromJobs = new Map();
+    for (const job of jobs) {
+        if (job.aiRunId) {
+            runIdsFromJobs.set(job.aiRunId.toString(), job.aiRunId);
+        }
+        if (job.autoRemediationSuggestionId) {
+            suggestionIdsFromJobs.set(job.autoRemediationSuggestionId.toString(), job.autoRemediationSuggestionId);
+        }
+    }
+    // Only the incidents and alerts the caller may read (see above).
+    const [linkedIncidents, linkedAlerts] = await Promise.all([
+        readIfPermitted(() => {
+            return IncidentService.findBy({
+                query: {
+                    projectId,
+                    [subjectRelation]: QueryHelper.inRelationArray([resourceId]),
+                },
+                select: { _id: true },
+                sort: { createdAt: SortOrder.Descending },
+                limit: INSIGHTS_LINKED_SUBJECT_LIMIT,
+                skip: 0,
+                props,
+            });
+        }),
+        readIfPermitted(() => {
+            return AlertService.findBy({
+                query: {
+                    projectId,
+                    [subjectRelation]: QueryHelper.inRelationArray([resourceId]),
+                },
+                select: { _id: true },
+                sort: { createdAt: SortOrder.Descending },
+                limit: INSIGHTS_LINKED_SUBJECT_LIMIT,
+                skip: 0,
+                props,
+            });
+        }),
+    ]);
+    const investigations = await getInsightInvestigations({
+        projectId,
+        props,
+        runIds: Array.from(runIdsFromJobs.values()),
+        incidentIds: getIds(linkedIncidents),
+        alertIds: getIds(linkedAlerts),
+        limit,
+    });
+    const fixes = await getInsightFixes({
+        projectId,
+        props,
+        resourceType,
+        resourceId,
+        suggestionIds: Array.from(suggestionIdsFromJobs.values()),
+        limit,
+    });
+    const since = OneUptimeDate.getSomeDaysAgo(RESOURCE_AI_INSIGHTS_COMMAND_WINDOW_IN_DAYS);
+    const countCommands = async (query) => {
+        return (await RunnerJobService.countBy({
+            query: Object.assign(Object.assign({}, query), { projectId,
+                resourceType,
+                resourceId, stepType: RunbookStepType.ResourceCommand, createdAt: QueryHelper.greaterThan(since) }),
+            props: { isRoot: true },
+        })).toNumber();
+    };
+    const [investigationCommands, remediationCommands] = await Promise.all([
+        countCommands({
+            origin: RunnerJobOrigin.AiInvestigation,
+            // Connection tests have no AI run; every investigation command does.
+            aiRunId: QueryHelper.notNull(),
+        }),
+        countCommands({ origin: RunnerJobOrigin.AiRemediation }),
+    ]);
+    return {
+        resourceType,
+        resourceId: resourceId.toString(),
+        investigations,
+        fixes,
+        commandCounts: {
+            investigation: investigationCommands,
+            remediation: remediationCommands,
+        },
+    };
+}
+const INSIGHT_RUN_SELECT = {
+    _id: true,
+    status: true,
+    analysisTldr: true,
+    createdAt: true,
+    completedAt: true,
+    triggeredByIncidentId: true,
+    triggeredByAlertId: true,
+};
+/*
+ * Who may read the TL;DR of a run that has no incident or alert to decide
+ * it: the people who may read that column of AIRun.
+ */
+function getSubjectlessRunTldrReadPermissions() {
+    var _a;
+    return ((_a = new AIRun().getColumnAccessControlFor("analysisTldr")) === null || _a === void 0 ? void 0 : _a.read) || [];
+}
+async function getInsightInvestigations(data) {
+    const reads = [];
+    const readRuns = (query) => {
+        return AIRunService.findBy({
+            query: Object.assign(Object.assign({}, query), { projectId: data.projectId, runType: AIRunType.Investigation }),
+            select: INSIGHT_RUN_SELECT,
+            sort: { createdAt: SortOrder.Descending },
+            limit: data.limit,
+            skip: 0,
+            props: { isRoot: true },
+        });
+    };
+    if (data.runIds.length > 0) {
+        reads.push(readRuns({ _id: QueryHelper.any(data.runIds) }));
+    }
+    if (data.incidentIds.length > 0) {
+        reads.push(readRuns({ triggeredByIncidentId: QueryHelper.any(data.incidentIds) }));
+    }
+    if (data.alertIds.length > 0) {
+        reads.push(readRuns({ triggeredByAlertId: QueryHelper.any(data.alertIds) }));
+    }
+    if (reads.length === 0) {
+        return [];
+    }
+    // Every candidate, newest first: the limit applies after the subject check.
+    const candidates = mergeNewest(await Promise.all(reads));
+    const incidentIds = getUniqueIds(candidates.map((run) => {
+        return run.triggeredByIncidentId;
+    }));
+    const alertIds = getUniqueIds(candidates.map((run) => {
+        return run.triggeredByAlertId;
+    }));
+    /*
+     * The subjects under the CALLER's props: tenant, labels and private
+     * incidents apply exactly as on a CRUD read, and a role that may not read
+     * incidents (or alerts) at all reads none.
+     */
+    const [incidents, alerts] = await Promise.all([
+        incidentIds.length > 0
+            ? readIfPermitted(() => {
+                return IncidentService.findBy({
+                    query: {
+                        projectId: data.projectId,
+                        _id: QueryHelper.any(incidentIds),
+                    },
+                    select: { _id: true, title: true, incidentNumber: true },
+                    limit: incidentIds.length,
+                    skip: 0,
+                    props: data.props,
+                });
+            })
+            : Promise.resolve([]),
+        alertIds.length > 0
+            ? readIfPermitted(() => {
+                return AlertService.findBy({
+                    query: {
+                        projectId: data.projectId,
+                        _id: QueryHelper.any(alertIds),
+                    },
+                    select: { _id: true, title: true },
+                    limit: alertIds.length,
+                    skip: 0,
+                    props: data.props,
+                });
+            })
+            : Promise.resolve([]),
+    ]);
+    const incidentsById = new Map(incidents.map((incident) => {
+        var _a;
+        return [((_a = incident.id) === null || _a === void 0 ? void 0 : _a.toString()) || "", incident];
+    }));
+    const alertsById = new Map(alerts.map((alert) => {
+        var _a;
+        return [((_a = alert.id) === null || _a === void 0 ? void 0 : _a.toString()) || "", alert];
+    }));
+    // A run whose incident or alert the caller cannot read is left out.
+    const runs = candidates
+        .filter((run) => {
+        if (run.triggeredByIncidentId &&
+            !incidentsById.has(run.triggeredByIncidentId.toString())) {
+            return false;
+        }
+        if (run.triggeredByAlertId &&
+            !alertsById.has(run.triggeredByAlertId.toString())) {
+            return false;
+        }
+        return true;
+    })
+        .slice(0, data.limit);
+    const mayReadSubjectlessTldr = holdsAnyUnblockedPermission({
+        props: data.props,
+        projectId: data.projectId,
+        allowed: getSubjectlessRunTldrReadPermissions(),
+    });
+    return runs.map((run) => {
+        const incident = run.triggeredByIncidentId
+            ? incidentsById.get(run.triggeredByIncidentId.toString())
+            : undefined;
+        const alert = run.triggeredByAlertId
+            ? alertsById.get(run.triggeredByAlertId.toString())
+            : undefined;
+        /*
+         * Every run left here with a subject has a subject the caller may read,
+         * and the subject's own AI panel shows its analysis to them.
+         */
+        const mayReadTldr = Boolean(run.triggeredByIncidentId || run.triggeredByAlertId) ||
+            mayReadSubjectlessTldr;
+        return {
+            aiRunId: run.id.toString(),
+            status: run.status,
+            analysisTldr: mayReadTldr ? run.analysisTldr || undefined : undefined,
+            createdAt: toIsoString(run.createdAt),
+            completedAt: toIsoString(run.completedAt),
+            incident: run.triggeredByIncidentId
+                ? {
+                    id: run.triggeredByIncidentId.toString(),
+                    title: incident === null || incident === void 0 ? void 0 : incident.title,
+                    number: incident === null || incident === void 0 ? void 0 : incident.incidentNumber,
+                }
+                : undefined,
+            alert: run.triggeredByAlertId
+                ? {
+                    id: run.triggeredByAlertId.toString(),
+                    title: alert === null || alert === void 0 ? void 0 : alert.title,
+                }
+                : undefined,
+        };
+    });
+}
+/*
+ * Read as root, so never the rationale: that is read under the caller's
+ * props (getInsightFixes).
+ */
+const INSIGHT_FIX_SELECT = {
+    _id: true,
+    status: true,
+    executionMode: true,
+    suggestionType: true,
+    createdAt: true,
+    approvedAt: true,
+    incidentId: true,
+    alertId: true,
+};
+async function getInsightFixes(data) {
+    const readSuggestions = (query) => {
+        return AutoRemediationSuggestionService.findBy({
+            query: Object.assign(Object.assign({}, query), { projectId: data.projectId }),
+            select: INSIGHT_FIX_SELECT,
+            sort: { createdAt: SortOrder.Descending },
+            limit: data.limit,
+            skip: 0,
+            props: { isRoot: true },
+        });
+    };
+    const reads = [
+        readSuggestions({
+            resourceType: data.resourceType,
+            resourceId: data.resourceId,
+        }),
+    ];
+    if (data.suggestionIds.length > 0) {
+        reads.push(readSuggestions({ _id: QueryHelper.any(data.suggestionIds) }));
+    }
+    const suggestions = mergeNewest(await Promise.all(reads), data.limit);
+    const ids = getIds(suggestions);
+    /*
+     * The rationale quotes the AI's analysis of the incident or alert, so it
+     * is read under the CALLER's props: the suggestion's read ACL and its
+     * subject's privacy decide it, as on a CRUD read.
+     */
+    const readable = ids.length > 0
+        ? await readIfPermitted(() => {
+            return AutoRemediationSuggestionService.findBy({
+                query: {
+                    projectId: data.projectId,
+                    _id: QueryHelper.any(ids),
+                },
+                select: { _id: true, rationaleMarkdown: true },
+                limit: ids.length,
+                skip: 0,
+                props: data.props,
+            });
+        })
+        : [];
+    const rationaleById = new Map();
+    for (const suggestion of readable) {
+        if (suggestion.id && suggestion.rationaleMarkdown) {
+            rationaleById.set(suggestion.id.toString(), suggestion.rationaleMarkdown);
+        }
+    }
+    return suggestions.map((suggestion) => {
+        var _a, _b;
+        const rationale = rationaleById.get(suggestion.id.toString());
+        return {
+            id: suggestion.id.toString(),
+            status: suggestion.status,
+            executionMode: suggestion.executionMode,
+            suggestionType: suggestion.suggestionType,
+            rationale: rationale
+                ? rationale.slice(0, RESOURCE_AI_INSIGHTS_RATIONALE_MAX_LENGTH)
+                : undefined,
+            createdAt: toIsoString(suggestion.createdAt),
+            approvedAt: toIsoString(suggestion.approvedAt),
+            incidentId: (_a = suggestion.incidentId) === null || _a === void 0 ? void 0 : _a.toString(),
+            alertId: (_b = suggestion.alertId) === null || _b === void 0 ? void 0 : _b.toString(),
+        };
+    });
+}
+router.post(RESOURCE_AI_ACCESS_INSIGHTS_PATH, UserMiddleware.getUserMiddleware, async (req, res, next) => {
+    try {
+        const props = await getLoggedInProps(req);
+        const tenantId = CommonAPI.assertTenantScoped(props);
+        const resource = await findAccessibleResource({
+            req,
+            props,
+            tenantId,
+        });
+        const insights = await getResourceAiInsights({
+            projectId: tenantId,
+            resourceType: resource.resourceType,
+            resourceId: resource.id,
+            props,
+        });
+        Response.sendJsonObjectResponse(req, res, insights);
+        return;
+    }
+    catch (err) {
+        next(err);
+        return;
+    }
+});
+export default router;
+//# sourceMappingURL=ResourceAiAccessAPI.js.map