@oneuptime/common 14.0.8 → 14.0.9

package/Server/Infrastructure/Postgres/SchemaMigrations/1796100000000-AddKubernetesAiAgentAndAiDefaults.ts

@@ -0,0 +1,137 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * The Kubernetes AI agent, and AI on by default.
+ *
+ * Schema (generated):
+ *   - KubernetesAiAgent: one row per cluster for the slim in-cluster kubectl
+ *     executor the kubernetes-agent chart now installs (its own identity,
+ *     never a Runner). Unique per cluster; cascades with its project and
+ *     cluster.
+ *   - RunnerJob.targetKubernetesAiAgentId: the agent an AI kubectl job is
+ *     targeted at (SET NULL when the agent row goes).
+ *   - Project.enableAutomaticPostmortemDraft: drafting a postmortem on
+ *     resolve, split off enableAutomaticIncidentInvestigation.
+ *   - KubernetesCluster.isAiInvestigationEnabled now defaults to true.
+ *
+ * Data (hand-added after the DDL, in this order):
+ *   1. Postmortem drafts keep working exactly where they worked: the new
+ *      flag is copied from the investigation flag it used to ride on. The
+ *      column was just added as NOT NULL DEFAULT false, so setting it to true
+ *      where investigation is on is the whole copy, and it rewrites only
+ *      those rows.
+ *   2. Clusters nobody ever configured AI access for (aiAccessConfiguredAt
+ *      IS NULL) get investigation on, like a new cluster. A cluster an
+ *      operator configured keeps its switch, whatever it is.
+ *   3. Revocations carry over. Before the AI agent, unbinding or deleting a
+ *      cluster's Runner was the off switch: the cluster has no Runner now
+ *      but one was bound once, or AI already ran (or failed to run) kubectl
+ *      on it. The agent must not quietly switch AI back on there, so
+ *      investigation goes off and remediation goes to Disabled. Runs after
+ *      2, so a revocation always wins.
+ *   4. No unattended surprise. Cluster rounds through the in-cluster agent
+ *      no longer need the project's "Enable AI command execution" opt-in, so
+ *      a cluster left in Automatic or Bypass approval in a project that never
+ *      opted in would start changing the cluster without a human on upgrade.
+ *      Those clusters drop to RequireApproval: fixes are proposed, a human
+ *      approves each.
+ *
+ * down() reverts the defaults and the schema only. The data updates are not
+ * reverted: afterwards those rows are indistinguishable from choices an
+ * operator made, and flipping them back could switch AI on or off for
+ * somebody who chose it.
+ */
+export class AddKubernetesAiAgentAndAiDefaults1796100000000
+  implements MigrationInterface
+{
+  public name: string = "AddKubernetesAiAgentAndAiDefaults1796100000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `CREATE TABLE "KubernetesAiAgent" ("_id" uuid NOT NULL DEFAULT uuid_generate_v4(), "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "deletedAt" TIMESTAMP WITH TIME ZONE, "version" integer NOT NULL, "projectId" uuid NOT NULL, "kubernetesClusterId" uuid NOT NULL, "keyHash" character varying(100), "agentVersion" character varying(100), "posture" jsonb, "lastAliveAt" TIMESTAMP WITH TIME ZONE, "connectionStatus" character varying(100) NOT NULL DEFAULT 'disconnected', "lastRegisteredAt" TIMESTAMP WITH TIME ZONE, "registeredWithIngestionKeyId" uuid, "lastRefusedRegistrationAt" TIMESTAMP WITH TIME ZONE, "lastRefusedRegistrationReason" character varying(100), CONSTRAINT "PK_7d9287fc069699b2f22ad4f88ee" PRIMARY KEY ("_id"))`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_33622e9406d3bffee1ca9c3a6a" ON "KubernetesAiAgent" ("projectId") `,
+    );
+    await queryRunner.query(
+      `CREATE UNIQUE INDEX "IDX_KubernetesAiAgent_kubernetesClusterId" ON "KubernetesAiAgent" ("kubernetesClusterId") WHERE "deletedAt" IS NULL`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" ADD "enableAutomaticPostmortemDraft" boolean NOT NULL DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "RunnerJob" ADD "targetKubernetesAiAgentId" uuid`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesCluster" ALTER COLUMN "isAiInvestigationEnabled" SET DEFAULT true`,
+    );
+    await queryRunner.query(
+      `CREATE INDEX "IDX_df20ccc2683c9be5fbc8399e09" ON "RunnerJob" ("targetKubernetesAiAgentId") `,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesAiAgent" ADD CONSTRAINT "FK_33622e9406d3bffee1ca9c3a6a9" FOREIGN KEY ("projectId") REFERENCES "Project"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesAiAgent" ADD CONSTRAINT "FK_d6ed70c680c438a5ea3080071ca" FOREIGN KEY ("kubernetesClusterId") REFERENCES "KubernetesCluster"("_id") ON DELETE CASCADE ON UPDATE NO ACTION`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "RunnerJob" ADD CONSTRAINT "FK_df20ccc2683c9be5fbc8399e098" FOREIGN KEY ("targetKubernetesAiAgentId") REFERENCES "KubernetesAiAgent"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+
+    // 1. Postmortem drafts stay on wherever investigations were on.
+    await queryRunner.query(
+      `UPDATE "Project" SET "enableAutomaticPostmortemDraft" = true WHERE "enableAutomaticIncidentInvestigation" = true`,
+    );
+
+    // 2. Never-configured clusters investigate, like a new cluster.
+    await queryRunner.query(
+      `UPDATE "KubernetesCluster" SET "isAiInvestigationEnabled" = true WHERE "aiAccessConfiguredAt" IS NULL`,
+    );
+
+    /*
+     * 3. A cluster whose Runner was unbound or deleted stays off. It is also
+     * marked configured, so the Kubernetes AI agent's first-connection
+     * defaults (which only touch never-configured clusters) can never switch
+     * a revoked cluster back on.
+     */
+    await queryRunner.query(
+      `UPDATE "KubernetesCluster" SET "isAiInvestigationEnabled" = false, "aiRemediationMode" = 'Disabled', "aiAccessConfiguredAt" = COALESCE("aiAccessConfiguredAt", NOW()) WHERE "aiAccessRunnerId" IS NULL AND ("aiAccessRunnerBoundAt" IS NOT NULL OR "aiAccessLastVerifiedAt" IS NOT NULL OR "aiAccessLastError" IS NOT NULL)`,
+    );
+
+    // 4. No unattended mode goes live without the project's opt-in.
+    await queryRunner.query(
+      `UPDATE "KubernetesCluster" c SET "aiRemediationMode" = 'RequireApproval' FROM "Project" p WHERE c."projectId" = p."_id" AND p."enableAiCommandExecution" IS NOT TRUE AND c."aiRemediationMode" IN ('Automatic','BypassApproval')`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "RunnerJob" DROP CONSTRAINT "FK_df20ccc2683c9be5fbc8399e098"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesAiAgent" DROP CONSTRAINT "FK_d6ed70c680c438a5ea3080071ca"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesAiAgent" DROP CONSTRAINT "FK_33622e9406d3bffee1ca9c3a6a9"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_df20ccc2683c9be5fbc8399e09"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesCluster" ALTER COLUMN "isAiInvestigationEnabled" SET DEFAULT false`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "RunnerJob" DROP COLUMN "targetKubernetesAiAgentId"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "Project" DROP COLUMN "enableAutomaticPostmortemDraft"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_KubernetesAiAgent_kubernetesClusterId"`,
+    );
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_33622e9406d3bffee1ca9c3a6a"`,
+    );
+    await queryRunner.query(`DROP TABLE "KubernetesAiAgent"`);
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1796200000000-AddProxmoxResourceNativePushColumns.ts

@@ -0,0 +1,42 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Two ProxmoxResource columns for Proxmox VE's built-in OpenTelemetry push,
+ * where each node reports only itself:
+ *
+ * - "isNativePush": the row comes from the native push rather than the
+ *   Proxmox Agent. Such a Node row is kept — and reported Offline by the
+ *   nodes still alive — for the retention window instead of being pruned
+ *   after 15 minutes, because the native push carries no membership.
+ * - "notReportingMarkedAt": when the nodes still alive last reported this
+ *   node as having stopped reporting; cleared by its own next report. It
+ *   lets those reports carry on through a short gap in OneUptime's own
+ *   processing.
+ *
+ * Both nullable with no default: every existing row reads as an agent row
+ * that was never reported down, which keeps the behaviour it already had,
+ * and the next push of a native-push cluster sets them.
+ */
+export class AddProxmoxResourceNativePushColumns1796200000000
+  implements MigrationInterface
+{
+  public name: string = "AddProxmoxResourceNativePushColumns1796200000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxResource" ADD "isNativePush" boolean`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxResource" ADD "notReportingMarkedAt" TIMESTAMP WITH TIME ZONE`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxResource" DROP COLUMN "notReportingMarkedAt"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxResource" DROP COLUMN "isNativePush"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -20,6 +20,9 @@ import { AddStatusPageSubscriberUnsubscribeToken1795600000000 } from "./17956000
 import { AddStatusPageSubscriberIsAddedByTeam1795700000000 } from "./1795700000000-AddStatusPageSubscriberIsAddedByTeam";
 import { AddIncidentCustomFieldCreateAndNotificationSettings1795800000000 } from "./1795800000000-AddIncidentCustomFieldCreateAndNotificationSettings";
 import { AddSubscriberNotificationClaimedAt1795900000000 } from "./1795900000000-AddSubscriberNotificationClaimedAt";
+import { AddTeamComplianceRuleScope1796000000000 } from "./1796000000000-AddTeamComplianceRuleScope";
+import { AddKubernetesAiAgentAndAiDefaults1796100000000 } from "./1796100000000-AddKubernetesAiAgentAndAiDefaults";
+import { AddProxmoxResourceNativePushColumns1796200000000 } from "./1796200000000-AddProxmoxResourceNativePushColumns";
 import { AddConfigurableRuleCriteria1792500000000 } from "./1792500000000-AddConfigurableRuleCriteria";
 import { AddGoogleSecOpsDiagnostics1792300000000 } from "./1792300000000-AddGoogleSecOpsDiagnostics";
 import { AddIncomingCallPolicyPhoneNumbers1792600000000 } from "./1792600000000-AddIncomingCallPolicyPhoneNumbers";
@@ -1240,4 +1243,7 @@ export default [
   AddStatusPageSubscriberIsAddedByTeam1795700000000,
   AddIncidentCustomFieldCreateAndNotificationSettings1795800000000,
   AddSubscriberNotificationClaimedAt1795900000000,
+  AddTeamComplianceRuleScope1796000000000,
+  AddKubernetesAiAgentAndAiDefaults1796100000000,
+  AddProxmoxResourceNativePushColumns1796200000000,
 ];

package/Server/Middleware/TelemetryIngest.ts

@@ -14,6 +14,7 @@ import TelemetryIngestionKeyPolicy, {
 import TelemetryIngestionKeyType from "../../Types/Telemetry/TelemetryIngestionKeyType";
 import TelemetryIngestSurface, {
   BROWSER_ALLOWED_INGEST_SURFACES,
+  IDENTITY_REGISTRATION_SURFACES,
   getIngestSurfaceReadableName,
 } from "../../Types/Telemetry/TelemetryIngestSurface";
 import OriginAllowList from "../../Utils/Telemetry/OriginAllowList";
@@ -72,14 +73,32 @@ const COUNTER_UNAVAILABLE_LOG_INTERVAL_MS: number = 60 * 1000;
 const UNNAMED_SURFACE_READABLE_NAME: string = "this ingest endpoint";
 
 /*
- * The ceiling a server key gets on the Kubernetes agent Runner registration
- * surface when its policy names none. Registration is a handful of calls
- * per agent pod lifetime (start-up, then heartbeats through the Runner's
- * own credential), so a leaked ingestion key hammering it is abuse, not
+ * The ceiling a server key gets on an identity registration surface
+ * (IDENTITY_REGISTRATION_SURFACES: the Kubernetes agent Runner and the
+ * Kubernetes AI agent) when its policy names none. Registration is a handful
+ * of calls per pod lifetime (start-up, then heartbeats through the minted
+ * identity's own key), so a leaked ingestion key hammering it is abuse, not
  * traffic - unlike OTLP ingest, where "no limit" is the historical contract
  * and must stay that way. An explicit per-key limit still wins.
  */
-export const DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE: number = 30;
+export const DEFAULT_IDENTITY_REGISTRATION_REQUESTS_PER_MINUTE: number = 30;
+
+// The name this ceiling shipped under, when only the Runner registered.
+export const DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE: number =
+  DEFAULT_IDENTITY_REGISTRATION_REQUESTS_PER_MINUTE;
+
+/*
+ * The refusal a service-pinned key gets on an identity registration surface.
+ * Worded for whichever identity the surface mints, and it never names the
+ * key, its pinned service or anything else about it.
+ */
+export const getPinnedKeyRegistrationRefusalMessage: (
+  surface: TelemetryIngestSurface,
+) => string = (surface: TelemetryIngestSurface): string => {
+  return `This telemetry ingestion key is pinned to a single service, so it cannot be used for ${getIngestSurfaceReadableName(
+    surface,
+  )}: that grants access to a Kubernetes cluster, not to one service's telemetry. Use an unpinned server ingestion key for the Kubernetes agent.`;
+};
 
 type GetEffectiveRequestsPerMinuteLimitFunction = (
   policy: TelemetryIngestionKeyPolicy,
@@ -101,11 +120,13 @@ type GetEffectiveRequestsPerMinuteLimitFunction = (
  * with no ceiling is the thing we are trying to stop shipping: "the customer
  * did not configure a limit" cannot be allowed to mean "unlimited" there.
  *
- * The one surface-dependent case is Kubernetes agent Runner registration:
- * a server key with no configured limit is held to a conservative default
- * there (see DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE), because
- * that endpoint mints Runner identities rather than accepting telemetry.
- * Every other surface keeps the server-key contract above untouched.
+ * The one surface-dependent case is identity registration
+ * (IDENTITY_REGISTRATION_SURFACES: the Kubernetes agent Runner and the
+ * Kubernetes AI agent): a server key with no configured limit is held to a
+ * conservative default there (see
+ * DEFAULT_IDENTITY_REGISTRATION_REQUESTS_PER_MINUTE), because those
+ * endpoints mint identities rather than accepting telemetry. Every other
+ * surface keeps the server-key contract above untouched.
  */
 export const getEffectiveRequestsPerMinuteLimit: GetEffectiveRequestsPerMinuteLimitFunction =
   (
@@ -129,8 +150,8 @@ export const getEffectiveRequestsPerMinuteLimit: GetEffectiveRequestsPerMinuteLi
       return DEFAULT_BROWSER_KEY_REQUESTS_PER_MINUTE;
     }
 
-    if (surface === TelemetryIngestSurface.KubernetesAgentRunner) {
-      return DEFAULT_KUBERNETES_AGENT_RUNNER_REQUESTS_PER_MINUTE;
+    if (surface && IDENTITY_REGISTRATION_SURFACES.has(surface)) {
+      return DEFAULT_IDENTITY_REGISTRATION_REQUESTS_PER_MINUTE;
     }
 
     return null;
@@ -460,22 +481,24 @@ export default class TelemetryIngest {
       }
 
       /*
-       * Kubernetes agent Runner registration mints a Runner identity — the
-       * one every kubectl job for a cluster is targeted at — so it takes a
-       * project-wide server key. A key pinned to one service's name was
-       * scoped by its owner to that service's telemetry; it is refused here
+       * An identity registration surface (the Kubernetes agent Runner, the
+       * Kubernetes AI agent) mints the identity every kubectl job for a
+       * cluster is targeted at, so it takes a project-wide server key. A key
+       * pinned to one service's name was scoped by its owner to that
+       * service's telemetry; it is refused here, before the rate limiter,
        * rather than letting it stand up cluster access it was never meant
        * to reach.
        */
       if (
-        surface === TelemetryIngestSurface.KubernetesAgentRunner &&
+        surface &&
+        IDENTITY_REGISTRATION_SURFACES.has(surface) &&
         policy.pinnedServiceName
       ) {
         return Response.sendErrorResponse(
           req,
           res,
           new NotAuthorizedException(
-            "This telemetry ingestion key is pinned to a single service, so it cannot register the Kubernetes agent's in-cluster Runner. Use an unpinned server ingestion key for the Kubernetes agent.",
+            getPinnedKeyRegistrationRefusalMessage(surface),
           ),
         );
       }

package/Server/Services/AIService.ts

@@ -313,7 +313,14 @@ export const INTERACTIVE_AI_GENERATION_TIMEOUT_IN_MS: number = 4 * 60 * 1000;
  * screen where the switch can be turned back on.
  */
 export const AI_DISABLED_MESSAGE: string =
-  "AI features are disabled for this project. Enable them in Project Settings > AI Credits.";
+  "AI features are disabled for this project. Enable them in Project Settings → AI Features.";
+
+/*
+ * Why no AI run can start in a project whose OneUptime AI credits are used
+ * up (getAiBalanceBlocker), in the words every surface that says so shares.
+ */
+export const AI_BALANCE_INSUFFICIENT_MESSAGE: string =
+  "This project's AI credit balance is used up and auto-recharge is off, so OneUptime AI cannot run.";
 
 export interface AILogRequest {
   projectId: ObjectID;
@@ -486,6 +493,68 @@ export class Service extends BaseService {
     }
   }
 
+  /*
+   * Would an AI call in this project be refused for lack of AI credits?
+   * Returns why (AI_BALANCE_INSUFFICIENT_MESSAGE), or null when nothing
+   * about the balance stands in the way. The same test executeWithLogging
+   * applies to every call — billing is on, the project's provider is the
+   * OneUptime-hosted (global) one and it has a per-token cost, and the
+   * balance is at or below zero — plus one it cannot see: auto-recharge.
+   * A project with auto-recharge on is topped up when its balance runs low,
+   * so an empty balance there is not a standing blocker.
+   *
+   * Asked in advance by the readiness checks (the cluster AI status, the
+   * investigation eligibility), so "out of credits" is said once, up front,
+   * instead of as a failed run per incident. Takes the provider when the
+   * caller already resolved it, to spare a second lookup.
+   */
+  @CaptureSpan()
+  public async getAiBalanceBlocker(data: {
+    projectId: ObjectID;
+    llmProvider?: LlmProvider | null | undefined;
+  }): Promise<string | null> {
+    if (!IsBillingEnabled) {
+      return null;
+    }
+
+    const llmProvider: LlmProvider | null =
+      data.llmProvider !== undefined
+        ? data.llmProvider
+        : await LlmProviderService.getLLMProviderForProject(data.projectId);
+
+    // No provider is its own blocker, reported as such by the callers.
+    if (
+      !llmProvider ||
+      llmProvider.isGlobalLlm !== true ||
+      (llmProvider.costPerMillionTokensInUSDCents || 0) <= 0
+    ) {
+      return null;
+    }
+
+    const project: Project | null = await ProjectService.findOneById({
+      id: data.projectId,
+      select: {
+        aiCurrentBalanceInUSDCents: true,
+        enableAutoRechargeAiBalance: true,
+      },
+      props: { isRoot: true },
+    });
+
+    if (!project) {
+      return null;
+    }
+
+    if ((project.aiCurrentBalanceInUSDCents || 0) > 0) {
+      return null;
+    }
+
+    if (project.enableAutoRechargeAiBalance === true) {
+      return null;
+    }
+
+    return AI_BALANCE_INSUFFICIENT_MESSAGE;
+  }
+
   /*
    * G4 daily budget: has this project consumed the selected subject lane's
    * daily autonomous-token allowance (UTC day)? Counts only that lane's