@oneuptime/common 14.0.8 → 14.0.10

package/Server/Services/TelemetryUsageBillingService.ts

@@ -44,6 +44,7 @@ import IoTFleet from "../../Models/DatabaseModels/IoTFleet";
 import CephCluster from "../../Models/DatabaseModels/CephCluster";
 import DatabaseServer from "../../Models/DatabaseModels/DatabaseServer";
 import ServiceType from "../../Types/Telemetry/ServiceType";
+import SessionReplayBudgetMetricTypeUtil from "../../Utils/SessionReplay/SessionReplayBudgetMetricType";
 import {
   AverageSpanRowSizeInBytes,
   AverageLogRowSizeInBytes,
@@ -96,6 +97,27 @@ export const isTelemetryBillingExcludedEntityType: IsTelemetryBillingExcludedEnt
     );
   };
 
+/*
+ * Metric names whose rows are never billed as ingested volume. The session
+ * replay budget sweep posts its oneuptime.rum.session.replay.budget.*
+ * readings every five minutes: OneUptime's own operational data about the
+ * customer's replay budget, like the SLO rows above, and billing them would
+ * charge a project for having recorded a replay.
+ *
+ * Left out by NAME, not by type: the rows keep primaryEntityType =
+ * RealUserMonitor, keyed to the application, so monitors scoped to that
+ * application see them - and the application's real web-vitals rows share
+ * that id and type, so an entity-type exclusion would stop billing those
+ * too. That is only safe because nothing but the budget sweep can write a
+ * name under the session replay prefix (isReservedMetricName in
+ * Common/Utils/SessionReplay/SessionReplayBudgetMetricType): OTLP ingest drops them,
+ * and a metric or trace recording rule cannot be saved with one as its
+ * output. So no customer telemetry can be stored under one of these names to
+ * be kept for free.
+ */
+export const TELEMETRY_BILLING_EXCLUDED_METRIC_NAMES: ReadonlyArray<string> =
+  SessionReplayBudgetMetricTypeUtil.getAll();
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
@@ -329,12 +351,18 @@ export class Service extends DatabaseService<Model> {
           averageRowSizeInBytes,
         );
       } else if (data.productType === ProductType.Metrics) {
+        /*
+         * The session replay budget rows share their RUM application's
+         * (primaryEntityId, primaryEntityType) with its web vitals, so they
+         * can only be told apart - and left out - inside the scan.
+         */
         addUsage(
           await MetricService.groupTelemetryUsageByService({
             projectId: data.projectId,
             timestampColumnName: "time",
             startDate: startOfDay,
             endDate: endOfDay,
+            excludeNames: [...TELEMETRY_BILLING_EXCLUDED_METRIC_NAMES],
           }),
           averageRowSizeInBytes,
         );

package/Server/Services/TraceRecordingRuleService.ts

@@ -1,10 +1,43 @@
 import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/TraceRecordingRule";
+import CreateBy from "../Types/Database/CreateBy";
+import UpdateBy from "../Types/Database/UpdateBy";
+import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import { Service as MetricRecordingRuleService } from "./MetricRecordingRuleService";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
   }
+
+  /*
+   * A trace recording rule writes a metric too, and straight into the metric
+   * store: it may not claim a reserved session replay name any more than a
+   * metric recording rule may (MetricRecordingRuleService).
+   */
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    MetricRecordingRuleService.assertOutputMetricNameAllowed(
+      createBy.data.outputMetricName,
+    );
+
+    return { createBy, carryForward: null };
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<Model>,
+  ): Promise<OnUpdate<Model>> {
+    // Only an update that sets the name can make it reserved.
+    MetricRecordingRuleService.assertOutputMetricNameAllowed(
+      (updateBy.data as unknown as Record<string, unknown>)["outputMetricName"],
+    );
+
+    return { updateBy, carryForward: null };
+  }
 }
 
 export default new Service();

package/Server/Services/VMwareVCenterService.ts

@@ -3,7 +3,15 @@ import VMwareVCenterLabelRuleEngineService from "./VMwareVCenterLabelRuleEngineS
 import VMwareVCenterOwnerRuleEngineService from "./VMwareVCenterOwnerRuleEngineService";
 import Model from "../../Models/DatabaseModels/VMwareVCenter";
 import Label from "../../Models/DatabaseModels/Label";
-import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
+import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
+import CreateBy from "../Types/Database/CreateBy";
+import DeleteBy from "../Types/Database/DeleteBy";
+import UpdateBy from "../Types/Database/UpdateBy";
+import ResourceAiAccessSettings, {
+  ResourceAiAccessFeedItem,
+} from "../Utils/AI/ResourceAccess/ResourceAiAccessSettings";
+import ResourceAiDeleteCleanup from "../Utils/AI/ResourceAccess/ResourceAiDeleteCleanup";
+import AiResourceType from "../../Types/ResourceAiAgent/AiResourceType";
 import VMwareVCenterFeedService from "./VMwareVCenterFeedService";
 import { VMwareVCenterFeedEventType } from "../../Models/DatabaseModels/VMwareVCenterFeed";
 import ResourceFeedUtil from "../Utils/ResourceFeed/ResourceFeedUtil";
@@ -513,6 +521,79 @@ export class Service extends DatabaseService<Model> {
     });
   }
 
+  /*
+   * Deleting a vCenter settles its in-flight AI remediation rounds and removes
+   * its resource AI agent — read before the delete, done after it for the
+   * vCenters actually deleted (ResourceAiDeleteCleanup, shared by every
+   * resource AI agent's resource).
+   */
+  @CaptureSpan()
+  protected override async onBeforeDelete(
+    deleteBy: DeleteBy<Model>,
+  ): Promise<OnDelete<Model>> {
+    return {
+      deleteBy,
+      carryForward: await ResourceAiDeleteCleanup.beforeDelete({
+        resourceType: AiResourceType.VMwareVCenter,
+        service: this,
+        deleteBy,
+      }),
+    };
+  }
+
+  @CaptureSpan()
+  protected override async onDeleteSuccess(
+    onDelete: OnDelete<Model>,
+    deletedItemIds: Array<ObjectID>,
+  ): Promise<OnDelete<Model>> {
+    await ResourceAiDeleteCleanup.afterDelete({
+      resourceType: AiResourceType.VMwareVCenter,
+      onDelete,
+      deletedItemIds,
+    });
+
+    return onDelete;
+  }
+
+  /*
+   * A create is held to the same AI access rules as an update, judged
+   * against the never-configured defaults a new vCenter starts from: an
+   * unusable mode or allowlist is refused for every caller, and the
+   * server-only aiAccess* columns for every caller but root (a master admin
+   * included, whom the create column ACLs never check).
+   */
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    ResourceAiAccessSettings.checkCreate({
+      resourceType: AiResourceType.VMwareVCenter,
+      createBy,
+    });
+
+    return { createBy, carryForward: null };
+  }
+
+  /*
+   * An operator's write of an AI access setting (the investigation switch, the
+   * remediation mode, the command allowlist) is validated and checked against
+   * who may make AI do more on this vCenter — the rules every resource AI
+   * agent's resource shares (ResourceAiAccessSettings).
+   */
+  @CaptureSpan()
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<Model>,
+  ): Promise<OnUpdate<Model>> {
+    return {
+      updateBy,
+      carryForward: await ResourceAiAccessSettings.checkUpdate({
+        resourceType: AiResourceType.VMwareVCenter,
+        service: this,
+        updateBy,
+      }),
+    };
+  }
+
   @CaptureSpan()
   protected override async onUpdateSuccess(
     onUpdate: OnUpdate<Model>,
@@ -524,6 +605,34 @@ export class Service extends DatabaseService<Model> {
       },
     );
 
+    /*
+     * An operator's AI access write: recorded on the feed, and the vCenter
+     * marked AI-configured.
+     */
+    await ResourceAiAccessSettings.afterUpdate({
+      service: this,
+      onUpdate,
+      updatedItemIds,
+      getResourceMarkdownLink: (
+        projectId: ObjectID,
+        vmwareVCenterId: ObjectID,
+      ): Promise<string> => {
+        return this.getVMwareVCenterMarkdownLink(projectId, vmwareVCenterId);
+      },
+      createFeedItem: async (item: ResourceAiAccessFeedItem): Promise<void> => {
+        await VMwareVCenterFeedService.createVMwareVCenterFeedItem({
+          vmwareVCenterId: item.resourceId,
+          projectId: item.projectId,
+          vmwareVCenterFeedEventType:
+            VMwareVCenterFeedEventType.VMwareVCenterUpdated,
+          displayColor: item.displayColor,
+          feedInfoInMarkdown: item.feedInfoInMarkdown,
+          moreInformationInMarkdown: item.moreInformationInMarkdown,
+          userId: item.userId,
+        });
+      },
+    });
+
     return onUpdate;
   }
 

package/Server/Utils/AI/ClusterAccess/ClusterAccessContext.ts

@@ -2,6 +2,7 @@ import {
   KubernetesAiAccessGap,
   KubernetesAiRemediationMode,
   KubernetesClusterAiAccessStatus,
+  getKubernetesAiAccessTargetKind,
 } from "../../../../Types/Kubernetes/KubernetesClusterAiAccess";
 import { KUBECTL_ALWAYS_ASKS_SUMMARY } from "../../../../Types/AutoRemediation/AiRemediationCommandPlan";
 
@@ -20,6 +21,46 @@ function capitalizeFirst(text: string): string {
   return text.charAt(0).toUpperCase() + text.slice(1);
 }
 
+/*
+ * Who runs kubectl for a cluster, by name, for copy the model and humans
+ * read: "the Kubernetes AI agent", or `Runner "<name>"`. Told apart by
+ * runner.kind only — accessMethod is "in_cluster" for both the agent and
+ * the chart's previous in-cluster Runner.
+ */
+export function describeClusterAccessTarget(
+  status: Pick<KubernetesClusterAiAccessStatus, "runner">,
+): string {
+  if (getKubernetesAiAccessTargetKind(status.runner) === "ai_agent") {
+    return "the Kubernetes AI agent";
+  }
+
+  return `Runner "${status.runner?.name || "Runner"}"`;
+}
+
+/*
+ * The same with "its" ("its Kubernetes AI agent", "its Runner"), for copy
+ * that has already named the cluster.
+ */
+export function describeClusterAccessTargetOfCluster(
+  status: Pick<KubernetesClusterAiAccessStatus, "runner">,
+): string {
+  return getKubernetesAiAccessTargetKind(status.runner) === "ai_agent"
+    ? "its Kubernetes AI agent"
+    : "its Runner";
+}
+
+/*
+ * The same as a role, for copy about the cluster in general: "the
+ * cluster's Kubernetes AI agent" or "the cluster's Runner".
+ */
+export function describeClusterAccessTargetRole(
+  status: Pick<KubernetesClusterAiAccessStatus, "runner">,
+): string {
+  return getKubernetesAiAccessTargetKind(status.runner) === "ai_agent"
+    ? "the cluster's Kubernetes AI agent"
+    : "the cluster's Runner";
+}
+
 /*
  * The prompt-side view of cluster access.
  *
@@ -55,7 +96,7 @@ export default class ClusterAccessContext {
           })
           .join(
             ", ",
-          )}. Use run_kubectl for READ-ONLY inspection — kubectl get/describe/events/logs/top/rollout status — the way an on-call engineer would open a terminal: describe the failing pod, read its recent events, check node capacity and pending-pod reasons, tail the crashing container's logs. Prefer direct cluster inspection over guessing from metrics when the two disagree. Every kubectl command that ran is cited like any other tool result; a command that could not run comes back as an error, is not evidence, and must never be described as inspected. The Runner that runs kubectl lives in the cluster and can itself be down: if run_kubectl says a cluster's Runner did not pick up a command, that cluster is unreachable for the rest of this investigation — do not call run_kubectl on it again, continue with OneUptime telemetry, and say so in your report. Reading Secrets is refused and credential-looking values (Secret data, passwords, tokens, keys) are redacted from every output before you see it — never ask for them and never treat a redaction marker as a finding.`,
+          )}. Use run_kubectl for READ-ONLY inspection — kubectl get/describe/events/logs/top/rollout status — the way an on-call engineer would open a terminal: describe the failing pod, read its recent events, check node capacity and pending-pod reasons, tail the crashing container's logs. Prefer direct cluster inspection over guessing from metrics when the two disagree. Every kubectl command that ran is cited like any other tool result; a command that could not run comes back as an error, is not evidence, and must never be described as inspected. The agent (or Runner) that runs kubectl lives in the cluster and can itself be down: if run_kubectl says a cluster's agent or Runner did not pick up a command, that cluster is unreachable for the rest of this investigation — do not call run_kubectl on it again, continue with OneUptime telemetry, and say so in your report. Reading Secrets is refused and credential-looking values (Secret data, passwords, tokens, keys) are redacted from every output before you see it — never ask for them and never treat a redaction marker as a finding.`,
       );
     }
 
@@ -78,7 +119,7 @@ export default class ClusterAccessContext {
     }
 
     lines.push(
-      `Add a section **${ClusterAccessContext.REPORT_SECTION_HEADING}** before Suggested next steps: one or two sentences on what you inspected directly on the cluster (or that you could not — access not set up, or the cluster's Runner not responding — and that the human should check the cluster's AI page).`,
+      `Add a section **${ClusterAccessContext.REPORT_SECTION_HEADING}** before Suggested next steps: one or two sentences on what you inspected directly on the cluster (or that you could not — access not set up, or the cluster's AI agent or Runner not responding — and that the human should check the cluster's AI agent page (AI → Agent)).`,
     );
 
     return lines.join("\n");
@@ -97,13 +138,9 @@ export default class ClusterAccessContext {
     for (const status of statuses) {
       if (status.isInvestigationReady) {
         lines.push(
-          `- Cluster "${status.clusterName}" (clusterId: ${status.clusterId}): kubectl READ access available via run_kubectl${
-            status.accessMethod === "in_cluster"
-              ? " (in-cluster Runner)"
-              : status.credentialName
-                ? ` (Runner "${status.runner?.name}" with credential "${status.credentialName}")`
-                : ""
-          }. Remediation: ${ClusterAccessContext.describeRemediationMode(status)}.`,
+          `- Cluster "${status.clusterName}" (clusterId: ${status.clusterId}): kubectl READ access available via run_kubectl${ClusterAccessContext.describeAccessPath(
+            status,
+          )}. Remediation: ${ClusterAccessContext.describeRemediationMode(status)}.`,
         );
         continue;
       }
@@ -126,6 +163,27 @@ export default class ClusterAccessContext {
     return lines.join("\n");
   }
 
+  /*
+   * How kubectl reaches a ready cluster, as a parenthetical: the Kubernetes
+   * AI agent, the chart's previous in-cluster Runner, or a Runner with a
+   * credential.
+   */
+  private static describeAccessPath(
+    status: KubernetesClusterAiAccessStatus,
+  ): string {
+    if (getKubernetesAiAccessTargetKind(status.runner) === "ai_agent") {
+      return " (via the in-cluster Kubernetes AI agent)";
+    }
+
+    if (status.accessMethod === "in_cluster") {
+      return " (in-cluster Runner)";
+    }
+
+    return status.credentialName
+      ? ` (Runner "${status.runner?.name}" with credential "${status.credentialName}")`
+      : "";
+  }
+
   /*
    * The sentence the investigation panel shows humans when AI investigated
    * with OneUptime data only. Deterministic so it reads the same on every

package/Server/Utils/AI/ClusterAccess/KubectlInvestigationToolkit.ts

@@ -7,6 +7,7 @@ import {
   KubernetesClusterAiAccessStatus,
   MAX_KUBECTL_COMMANDS_PER_INVESTIGATION,
   MAX_KUBECTL_TIMEOUT_MS,
+  getKubernetesAiAccessTargetKind,
 } from "../../../../Types/Kubernetes/KubernetesClusterAiAccess";
 import KubectlPolicy, {
   KubectlPolicyResult,
@@ -30,6 +31,11 @@ import {
   LIST_CLUSTER_ACCESS_TOOL_NAME,
   RUN_KUBECTL_TOOL_NAME,
 } from "../../../../Types/Kubernetes/KubernetesClusterAiAccessToolNames";
+import {
+  describeClusterAccessTarget,
+  describeClusterAccessTargetOfCluster,
+  describeClusterAccessTargetRole,
+} from "./ClusterAccessContext";
 
 /*
  * The run-scoped, READ-ONLY kubectl toolkit for an investigation.
@@ -227,8 +233,13 @@ export default class KubectlInvestigationToolkit {
                 : ""
             } — ${
               this.isClusterUnreachable(cluster.clusterId)
-                ? `UNREACHABLE for the rest of this investigation: its Runner "${cluster.runner?.name}" did not pick up an earlier command. Do not call run_kubectl on it again.`
-                : `read-only kubectl via Runner "${cluster.runner?.name}"`
+                ? `UNREACHABLE for the rest of this investigation: its ${describeClusterAccessTarget(
+                    cluster,
+                  ).replace(
+                    /^the /,
+                    "",
+                  )} did not pick up an earlier command. Do not call run_kubectl on it again.`
+                : `read-only kubectl via ${describeClusterAccessTarget(cluster)}`
             }`;
           })
           .join("\n");
@@ -345,15 +356,20 @@ export default class KubectlInvestigationToolkit {
     );
 
     if (tripped !== undefined) {
+      const target: string = describeClusterAccessTarget(cluster);
+      const role: string = describeClusterAccessTargetRole(cluster);
+
       return this.failure(
-        `Runner "${cluster.runner.name}", which serves cluster "${cluster.clusterName}", did not pick up an earlier command${
+        `${target.charAt(0).toUpperCase()}${target.slice(1)}, which serves cluster "${cluster.clusterName}", did not pick up an earlier command${
           tripped.clusterName !== cluster.clusterName
             ? ` (for cluster "${tripped.clusterName}")`
             : ""
         } within ${KubectlInvestigationToolkit.describeSeconds(
           tripped.claimWindowMs,
-        )}, so the cluster is treated as unreachable for the rest of this investigation. Nothing was run. Do not call run_kubectl on this cluster again: continue with OneUptime telemetry and say in **Cluster access** that the cluster's Runner did not respond.`,
-        `kubectl was not run on cluster "${cluster.clusterName}": its Runner did not pick up an earlier command, so the cluster was unreachable for the rest of this investigation.`,
+        )}, so the cluster is treated as unreachable for the rest of this investigation. Nothing was run. Do not call run_kubectl on this cluster again: continue with OneUptime telemetry and say in **Cluster access** that ${role} did not respond.`,
+        `kubectl was not run on cluster "${cluster.clusterName}": ${describeClusterAccessTargetOfCluster(
+          cluster,
+        )} did not pick up an earlier command, so the cluster was unreachable for the rest of this investigation.`,
       );
     }
 
@@ -427,8 +443,12 @@ export default class KubectlInvestigationToolkit {
       const alsoUnreachable: Array<KubernetesClusterAiAccessStatus> =
         this.getOtherClustersOnRunner(cluster);
 
+      const role: string = describeClusterAccessTargetRole(cluster);
+
       return this.failure(
-        `kubectl was NOT run on cluster "${cluster.clusterName}": its Runner did not pick up "${outcome.displayCommand}" within ${KubectlInvestigationToolkit.describeSeconds(
+        `kubectl was NOT run on cluster "${cluster.clusterName}": ${describeClusterAccessTargetOfCluster(
+          cluster,
+        )} did not pick up "${outcome.displayCommand}" within ${KubectlInvestigationToolkit.describeSeconds(
           budget.plan.claimTimeoutInMs,
         )} (it may be offline, restarting or busy). The cluster is treated as unreachable for the rest of this investigation — do not call run_kubectl on it again.${
           alsoUnreachable.length > 0
@@ -440,8 +460,10 @@ export default class KubectlInvestigationToolkit {
                   ", ",
                 )}, so ${alsoUnreachable.length === 1 ? "that cluster is" : "those clusters are"} unreachable too.`
             : ""
-        } Continue with OneUptime telemetry and say in **Cluster access** that the cluster's Runner did not respond.`,
-        `kubectl was not run on cluster "${cluster.clusterName}": its Runner did not pick up the command in time.`,
+        } Continue with OneUptime telemetry and say in **Cluster access** that ${role} did not respond.`,
+        `kubectl was not run on cluster "${cluster.clusterName}": ${describeClusterAccessTargetOfCluster(
+          cluster,
+        )} did not pick up the command in time.`,
       );
     }
 
@@ -460,9 +482,16 @@ export default class KubectlInvestigationToolkit {
         ? redactedReason
         : `${redactedReason}.`;
 
+      const isAgent: boolean =
+        getKubernetesAiAccessTargetKind(cluster.runner) === "ai_agent";
+
       return this.failure(
-        `No kubectl result came back from cluster "${cluster.clusterName}" for "${outcome.displayCommand}": ${reason} Nothing from this command is evidence. Continue with OneUptime telemetry and mention in **Cluster access** that the cluster's Runner stopped responding.`,
-        `${KUBECTL_RESULT_UNKNOWN_EVENT_PREFIX} the Runner of cluster "${cluster.clusterName}" took the command, but no result came back, so whether it ran is unknown.`,
+        `No kubectl result came back from cluster "${cluster.clusterName}" for "${outcome.displayCommand}": ${reason} Nothing from this command is evidence. Continue with OneUptime telemetry and mention in **Cluster access** that ${describeClusterAccessTargetRole(
+          cluster,
+        )} stopped responding.`,
+        `${KUBECTL_RESULT_UNKNOWN_EVENT_PREFIX} the ${
+          isAgent ? "Kubernetes AI agent" : "Runner"
+        } of cluster "${cluster.clusterName}" took the command, but no result came back, so whether it ran is unknown.`,
       );
     }
 

package/Server/Utils/AI/ClusterAccess/KubectlJobRunner.ts

@@ -392,6 +392,9 @@ export default class KubectlJobRunner {
                   wasClaimed: claimState.wasClaimed,
                   claimTimeoutInMs: data.claimTimeoutInMs,
                   executionTimeoutInMs: data.executionTimeoutInMs,
+                  isForKubernetesAiAgent: Boolean(
+                    job.targetKubernetesAiAgentId,
+                  ),
                 })
               : terminalJob.errorMessage ||
                   `Command ended with status ${terminalJob.status}.`,
@@ -612,18 +615,26 @@ export default class KubectlJobRunner {
     wasClaimed: boolean | undefined;
     claimTimeoutInMs: number;
     executionTimeoutInMs: number;
+    // The job was for the cluster's Kubernetes AI agent, not a Runner.
+    isForKubernetesAiAgent?: boolean | undefined;
   }): string {
+    const executor: string = data.isForKubernetesAiAgent
+      ? "Kubernetes AI agent"
+      : "Runner";
+
     if (data.wasClaimed === false) {
-      return `The cluster's Runner did not pick up this kubectl command within ${KubectlJobRunner.describeSeconds(
+      return `The cluster's ${executor} did not pick up this kubectl command within ${KubectlJobRunner.describeSeconds(
         data.claimTimeoutInMs,
       )} — it may be offline, restarting or busy with other work. Nothing was run on the cluster.`;
     }
 
     if (data.wasClaimed === undefined) {
-      return "No result came back for this kubectl command in time, and whether a Runner picked it up could not be read. What the command did is unknown.";
+      return `No result came back for this kubectl command in time, and whether ${
+        data.isForKubernetesAiAgent ? "the Kubernetes AI agent" : "a Runner"
+      } picked it up could not be read. What the command did is unknown.`;
     }
 
-    return `The Runner took this kubectl command but did not report a result in time — it stopped responding, or kubectl outlived its ${KubectlJobRunner.describeSeconds(
+    return `The ${executor} took this kubectl command but did not report a result in time — it stopped responding, or kubectl outlived its ${KubectlJobRunner.describeSeconds(
       data.executionTimeoutInMs,
     )} timeout. Whether it ran, and what it did, is unknown.`;
   }