@oneuptime/common 14.0.8 → 14.0.10

package/Types/Kubernetes/KubernetesClusterAiAccess.ts

@@ -1,12 +1,18 @@
 /*
  * OneUptime AI access to a Kubernetes cluster.
  *
- * A cluster's AI page binds one Runner (in-cluster, installed by the
- * kubernetes-agent chart, or an existing Runner plus a Kubernetes
- * credential) and two switches: whether AI may run read-only kubectl during
- * investigations, and how AI may remediate. Everything here is shared by the
- * server, the Runner binary and the dashboard so the three never disagree
- * about what a mode or a tier means.
+ * AI reaches a cluster through one access target: the Kubernetes AI agent
+ * the kubernetes-agent chart installs (its own identity, never a Runner), the
+ * chart's previous in-cluster Runner, or an existing Runner plus a Kubernetes
+ * credential. Two switches decide what AI may do there: whether it may run
+ * read-only kubectl during investigations, and how it may remediate.
+ * Everything here is shared by the server, the Runner binary, the Kubernetes
+ * AI agent and the dashboard so they never disagree about what a mode or a
+ * tier means.
+ *
+ * This file has NO imports on purpose: the Kubernetes AI agent
+ * (agents/KubernetesAIAgent) carries a byte-identical copy of it and must
+ * compile without the rest of Common.
  */
 
 /*
@@ -121,7 +127,26 @@ export type KubernetesAiAccessGapCode =
   | "project_auto_remediation_disabled"
   | "project_ai_command_execution_disabled"
   | "llm_provider_missing"
-  | "last_access_check_failed";
+  | "last_access_check_failed"
+  /*
+   * Neither the cluster's Kubernetes AI agent nor any Runner can reach the
+   * cluster: the agent never registered (the chart was not upgraded, or was
+   * installed with aiAgent.enabled=false) and no Runner is bound. Blocks
+   * both. Replaces no_runner_bound, which the server no longer produces.
+   */
+  | "ai_agent_not_connected"
+  /*
+   * The resolved access target is the cluster's Kubernetes AI agent and it
+   * has not been seen within KUBERNETES_AI_AGENT_ALIVE_WINDOW_IN_MINUTES.
+   * Blocks both.
+   */
+  | "ai_agent_offline"
+  /*
+   * Billing is on, the project's LLM provider is OneUptime's own (which
+   * costs AI credits), the balance is used up and auto-recharge is off, so
+   * no AI run can start. Blocks both.
+   */
+  | "ai_balance_insufficient";
 
 export interface KubernetesAiAccessGap {
   code: KubernetesAiAccessGapCode;
@@ -168,15 +193,104 @@ export interface KubernetesRunnerPosture {
   allowNodeOperations?: boolean | undefined;
 }
 
+/*
+ * What the Kubernetes AI agent reports about itself on registration and on
+ * every heartbeat. The same shape as a Runner's posture — the agent is the
+ * chart's in-cluster kubectl executor, just with its own identity — but
+ * stored BARE on KubernetesAiAgent.posture rather than under
+ * hostInfo.kubernetes, so it is parsed with parseKubernetesAgentPosture.
+ */
+export type KubernetesAgentPosture = KubernetesRunnerPosture;
+
+/*
+ * Which kind of access target AI reaches a cluster through.
+ *
+ * ai_agent: the cluster's Kubernetes AI agent (a KubernetesAiAgent row, never
+ *           a Runner). It authenticates to the API server with its own
+ *           ServiceAccount, so its accessMethod is "in_cluster".
+ * runner:   a Runner row — the chart's previous in-cluster Runner
+ *           (accessMethod "in_cluster") or a Runner holding a Kubernetes
+ *           credential (accessMethod "credential").
+ *
+ * accessMethod says how kubectl authenticates; this says which identity runs
+ * it. Tell the AI agent and the legacy in-cluster Runner apart ONLY by this.
+ */
+export type KubernetesAiAccessTargetKind = "ai_agent" | "runner";
+
+/*
+ * The access target AI reaches a cluster through. Named "runner" for
+ * compatibility: plans and jobs store the target id as runnerId whatever its
+ * kind. When the target is the Kubernetes AI agent, id is the
+ * KubernetesAiAgent row id, name is KUBERNETES_AI_AGENT_DISPLAY_NAME, kind is
+ * "ai_agent" and canRunAiCommands is always true.
+ */
 export interface KubernetesAiAccessRunnerSummary {
   id: string;
   name: string;
+  // Absent means "runner" (every summary built before the AI agent existed).
+  kind?: KubernetesAiAccessTargetKind | undefined;
   isOnline: boolean;
   lastAliveAt?: string | undefined;
   canRunAiCommands: boolean;
   posture?: KubernetesRunnerPosture | undefined;
 }
 
+/*
+ * The kind of an access target summary, defaulting an absent kind to
+ * "runner" the way the field is documented.
+ */
+export function getKubernetesAiAccessTargetKind(
+  runner:
+    | { kind?: KubernetesAiAccessTargetKind | undefined }
+    | null
+    | undefined,
+): KubernetesAiAccessTargetKind | null {
+  if (!runner) {
+    return null;
+  }
+
+  return runner.kind === "ai_agent" ? "ai_agent" : "runner";
+}
+
+/*
+ * What the Kubernetes AI agent last told the server about its connection:
+ * "connected" from registration and every heartbeat, "disconnected" when it
+ * signs off (or an admin resets it). Online additionally needs a heartbeat
+ * within KUBERNETES_AI_AGENT_ALIVE_WINDOW_IN_MINUTES.
+ */
+export type KubernetesAiAgentConnectionStatus = "connected" | "disconnected";
+
+/*
+ * The cluster's Kubernetes AI agent as the dashboard shows it, whether or not
+ * it is the target AI currently uses (an advanced Runner binding wins over
+ * it). Never carries the agent key or its hash.
+ */
+export interface KubernetesAiAgentSummary {
+  id: string;
+  isOnline: boolean;
+  connectionStatus: KubernetesAiAgentConnectionStatus;
+  lastAliveAt?: string | undefined;
+  lastRegisteredAt?: string | undefined;
+  agentVersion?: string | undefined;
+  posture?: KubernetesAgentPosture | undefined;
+  /*
+   * The last time another agent pod tried to register for this cluster
+   * while this one was online, and why it was refused.
+   */
+  lastRefusedRegistrationAt?: string | undefined;
+  lastRefusedRegistrationReason?: string | undefined;
+}
+
+/*
+ * The project's automatic-investigation opt-ins, shown as a non-blocking
+ * line on the cluster's AI agent page. They are project-wide, not per
+ * cluster, and never produce a gap.
+ */
+export interface KubernetesAiAutomaticInvestigationSettings {
+  incidents: boolean;
+  alerts: boolean;
+}
+
 /*
  * The complete answer to "can OneUptime AI reach this cluster, and how?".
  * Computed from current configuration every time it is asked for — it is
@@ -186,9 +300,25 @@ export interface KubernetesClusterAiAccessStatus {
   clusterId: string;
   clusterName: string;
   clusterIdentifier?: string | undefined;
+  // The resolved access target (see KubernetesAiAccessRunnerSummary.kind).
   runner: KubernetesAiAccessRunnerSummary | null;
-  // "in_cluster" when the bound Runner uses its own ServiceAccount.
+  /*
+   * "in_cluster" when the target uses its own ServiceAccount (the
+   * Kubernetes AI agent or the chart's previous in-cluster Runner),
+   * "credential" when it is a Runner using a Kubernetes credential.
+   */
   accessMethod: "in_cluster" | "credential" | "none";
+  /*
+   * The cluster's Kubernetes AI agent row, whether or not it is the active
+   * target; null when no agent ever registered for the cluster.
+   */
+  aiAgent: KubernetesAiAgentSummary | null;
+  /*
+   * The project's automatic-investigation opt-ins. Not gaps: the AI agent
+   * page shows them as a footer line, because investigating automatically
+   * is a project-wide choice, not something this cluster is missing.
+   */
+  automaticInvestigation: KubernetesAiAutomaticInvestigationSettings;
   // Set with accessMethod "credential": the RunbookCredential the jobs name.
   credentialId?: string | undefined;
   credentialName?: string | undefined;
@@ -240,28 +370,43 @@ export function isKubernetesAgentRunnerName(name: unknown): boolean {
 }
 
 /*
- * The Runner is the only component that ever holds cluster credentials, so
- * the server learns the Runner's write posture from the Runner itself. A
- * Runner installed with read-only RBAC reports allowWrites=false and the
- * server then never enqueues a write for it.
+ * The in-cluster executor (the Kubernetes AI agent, or a Runner) is the only
+ * component that ever holds cluster credentials, so the server learns its
+ * write posture from the executor itself. One installed with read-only RBAC
+ * reports allowWrites=false and the server then never enqueues a write for
+ * it.
+ *
+ * parseKubernetesAgentPosture reads a BARE posture object: the body the
+ * Kubernetes AI agent sends and the KubernetesAiAgent.posture column.
+ * parseKubernetesRunnerPosture reads a Runner's hostInfo, where the same
+ * object sits under hostInfo.kubernetes. Both validate every field the same
+ * way: anything of the wrong type is dropped, never trusted.
  */
-export function parseKubernetesRunnerPosture(
-  hostInfo: unknown,
-): KubernetesRunnerPosture | undefined {
-  if (!hostInfo || typeof hostInfo !== "object") {
+export function parseKubernetesAgentPosture(
+  raw: unknown,
+): KubernetesAgentPosture | undefined {
+  if (!raw || typeof raw !== "object") {
     return undefined;
   }
 
-  const kubernetes: unknown = (hostInfo as Record<string, unknown>)[
-    "kubernetes"
-  ];
+  return parsePostureFields(raw as Record<string, unknown>);
+}
 
-  if (!kubernetes || typeof kubernetes !== "object") {
+export function parseKubernetesRunnerPosture(
+  hostInfo: unknown,
+): KubernetesRunnerPosture | undefined {
+  if (!hostInfo || typeof hostInfo !== "object") {
     return undefined;
   }
 
-  const raw: Record<string, unknown> = kubernetes as Record<string, unknown>;
+  return parseKubernetesAgentPosture(
+    (hostInfo as Record<string, unknown>)["kubernetes"],
+  );
+}
 
+function parsePostureFields(
+  raw: Record<string, unknown>,
+): KubernetesAgentPosture {
   return {
     clusterIdentifier:
       typeof raw["clusterIdentifier"] === "string"
@@ -381,15 +526,18 @@ export function isInClusterPostureForCluster(
  * bound_to_other_runner:            an operator bound the cluster to a
  *                                   different Runner in the dashboard; left
  *                                   alone.
- * left_unbound_by_operator:         a Runner was bound to this cluster
- *                                   before and no Runner is bound now — an
- *                                   operator cleared the binding, or the
- *                                   bound Runner row was deleted (the
- *                                   foreign key nulls it). Registration does
- *                                   not re-bind or flip any switch; the agent
- *                                   Runner row exists and heartbeats, and the
- *                                   operator selects it on the cluster's AI
- *                                   page when they want it back.
+ * left_unbound_by_operator:         no Runner is bound now, but one was
+ *                                   before or AI already ran kubectl on the
+ *                                   cluster — an operator cleared the
+ *                                   binding, the bound Runner row was deleted
+ *                                   (the foreign key nulls it), or the
+ *                                   commands came through the cluster's
+ *                                   Kubernetes AI agent. Registration does not
+ *                                   re-bind or flip any switch; the Runner
+ *                                   row exists and heartbeats, and binding it
+ *                                   again (through the API) needs the admin
+ *                                   permissions. The Kubernetes AI agent is
+ *                                   the way back.
  */
 export type KubernetesAgentRunnerBindingState =
   | "first_bind"
@@ -455,6 +603,33 @@ export const RUNNER_POD_NAMESPACE_ENV: string =
 export const KUBECTL_ALLOW_NODE_OPERATIONS_ENV: string =
   "ONEUPTIME_KUBECTL_ALLOW_NODE_OPERATIONS";
 
+/*
+ * The Kubernetes AI agent (image KUBERNETES_AI_AGENT_IMAGE_REPOSITORY) is
+ * the chart's in-cluster kubectl executor from this release on. It reads
+ * KUBECTL_ALLOW_WRITES_ENV, KUBECTL_WRITE_NAMESPACES_ENV and
+ * KUBECTL_ALLOW_NODE_OPERATIONS_ENV with exactly the meanings above (the
+ * chart fills them from aiAgent.remediation.*), plus its own pod namespace
+ * from AI_AGENT_POD_NAMESPACE_ENV (downward API) — the namespace it never
+ * writes into, reported as posture.podNamespace.
+ */
+export const AI_AGENT_POD_NAMESPACE_ENV: string =
+  "ONEUPTIME_AI_AGENT_POD_NAMESPACE";
+
+// The chart labels the agent's Deployment and pod `component: ai-agent`.
+export const KUBERNETES_AI_AGENT_COMPONENT: string = "ai-agent";
+
+export const KUBERNETES_AI_AGENT_IMAGE_REPOSITORY: string =
+  "oneuptime/kubernetes-ai-agent";
+
+// How the agent is named wherever a Runner name would otherwise appear.
+export const KUBERNETES_AI_AGENT_DISPLAY_NAME: string = "Kubernetes AI agent";
+
+/*
+ * The agent heartbeats every 30 seconds; it counts as online while its last
+ * heartbeat is at most this old (the same window a Runner gets).
+ */
+export const KUBERNETES_AI_AGENT_ALIVE_WINDOW_IN_MINUTES: number = 5;
+
 /*
  * Why POST /runner-ingest/register-kubernetes-agent refused a registration
  * (HTTP 403). The server sends it as `reason` in the JSON error body so the
@@ -474,15 +649,71 @@ export const KUBECTL_ALLOW_NODE_OPERATIONS_ENV: string =
  * runner_belongs_to_another_cluster: the row this cluster's agent Runner
  *                                    name resolves to is another cluster's.
  *                                    Needs an operator.
+ * superseded_by_ai_agent:            this cluster's Kubernetes AI agent is
+ *                                    online, and it replaces the in-cluster
+ *                                    Runner. Refused only WHILE the agent is
+ *                                    online, so it clears on its own when the
+ *                                    agent stops (a helm rollback); the body
+ *                                    carries retryAfterSeconds.
  */
 export type KubernetesAgentRegistrationRefusalReason =
   | "previous_instance_online"
   | "runner_holds_more_than_defaults"
-  | "runner_belongs_to_another_cluster";
+  | "runner_belongs_to_another_cluster"
+  | "superseded_by_ai_agent";
+
+// The legacy Runner refusals that clear on their own.
+export const TRANSIENT_KUBERNETES_AGENT_REGISTRATION_REFUSALS: ReadonlyArray<KubernetesAgentRegistrationRefusalReason> =
+  ["previous_instance_online", "superseded_by_ai_agent"];
 
 // Refusals that clear without anyone doing anything; retrying is right.
 export function isTransientKubernetesAgentRegistrationRefusal(
   reason: unknown,
 ): boolean {
-  return reason === "previous_instance_online";
+  return (
+    typeof reason === "string" &&
+    (
+      TRANSIENT_KUBERNETES_AGENT_REGISTRATION_REFUSALS as ReadonlyArray<string>
+    ).includes(reason)
+  );
+}
+
+/*
+ * Why POST /kubernetes-ai-agent-ingest/register refused the Kubernetes AI
+ * agent (HTTP 403, `reason` in the JSON body; transient ones also carry
+ * retryAfterSeconds and a Retry-After header).
+ *
+ * previous_instance_online: this cluster's agent row is online and the
+ *                           request did not present its current key (a
+ *                           second install with the same clusterName, or a
+ *                           pod replaced without a clean shutdown). Clears
+ *                           on its own once the old instance goes quiet.
+ * legacy_runner_online:     this cluster's previous in-cluster Runner is
+ *                           still online — for a few seconds during a helm
+ *                           upgrade, while the old pod terminates. Clears on
+ *                           its own.
+ * cluster_name_invalid:     the clusterName is empty or too long. Needs an
+ *                           operator (fix the chart's clusterName).
+ * agent_cap_reached:        the project hit the agent row cap or the hourly
+ *                           new-agent brake. Needs an operator (or time).
+ */
+export type KubernetesAiAgentRegistrationRefusalReason =
+  | "previous_instance_online"
+  | "legacy_runner_online"
+  | "cluster_name_invalid"
+  | "agent_cap_reached";
+
+export const TRANSIENT_KUBERNETES_AI_AGENT_REGISTRATION_REFUSALS: ReadonlyArray<KubernetesAiAgentRegistrationRefusalReason> =
+  ["previous_instance_online", "legacy_runner_online"];
+
+// Agent refusals that clear without anyone doing anything; retrying is right.
+export function isTransientKubernetesAiAgentRegistrationRefusal(
+  reason: unknown,
+): boolean {
+  return (
+    typeof reason === "string" &&
+    (
+      TRANSIENT_KUBERNETES_AI_AGENT_REGISTRATION_REFUSALS as ReadonlyArray<string>
+    ).includes(reason)
+  );
 }

package/Types/Kubernetes/KubernetesClusterAiAccessPermissions.ts

@@ -8,16 +8,20 @@ import Permission, { PermissionHelper } from "../Permission";
  */
 
 /*
- * Who may LOOSEN a cluster's AI access: switch remediation to an unattended
- * mode (Automatic or Bypass approval), author a kubectl allowlist, or bind
- * the Runner / Kubernetes credential kubectl runs through. A cluster's AI
- * mode does the job of a FullAuto AutoRemediationRule without a rule row, so
- * it takes the same set AutoRemediationRule uses for executionMode,
- * commandAllowlist and commandRunners.
+ * Who may LOOSEN a cluster's AI access: turn AI fixes on (any move up from
+ * Off, Ask for approval included) or give them more autonomy (Automatic,
+ * Bypass approval), author a kubectl allowlist, bind the Runner / Kubernetes
+ * credential kubectl runs through, or clear that binding while the cluster
+ * has a Kubernetes AI agent (which hands the cluster to the agent), and
+ * reset the cluster's Kubernetes AI agent. A cluster's AI mode does the job
+ * of a FullAuto AutoRemediationRule without a rule row, so it takes the
+ * same set AutoRemediationRule uses for executionMode, commandAllowlist and
+ * commandRunners.
  *
- * Tightening (Off or Ask for approval, clearing the allowlist or the
- * binding) stays open to anyone who may edit the cluster: making AI do less
- * never needs more privilege than the cluster itself.
+ * Tightening (Off, moving down, clearing the allowlist, or clearing the
+ * binding of a cluster with no AI agent) stays open to anyone who may edit
+ * the cluster: making AI do less never needs more privilege than the
+ * cluster itself.
  */
 export const KUBERNETES_AI_ACCESS_ADMIN_PERMISSIONS: Array<Permission> = [
   Permission.ProjectOwner,
@@ -37,18 +41,18 @@ export const KUBERNETES_AI_ACCESS_CREDENTIAL_PERMISSIONS: Array<Permission> = [
 ];
 
 /*
- * Why deleting an in-cluster Runner is a two-step remedy. The binding's
- * foreign key is ON DELETE SET NULL, so a deleted Runner's clusters are
- * left with no Runner bound, and registration never binds a cluster that
- * had one (left_unbound_by_operator): the fresh Runner the agent registers
- * is used only once someone selects it on the cluster's AI page. Selecting
- * a Runner loosens AI access, so it needs one of these permissions — which
+ * What deleting a previous in-cluster Runner does. The binding's foreign
+ * key is ON DELETE SET NULL, so a deleted Runner's clusters are left with
+ * no Runner bound, and registration never binds a cluster that had one
+ * (left_unbound_by_operator): such a cluster is then reached through its
+ * Kubernetes AI agent, which the current chart installs. Binding a Runner
+ * again loosens AI access, so it needs one of these permissions — which
  * whoever deleted the Runner may not hold. Here, not in
  * KubernetesClusterAiAccessService, so RunnerService (which that service
  * imports) can name it too without an import cycle.
  */
 export function getDeletedAgentRunnerRebindNote(): string {
-  return `Deleting a Runner leaves the clusters it was bound to with no Runner bound, and a registering Runner never binds a cluster that had one. Selecting a Runner needs one of these permissions: ${PermissionHelper.getPermissionTitles(
+  return `Deleting a Runner leaves the clusters it was bound to with no Runner bound; they then use their Kubernetes AI agent (upgrade the Kubernetes agent chart to install it). A registering Runner never binds a cluster that had one, and binding a Runner again needs one of these permissions: ${PermissionHelper.getPermissionTitles(
     KUBERNETES_AI_ACCESS_ADMIN_PERMISSIONS,
   ).join(", ")}.`;
 }

package/Types/Kubernetes/KubernetesClusterAiInsights.ts

@@ -0,0 +1,81 @@
+/*
+ * What OneUptime AI investigated and changed on one Kubernetes cluster: the
+ * response of POST /api/kubernetes-cluster/ai-access/insights, which the
+ * cluster's AI Insights page renders.
+ *
+ * Summaries only. Nothing here carries kubectl output, a prompt or a
+ * command plan: the route is readable by everyone who may read the
+ * cluster, a wider audience than the AI runs, suggestions and Runner jobs
+ * it summarises.
+ */
+
+// How many investigations, and how many fixes, the route returns (newest).
+export const KUBERNETES_CLUSTER_AI_INSIGHTS_LIMIT: number = 25;
+
+// The window commandCounts covers.
+export const KUBERNETES_CLUSTER_AI_INSIGHTS_COMMAND_WINDOW_IN_DAYS: number = 30;
+
+// How much of a fix's rationale the route returns.
+export const KUBERNETES_CLUSTER_AI_INSIGHTS_RATIONALE_MAX_LENGTH: number = 300;
+
+/*
+ * One investigation that concerns the cluster: it ran kubectl on it, or it
+ * investigated an incident or alert linked to it.
+ */
+export interface KubernetesClusterAiInsightInvestigation {
+  aiRunId: string;
+  // AIRunStatus.
+  status?: string | undefined;
+  analysisTldr?: string | undefined;
+  // ISO dates.
+  createdAt?: string | undefined;
+  completedAt?: string | undefined;
+  incident?:
+    | {
+        id: string;
+        title?: string | undefined;
+        number?: number | undefined;
+      }
+    | undefined;
+  alert?:
+    | {
+        id: string;
+        title?: string | undefined;
+      }
+    | undefined;
+}
+
+/*
+ * One fix AI proposed or ran on the cluster: a round the cluster's Fixes
+ * setting produced, or any suggestion whose kubectl ran on the cluster.
+ */
+export interface KubernetesClusterAiInsightFix {
+  id: string;
+  // AutoRemediationSuggestionStatus.
+  status?: string | undefined;
+  // AutoRemediationExecutionMode.
+  executionMode?: string | undefined;
+  // AutoRemediationSuggestionType.
+  suggestionType?: string | undefined;
+  // The first KUBERNETES_CLUSTER_AI_INSIGHTS_RATIONALE_MAX_LENGTH characters.
+  rationale?: string | undefined;
+  createdAt?: string | undefined;
+  approvedAt?: string | undefined;
+  incidentId?: string | undefined;
+  alertId?: string | undefined;
+}
+
+export interface KubernetesClusterAiInsights {
+  clusterId: string;
+  investigations: Array<KubernetesClusterAiInsightInvestigation>;
+  fixes: Array<KubernetesClusterAiInsightFix>;
+  /*
+   * kubectl commands AI ran on the cluster in the last
+   * KUBERNETES_CLUSTER_AI_INSIGHTS_COMMAND_WINDOW_IN_DAYS days, by kind.
+   * The AI agent page's "Test connection" checks are not counted.
+   */
+  commandCounts: {
+    investigation: number;
+    remediation: number;
+  };
+}

package/Types/Monitor/ProxmoxAlertTemplates.ts

@@ -354,7 +354,7 @@ const nodeOfflineTemplate: ProxmoxAlertTemplate = {
   id: "pve-node-offline",
   name: "Node Offline",
   description:
-    "Alert when any Proxmox node reports as down (pve_up = 0, scoped to nodes via pve.scope). One incident per node.",
+    "Alert when any Proxmox node is down (pve_up = 0, scoped to nodes via pve.scope). One incident per node. On the Proxmox VE native OpenTelemetry push, a node that stops reporting is reported down by the nodes still alive, so this fires about 5 minutes after its last report; a standalone host or a whole cluster going silent turns the cluster Disconnected instead.",
   category: "Availability",
   severity: "Critical",
   getMonitorStep: (args: ProxmoxAlertTemplateArgs): MonitorStep => {
@@ -383,7 +383,7 @@ const nodeOfflineTemplate: ProxmoxAlertTemplate = {
         filterType: FilterType.LessThan,
         value: 1,
         incidentTitle: `[Proxmox] Node Offline - ${args.monitorName}`,
-        incidentDescription: `A Proxmox node is reporting as down (pve_up = 0). The node is unreachable or has crashed and every guest running on it may be offline. Check the root cause for the affected node id, verify the node's power/network state, and check whether HA has relocated its guests.`,
+        incidentDescription: `A Proxmox node is down (pve_up = 0): the cluster reports it offline or, on the Proxmox VE native OpenTelemetry push, it has stopped reporting and the nodes still alive report it (those points carry oneuptime.proxmox.inferred = not-reporting). The node may have crashed or lost power or network, and every guest running on it may be offline; a node that is up but whose pvestatd or pmxcfs has stopped, or that cannot reach OneUptime, is reported the same way. Check the root cause for the affected node id, verify the node's power/network state and its pvestatd service, and check whether HA has relocated its guests. If you took the node out of the cluster on purpose, use Remove Node on its page in OneUptime to resolve this.`,
         criteriaName: "Node Offline - pve_up < 1",
         criteriaDescription:
           "Triggers when any node reports pve_up below 1 over the monitoring window.",
@@ -525,7 +525,7 @@ const quorumRiskTemplate: ProxmoxAlertTemplate = {
   id: "pve-quorum-risk",
   name: "Cluster Quorum at Risk",
   description:
-    "Alert when 50% or fewer of the cluster's nodes are online. Derived from node visibility (online pve_up nodes ÷ pve_node_info node count) — pve-exporter exposes no corosync metric, so this is the honest quorum proxy: at ≤50% node availability the cluster has lost (or is about to lose) quorum and HA recovery stops.",
+    "Alert when 50% or fewer of the cluster's nodes are online. Derived from node visibility (online pve_up nodes ÷ pve_node_info node count) — pve-exporter exposes no corosync metric, so this is the honest quorum proxy: at ≤50% node availability the cluster has lost (or is about to lose) quorum and HA recovery stops. On the Proxmox VE native OpenTelemetry push, a node that stops reporting counts as offline (the nodes still alive report it); a whole cluster going silent turns the cluster Disconnected instead.",
   category: "Availability",
   severity: "Critical",
   getMonitorStep: (args: ProxmoxAlertTemplateArgs): MonitorStep => {
@@ -557,7 +557,7 @@ const quorumRiskTemplate: ProxmoxAlertTemplate = {
         filterType: FilterType.LessThanOrEqualTo,
         value: 50,
         incidentTitle: `[Proxmox] CRITICAL: Cluster Quorum at Risk - ${args.monitorName}`,
-        incidentDescription: `Half or more of the Proxmox cluster's nodes are offline. With ≤50% of nodes online the cluster has lost (or is about to lose) corosync quorum — pmxcfs goes read-only, guests cannot be started or migrated, and HA recovery stops. Identify and recover the offline nodes immediately. (This is derived from node visibility: online nodes ÷ total nodes; pve-exporter exposes no direct corosync metric.)`,
+        incidentDescription: `Half or more of the Proxmox cluster's nodes are offline. With ≤50% of nodes online the cluster has lost (or is about to lose) corosync quorum — pmxcfs goes read-only, guests cannot be started or migrated, and HA recovery stops. Identify and recover the offline nodes immediately. (This is derived from node visibility: online nodes ÷ total nodes; pve-exporter exposes no direct corosync metric. On the Proxmox VE native OpenTelemetry push, a node counts as offline once it has stopped reporting, as reported by the nodes still alive — so a node that is up but cannot reach OneUptime counts too.)`,
         criteriaName: "Quorum Risk - Node Availability <= 50%",
         criteriaDescription:
           "Triggers when 50% or fewer of the cluster's nodes report as online.",