@oneuptime/common 14.0.8 → 14.0.9

package/Server/Services/KubernetesClusterService.ts

@@ -30,10 +30,7 @@ import { holdsAnyPermission } from "../Utils/Runbook/RunbookExecutePermission";
 import DatabaseCommonInteractionProps from "../../Types/BaseDatabase/DatabaseCommonInteractionProps";
 import BadDataException from "../../Types/Exception/BadDataException";
 import NotAuthorizedException from "../../Types/Exception/NotAuthorizedException";
-import {
-  KubernetesAiRemediationMode,
-  isUnattendedRemediationMode,
-} from "../../Types/Kubernetes/KubernetesClusterAiAccess";
+import { KubernetesAiRemediationMode } from "../../Types/Kubernetes/KubernetesClusterAiAccess";
 import {
   KUBERNETES_AI_ACCESS_ADMIN_PERMISSIONS,
   KUBERNETES_AI_ACCESS_CREDENTIAL_PERMISSIONS,
@@ -41,6 +38,7 @@ import {
 import { PermissionHelper } from "../../Types/Permission";
 import RunbookCredentialType from "../../Types/Runbook/RunbookCredentialType";
 import KubernetesClusterFeedService from "./KubernetesClusterFeedService";
+import KubernetesAiAgentService from "./KubernetesAiAgentService";
 import { KubernetesClusterFeedEventType } from "../../Models/DatabaseModels/KubernetesClusterFeed";
 import ResourceFeedUtil from "../Utils/ResourceFeed/ResourceFeedUtil";
 import { Blue500, Gray500, Green500, Yellow500 } from "../../Types/BrandColors";
@@ -151,11 +149,11 @@ export const AI_ACCESS_CREDENTIAL_REFUSAL: string =
   "Credential not found, or it is not a Kubernetes credential in this project. The AI access credential must be a Kubernetes credential (API server URL and ServiceAccount token) that belongs to this project.";
 
 export function getAiAccessAdminRefusal(): string {
-  return `You need one of these permissions to let OneUptime AI do more on a Kubernetes cluster (switch AI remediation to Automatic or Bypass approval, add kubectl allowlist patterns, or bind a Runner or credential): ${PermissionHelper.getPermissionTitles(
+  return `You need one of these permissions to let OneUptime AI do more on a Kubernetes cluster (turn AI fixes on or give them more autonomy, add kubectl allowlist patterns, bind a Runner or credential, or clear one while the cluster has a Kubernetes AI agent): ${PermissionHelper.getPermissionTitles(
     KUBERNETES_AI_ACCESS_ADMIN_PERMISSIONS,
   ).join(
     ", ",
-  )}. Anyone who may edit the cluster can still turn AI remediation off or back to Ask for approval, remove allowlist patterns, and clear the Runner or credential.`;
+  )}. Anyone who may edit the cluster can still turn AI fixes off or down to Ask for approval, and remove allowlist patterns.`;
 }
 
 export function getAiAccessCredentialRefusal(): string {
@@ -183,11 +181,22 @@ export interface AiAccessSettingsSnapshot {
   aiKubectlCommandAllowlist: Array<string>;
   aiAccessRunnerId: string | null;
   aiAccessCredentialId: string | null;
+  /*
+   * Whether the cluster has a Kubernetes AI agent row. Read only for a
+   * write that clears the Runner or credential binding (absent otherwise):
+   * with an agent, clearing an advanced Runner hands the cluster to the
+   * agent, whose RBAC may reach more, so it is a loosening.
+   */
+  hasKubernetesAiAgent?: boolean | undefined;
 }
 
-// What a cluster that was never AI-configured has: the column defaults.
+/*
+ * What a cluster that was never AI-configured has: the column defaults
+ * (investigation on — it only ever runs read-only kubectl — and remediation
+ * Disabled).
+ */
 const NEVER_CONFIGURED_AI_ACCESS: AiAccessSettingsSnapshot = {
-  isAiInvestigationEnabled: false,
+  isAiInvestigationEnabled: true,
   aiRemediationMode: KubernetesAiRemediationMode.Disabled,
   aiKubectlCommandAllowlist: [],
   aiAccessRunnerId: null,
@@ -229,7 +238,7 @@ function getClusterDeleteCarryForward(
 }
 
 // What loosening an operator's write would do to one cluster.
-interface AiAccessLoosening {
+export interface AiAccessLoosening {
   loosens: boolean;
   bindsCredential: boolean;
 }
@@ -319,7 +328,9 @@ export class Service extends DatabaseService<Model> {
 
     if (!updateBy.props.isRoot && this.isAiAccessSettingWritten(data)) {
       const previousAiAccessSettings: Record<string, AiAccessSettingsSnapshot> =
-        await this.getAiAccessSettingsForUpdateQuery(updateBy);
+        await this.getAiAccessSettingsForUpdateQuery(updateBy, {
+          withKubernetesAiAgent: this.clearsAiAccessBinding(data),
+        });
 
       if (!updateBy.props.isMasterAdmin) {
         this.assertMayChangeAiAccess({
@@ -657,22 +668,29 @@ export class Service extends DatabaseService<Model> {
 
   /*
    * Does this write let AI do more on a cluster whose settings are
-   * `current`? Only a real change counts: the AI page posts every field of
-   * its form, so an editor who only flips the investigation switch re-posts
-   * an unchanged mode, allowlist and binding, and must not be refused for
-   * settings someone else chose.
+   * `current`? Only a real change counts: the AI agent page posts every
+   * field of its form, so an editor who only flips the investigation switch
+   * re-posts an unchanged mode, allowlist and binding, and must not be
+   * refused for settings someone else chose.
    *
-   * - mode: moving UP to Automatic or Bypass approval (Bypass approval ->
-   *   Automatic is a tightening);
+   * - mode: ANY move up — turning fixes on from Off (even to Ask for
+   *   approval: with the chart's write RBAC in place that is the step that
+   *   lets AI change the cluster at all, and no project opt-in stands behind
+   *   it any more for a cluster reached through its agent), or up to
+   *   Automatic or Bypass approval. Moving down (Bypass approval ->
+   *   Automatic, anything -> Off) tightens;
    * - allowlist: adding a pattern the cluster did not already have
    *   (removing patterns, or clearing the list, tightens);
-   * - Runner or credential: binding one other than the current one
-   *   (clearing tightens).
+   * - Runner or credential: binding one other than the current one; and
+   *   clearing one while the cluster has a Kubernetes AI agent, which hands
+   *   the cluster to the agent (resolveKubernetesAiAccessTarget) and its
+   *   RBAC. Without an agent, clearing leaves AI nothing to reach the
+   *   cluster through, and tightens.
    *
    * `data` has been through validateAiRemediationSettings, so the mode is a
    * known value and the allowlist a trimmed array or null.
    */
-  private getAiAccessLoosening(
+  public getAiAccessLoosening(
     data: JSONObject,
     current: AiAccessSettingsSnapshot,
   ): AiAccessLoosening {
@@ -685,7 +703,6 @@ export class Service extends DatabaseService<Model> {
 
     if (
       mode !== undefined &&
-      isUnattendedRemediationMode(mode) &&
       REMEDIATION_MODES_BY_AUTONOMY.indexOf(mode) >
         REMEDIATION_MODES_BY_AUTONOMY.indexOf(current.aiRemediationMode)
     ) {
@@ -729,9 +746,42 @@ export class Service extends DatabaseService<Model> {
       bindsCredential = true;
     }
 
+    if (current.hasKubernetesAiAgent === true) {
+      const clearsRunner: boolean =
+        isAnyKeyWritten(data, AI_ACCESS_RUNNER_KEYS) &&
+        !runnerId &&
+        current.aiAccessRunnerId !== null;
+      const clearsCredential: boolean =
+        isAnyKeyWritten(data, AI_ACCESS_CREDENTIAL_KEYS) &&
+        !credentialId &&
+        current.aiAccessCredentialId !== null;
+
+      if (clearsRunner || clearsCredential) {
+        loosens = true;
+      }
+    }
+
     return { loosens, bindsCredential };
   }
 
+  // The write clears the Runner or the credential binding (null).
+  private clearsAiAccessBinding(data: JSONObject): boolean {
+    return (
+      (isAnyKeyWritten(data, AI_ACCESS_RUNNER_KEYS) &&
+        !RelationIdUtil.readConsistent(
+          data,
+          AI_ACCESS_RUNNER_KEYS,
+          "AI access Runner",
+        )) ||
+      (isAnyKeyWritten(data, AI_ACCESS_CREDENTIAL_KEYS) &&
+        !RelationIdUtil.readConsistent(
+          data,
+          AI_ACCESS_CREDENTIAL_KEYS,
+          "AI access credential",
+        ))
+    );
+  }
+
   /*
    * The AI access settings of every cluster an operator's update reaches,
    * read before the write. onBeforeUpdate runs before the framework scopes
@@ -742,6 +792,10 @@ export class Service extends DatabaseService<Model> {
   @CaptureSpan()
   private async getAiAccessSettingsForUpdateQuery(
     updateBy: UpdateBy<Model>,
+    options: {
+      // Also read whether each cluster has a Kubernetes AI agent row.
+      withKubernetesAiAgent: boolean;
+    } = { withKubernetesAiAgent: false },
   ): Promise<Record<string, AiAccessSettingsSnapshot>> {
     const clusters: Array<Model> = await this.findBy({
       query: {
@@ -790,9 +844,63 @@ export class Service extends DatabaseService<Model> {
       };
     }
 
+    if (options.withKubernetesAiAgent) {
+      await this.readKubernetesAiAgentPresence(settings);
+    }
+
     return settings;
   }
 
+  /*
+   * Fill in hasKubernetesAiAgent on each snapshot, one query per project the
+   * write reaches. Fails closed: when the agent rows cannot be read, every
+   * cluster is treated as having one, so clearing a binding then needs the
+   * admin permissions rather than slipping through unchecked.
+   */
+  private async readKubernetesAiAgentPresence(
+    settings: Record<string, AiAccessSettingsSnapshot>,
+  ): Promise<void> {
+    const clusterIdsByProject: Map<string, Array<ObjectID>> = new Map<
+      string,
+      Array<ObjectID>
+    >();
+
+    for (const [clusterId, snapshot] of Object.entries(settings)) {
+      const projectId: string = snapshot.projectId?.toString() || "";
+      const clusterIds: Array<ObjectID> =
+        clusterIdsByProject.get(projectId) || [];
+      clusterIds.push(new ObjectID(clusterId));
+      clusterIdsByProject.set(projectId, clusterIds);
+    }
+
+    for (const [projectId, clusterIds] of clusterIdsByProject.entries()) {
+      let agentClusterIds: Set<string> | null = null;
+
+      if (projectId) {
+        try {
+          agentClusterIds = new Set<string>(
+            (
+              await KubernetesAiAgentService.findForClusters({
+                projectId: new ObjectID(projectId),
+                kubernetesClusterIds: clusterIds,
+              })
+            ).keys(),
+          );
+        } catch (error) {
+          logger.error(
+            `KubernetesClusterService: could not read the Kubernetes AI agents of project ${projectId}; treating every cluster as having one: ${error}`,
+          );
+        }
+      }
+
+      for (const clusterId of clusterIds) {
+        settings[clusterId.toString()]!.hasKubernetesAiAgent = agentClusterIds
+          ? agentClusterIds.has(clusterId.toString())
+          : true;
+      }
+    }
+  }
+
   /*
    * aiAccessRunnerId / aiAccessCredentialId are editable through ordinary
    * CRUD by every member who may edit a cluster, and nothing in the
@@ -977,7 +1085,7 @@ export class Service extends DatabaseService<Model> {
 
       if (isAgentRunner && !isThisClustersAgent && isRunnerChanged) {
         throw new BadDataException(
-          `Runner "${runner.name}" is the in-cluster Runner the Kubernetes agent chart installed on another cluster. Its ServiceAccount reaches only the cluster its pod runs in, and it is never given a credential, so it cannot run kubectl for this cluster. Install the in-cluster Runner on this cluster with --set aiAccess.enabled=true, or bind a Runner you created under Project Settings → Runners together with a Kubernetes credential for this cluster.`,
+          `Runner "${runner.name}" is the in-cluster Runner the Kubernetes agent chart installed on another cluster. Its ServiceAccount reaches only the cluster its pod runs in, and it is never given a credential, so it cannot run kubectl for this cluster. Leave the Runner empty to use this cluster's Kubernetes AI agent, or bind a Runner you created under Project Settings → Runners together with a Kubernetes credential for this cluster.`,
         );
       }
 

package/Server/Services/ProjectService.ts

@@ -554,9 +554,41 @@ export class ProjectService extends DatabaseService<Model> {
       data.data.alertEpisodeNumberPrefix = "AE-";
     }
 
+    this.applyNewProjectAiDefaults(data.data);
+
     return Promise.resolve({ createBy: data, carryForward: null });
   }
 
+  /*
+   * A new project starts with AI investigating its incidents and alerts:
+   * both automatic-investigation opt-ins are turned on unless the create
+   * request set them itself (a request that says false keeps false). The
+   * columns' database default stays false, so projects that existed before
+   * this are never switched on by an upgrade, and a create that bypasses
+   * this hook entirely gets the old behaviour rather than a surprise.
+   *
+   * Postmortem drafting is deliberately NOT turned on here: it is its own
+   * opt-in (enableAutomaticPostmortemDraft), off by default.
+   */
+  public applyNewProjectAiDefaults(data: {
+    enableAutomaticIncidentInvestigation?: boolean | null | undefined;
+    enableAutomaticAlertInvestigation?: boolean | null | undefined;
+  }): void {
+    if (
+      data.enableAutomaticIncidentInvestigation === undefined ||
+      data.enableAutomaticIncidentInvestigation === null
+    ) {
+      data.enableAutomaticIncidentInvestigation = true;
+    }
+
+    if (
+      data.enableAutomaticAlertInvestigation === undefined ||
+      data.enableAutomaticAlertInvestigation === null
+    ) {
+      data.enableAutomaticAlertInvestigation = true;
+    }
+  }
+
   /*
    * Session replay is gated on the project's org-wide allow flag, and the
    * ingest gate caches the resolved policy per pod. Turning the flag off has

package/Server/Services/ProxmoxResourceService.ts

@@ -7,6 +7,18 @@ import OneUptimeDate from "../../Types/Date";
 import QueryHelper from "../Types/Database/QueryHelper";
 import { truncateShortText } from "../Utils/Database/TruncateColumnValue";
 import logger from "../Utils/Logger";
+import {
+  ProxmoxRosterNode,
+  isProxmoxSilentNodeDetectionEnabled,
+} from "../Utils/Telemetry/ProxmoxNativeNodeLiveness";
+
+const NODE_ID_PREFIX: string = "node/";
+
+export type ProxmoxRemoveNodeResult =
+  | "removed"
+  | "not-found"
+  | "not-native"
+  | "still-reporting";
 
 /*
  * ------------------------------------------------------------------
@@ -88,6 +100,7 @@ const UPSERT_COLUMNS: Array<string> = [
   "isBackedUp",
   "uptimeSeconds",
   "lastSeenAt",
+  "isNativePush",
   "version",
 ];
 
@@ -127,12 +140,17 @@ export class Service extends DatabaseService<Model> {
    * a complete scrape, but a batch that happens to lack an info series
    * (e.g. only pve_up made it through a pipeline filter) must not blank
    * name/vmid/haState that an earlier batch already filled.
+   *
+   * isNativePush records where the batch came from — the Proxmox VE
+   * native push (true) or the agent (false) — and follows the latest
+   * observation, so a cluster moved from one to the other follows too.
    */
   @CaptureSpan()
   public async bulkUpsert(data: {
     projectId: ObjectID;
     proxmoxClusterId: ObjectID;
     resources: Array<ParsedProxmoxResource>;
+    isNativePush?: boolean | undefined;
   }): Promise<void> {
     if (data.resources.length === 0) {
       return;
@@ -183,6 +201,7 @@ export class Service extends DatabaseService<Model> {
           r.isBackedUp,
           r.uptimeSeconds !== null ? Math.trunc(r.uptimeSeconds) : null,
           r.lastSeenAt,
+          Boolean(data.isNativePush),
           0, // version (BaseModel @VersionColumn)
         );
       }
@@ -192,7 +211,7 @@ export class Service extends DatabaseService<Model> {
           "projectId", "proxmoxClusterId", "kind", "externalId",
           "name", "vmid", "guestType", "parentNodeName",
           "isUp", "haState", "onboot", "isBackedUp", "uptimeSeconds",
-          "lastSeenAt", "version"
+          "lastSeenAt", "isNativePush", "version"
         )
         VALUES ${valueFragments.join(", ")}
         ON CONFLICT ("projectId", "proxmoxClusterId", "kind", "externalId")
@@ -207,6 +226,8 @@ export class Service extends DatabaseService<Model> {
           "isBackedUp" = COALESCE(EXCLUDED."isBackedUp", "ProxmoxResource"."isBackedUp"),
           "uptimeSeconds" = COALESCE(EXCLUDED."uptimeSeconds", "ProxmoxResource"."uptimeSeconds"),
           "lastSeenAt" = EXCLUDED."lastSeenAt",
+          "isNativePush" = EXCLUDED."isNativePush",
+          "notReportingMarkedAt" = NULL,
           "updatedAt" = now()
         WHERE EXCLUDED."lastSeenAt" >= "ProxmoxResource"."lastSeenAt"
       `;
@@ -313,17 +334,48 @@ export class Service extends DatabaseService<Model> {
    * than olderThan. Returns the number of deleted rows. Only called by
    * the cleanup worker for clusters that are still connected — a
    * disconnected cluster keeps its last-known inventory.
+   *
+   * Exception: a Node that reports itself over the Proxmox VE native push
+   * (isNativePush) is kept until it has been silent for longer than the
+   * retention window, or is removed (removeOfflineNode). Those pushes
+   * never say which nodes the cluster has — the Node rows ARE the
+   * cluster's membership, the roster the nodes still alive report the
+   * silent ones from (getNodeRoster uses the same cutoff). Pruning a
+   * silent native node at the normal cutoff would drop it from the
+   * roster: it would never be reported down, its Node Offline alert
+   * would resolve, and Quorum at Risk would stop counting it. That
+   * holds whether or not it has been marked Offline yet — a node that
+   * died just before or during a OneUptime outage is only marked once
+   * the nodes have pushed again for two minutes after it. With silent-node
+   * detection switched off (PVE_NATIVE_NODE_SILENCE_DETECTION=false)
+   * nothing would ever mark such a node, so the keep is off too and every
+   * row ages out as before.
    */
   @CaptureSpan()
   public async deleteStaleForCluster(data: {
     proxmoxClusterId: ObjectID;
     olderThan: Date;
+    now?: Date | undefined;
   }): Promise<number> {
     const result: Array<{ affected?: number }> | { affected?: number } =
-      await this.getRepository().manager.query(
-        `DELETE FROM "ProxmoxResource" WHERE "proxmoxClusterId" = $1 AND "lastSeenAt" < $2`,
-        [data.proxmoxClusterId.toString(), data.olderThan],
-      );
+      isProxmoxSilentNodeDetectionEnabled()
+        ? await this.getRepository().manager.query(
+            `DELETE FROM "ProxmoxResource"
+             WHERE "proxmoxClusterId" = $1
+               AND "lastSeenAt" < $2
+               AND NOT ("kind" = 'Node'
+                        AND "isNativePush" IS TRUE
+                        AND "lastSeenAt" >= $3)`,
+            [
+              data.proxmoxClusterId.toString(),
+              data.olderThan,
+              this.getSilentNodeRetentionCutoff(data.now),
+            ],
+          )
+        : await this.getRepository().manager.query(
+            `DELETE FROM "ProxmoxResource" WHERE "proxmoxClusterId" = $1 AND "lastSeenAt" < $2`,
+            [data.proxmoxClusterId.toString(), data.olderThan],
+          );
 
     // Postgres driver returns [rows, affected] for DELETE — normalize.
     let affected: number = 0;
@@ -343,6 +395,271 @@ export class Service extends DatabaseService<Model> {
     return affected;
   }
 
+  /**
+   * The cluster's nodes, for deciding which ones have stopped reporting
+   * (ProxmoxNativeNodeLiveness). A node silent for longer than the
+   * retention window drops out, so a node removed from the cluster stops
+   * being reported after at most that long even if nobody removes it.
+   */
+  @CaptureSpan()
+  public async getNodeRoster(data: {
+    projectId: ObjectID;
+    proxmoxClusterId: ObjectID;
+    now?: Date | undefined;
+  }): Promise<Array<ProxmoxRosterNode>> {
+    const seenSince: Date = this.getSilentNodeRetentionCutoff(data.now);
+    const rows: Array<{
+      externalId: string;
+      lastSeenAt: Date | string;
+      isUp: boolean | null;
+      notReportingMarkedAt: Date | string | null;
+    }> = await this.getRepository().manager.query(
+      `SELECT "externalId", "lastSeenAt", "isUp", "notReportingMarkedAt" FROM "ProxmoxResource"
+         WHERE "projectId" = $1 AND "proxmoxClusterId" = $2
+           AND "kind" = 'Node' AND "deletedAt" IS NULL
+           AND "lastSeenAt" >= $3`,
+      [data.projectId.toString(), data.proxmoxClusterId.toString(), seenSince],
+    );
+
+    const roster: Array<ProxmoxRosterNode> = [];
+    for (const row of rows) {
+      if (!row.externalId || !row.externalId.startsWith(NODE_ID_PREFIX)) {
+        continue;
+      }
+      const nodeName: string = row.externalId.substring(NODE_ID_PREFIX.length);
+      const lastSeenAt: Date = new Date(row.lastSeenAt);
+      if (!nodeName || isNaN(lastSeenAt.getTime())) {
+        continue;
+      }
+      const markedAt: Date | null =
+        row.notReportingMarkedAt instanceof Date ||
+        typeof row.notReportingMarkedAt === "string"
+          ? new Date(row.notReportingMarkedAt)
+          : null;
+      roster.push({
+        nodeName,
+        lastSeenAt,
+        isUp: typeof row.isUp === "boolean" ? row.isUp : null,
+        notReportingMarkedAt:
+          markedAt && !isNaN(markedAt.getTime()) ? markedAt : null,
+      });
+    }
+    return roster;
+  }
+
+  /**
+   * Mark nodes that their live siblings report as not reporting
+   * (Proxmox VE native push) as Offline: isUp = false, uptimeSeconds
+   * cleared. Never touches lastSeenAt or metricsUpdatedAt, which stay the
+   * node's own last push. The node's next push flips it back through
+   * bulkUpsert (a newer lastSeenAt, isUp = true). A row the node itself
+   * refreshed after `silentBefore` is left alone, so a late report can
+   * never mark a live node down. Returns the number of rows written.
+   *
+   * A node the native pushes report is a member of a native-push cluster,
+   * so the mark also sets isNativePush: a node that was already down when
+   * the cluster moved from the agent to the native push (or before this
+   * column existed) is then kept like any other native node instead of
+   * being pruned while it is still being reported.
+   *
+   * The mark itself is notReportingMarkedAt, refreshed at most once a
+   * minute while the node stays reported: the "marked recently" the
+   * reports of an Offline node rely on to carry on through a gap
+   * (ProxmoxNativeNodeLiveness). It is written as `markedAt`, on the
+   * caller's (the ingest worker's) clock — the clock it is later judged
+   * against — never the database's now(), and only here; the node's own
+   * next push clears it (bulkUpsert).
+   */
+  @CaptureSpan()
+  public async markNodesNotReporting(data: {
+    projectId: ObjectID;
+    proxmoxClusterId: ObjectID;
+    nodeNames: Array<string>;
+    // The report's own time minus the silence window, on the PVE clock.
+    silentBefore: Date;
+    // Now, on the ingest worker's clock.
+    markedAt?: Date | undefined;
+  }): Promise<number> {
+    if (data.nodeNames.length === 0) {
+      return 0;
+    }
+    const markedAt: Date = data.markedAt || OneUptimeDate.getCurrentDate();
+    const externalIds: Array<string> = data.nodeNames.map(
+      (nodeName: string) => {
+        return truncateShortText(`${NODE_ID_PREFIX}${nodeName}`) as string;
+      },
+    );
+
+    const result: unknown = await this.getRepository().manager.query(
+      `UPDATE "ProxmoxResource"
+       SET "isUp" = false,
+           "uptimeSeconds" = NULL,
+           "isNativePush" = true,
+           "notReportingMarkedAt" = $5,
+           "updatedAt" = now()
+       WHERE "projectId" = $1 AND "proxmoxClusterId" = $2
+         AND "kind" = 'Node' AND "externalId" = ANY($3)
+         AND "deletedAt" IS NULL
+         AND "lastSeenAt" < $4
+         AND ("isUp" IS DISTINCT FROM false
+              OR "isNativePush" IS DISTINCT FROM true
+              OR "notReportingMarkedAt" IS NULL
+              OR "notReportingMarkedAt" < $6)`,
+      [
+        data.projectId.toString(),
+        data.proxmoxClusterId.toString(),
+        externalIds,
+        data.silentBefore,
+        markedAt,
+        OneUptimeDate.addRemoveSeconds(markedAt, -60),
+      ],
+    );
+
+    // Postgres driver returns [rows, affected] for UPDATE — normalize.
+    if (Array.isArray(result) && typeof result[1] === "number") {
+      return result[1];
+    }
+    return 0;
+  }
+
+  /**
+   * Take a cluster's Node rows into the native-push keep. Called when the
+   * cluster's native push is flushed (fenced by the caller): a node that
+   * was already down under the Proxmox Agent — or before isNativePush
+   * existed — never pushes itself, so without this its old row would be
+   * pruned in the two minutes before the live nodes first report it, and
+   * it would drop off the roster while still down. Only rows last seen no
+   * later than `seenUpTo` (the batch's newest observation) are taken: a
+   * native batch processed late, after the cluster moved back to the
+   * agent, must not take the agent's newer rows. It is not a report: the
+   * rows' notReportingMarkedAt is left alone. Returns the number of rows
+   * flagged.
+   */
+  @CaptureSpan()
+  public async adoptNodesAsNativePush(data: {
+    projectId: ObjectID;
+    proxmoxClusterId: ObjectID;
+    seenUpTo: Date;
+  }): Promise<number> {
+    const result: unknown = await this.getRepository().manager.query(
+      `UPDATE "ProxmoxResource"
+       SET "isNativePush" = true
+       WHERE "projectId" = $1 AND "proxmoxClusterId" = $2
+         AND "kind" = 'Node' AND "deletedAt" IS NULL
+         AND "isNativePush" IS DISTINCT FROM true
+         AND "lastSeenAt" <= $3`,
+      [
+        data.projectId.toString(),
+        data.proxmoxClusterId.toString(),
+        data.seenUpTo,
+      ],
+    );
+
+    // Postgres driver returns [rows, affected] for UPDATE — normalize.
+    if (Array.isArray(result) && typeof result[1] === "number") {
+      return result[1];
+    }
+    return 0;
+  }
+
+  /**
+   * Remove a node that has stopped reporting — for a node taken out of
+   * the cluster for good, which OneUptime cannot tell apart from a dead
+   * one on the Proxmox VE native push. Its siblings stop reporting it
+   * within a minute and its Node Offline alert resolves. Only such a node
+   * is removed:
+   *   - "not-found": no such node in this cluster;
+   *   - "not-native": an agent node — the agent lets a node go on its own
+   *     once the cluster no longer lists it, and would re-create a node
+   *     it still lists on the next scrape;
+   *   - "still-reporting": the node is up; it would reappear on its next
+   *     push.
+   */
+  @CaptureSpan()
+  public async removeOfflineNode(data: {
+    projectId: ObjectID;
+    proxmoxClusterId: ObjectID;
+    externalId: string;
+  }): Promise<ProxmoxRemoveNodeResult> {
+    if (!data.externalId.startsWith(NODE_ID_PREFIX)) {
+      return "not-found";
+    }
+    // Clamped the way bulkUpsert stored it.
+    const params: Array<string> = [
+      data.projectId.toString(),
+      data.proxmoxClusterId.toString(),
+      truncateShortText(data.externalId) as string,
+    ];
+
+    const result: unknown = await this.getRepository().manager.query(
+      `DELETE FROM "ProxmoxResource"
+       WHERE "projectId" = $1 AND "proxmoxClusterId" = $2
+         AND "kind" = 'Node' AND "externalId" = $3
+         AND "deletedAt" IS NULL
+         AND "isUp" IS FALSE AND "isNativePush" IS TRUE`,
+      params,
+    );
+
+    // Postgres driver returns [rows, affected] for DELETE — normalize.
+    if (
+      Array.isArray(result) &&
+      typeof result[1] === "number" &&
+      result[1] > 0
+    ) {
+      return "removed";
+    }
+
+    // Not deleted: say why, so the caller can answer precisely.
+    const rows: Array<{ isUp: boolean | null; isNativePush: boolean | null }> =
+      await this.getRepository().manager.query(
+        `SELECT "isUp", "isNativePush" FROM "ProxmoxResource"
+         WHERE "projectId" = $1 AND "proxmoxClusterId" = $2
+           AND "kind" = 'Node' AND "externalId" = $3
+           AND "deletedAt" IS NULL`,
+        params,
+      );
+    const row:
+      | { isUp: boolean | null; isNativePush: boolean | null }
+      | undefined = rows[0];
+    if (!row) {
+      return "not-found";
+    }
+    if (row.isNativePush !== true) {
+      return "not-native";
+    }
+    return "still-reporting";
+  }
+
+  /**
+   * The oldest last report a native-push node may have and still be kept
+   * and reported: now minus getSilentNodeRetentionHours().
+   */
+  public getSilentNodeRetentionCutoff(now?: Date | undefined): Date {
+    return OneUptimeDate.addRemoveHours(
+      now || OneUptimeDate.getCurrentDate(),
+      -this.getSilentNodeRetentionHours(),
+    );
+  }
+
+  /**
+   * How long a node that stopped reporting is still reported (and kept
+   * as Offline) before it is treated as gone. 7 days by default — long
+   * enough to outlast a weekend, short enough that a node removed from
+   * the cluster without using "Remove Node" eventually lets go. Tune via
+   * PVE_SILENT_NODE_RETENTION_HOURS (min 1).
+   */
+  public getSilentNodeRetentionHours(): number {
+    const raw: string | undefined =
+      process.env["PVE_SILENT_NODE_RETENTION_HOURS"];
+    if (raw) {
+      const parsed: number = parseInt(raw, 10);
+      if (!isNaN(parsed) && parsed >= 1) {
+        return parsed;
+      }
+    }
+    return 7 * 24;
+  }
+
   /**
    * Compute the sidebar/overview summary in Postgres: counts per kind
    * plus the online/running breakdowns, in a single round-trip.