@oneuptime/common 11.0.1 → 11.0.2

package/Server/Utils/Monitor/MonitorIncident.ts

@@ -1,3 +1,4 @@
+import CephCluster from "../../../Models/DatabaseModels/CephCluster";
 import DockerHost from "../../../Models/DatabaseModels/DockerHost";
 import Host from "../../../Models/DatabaseModels/Host";
 import Incident from "../../../Models/DatabaseModels/Incident";
@@ -8,6 +9,7 @@ import KubernetesCluster from "../../../Models/DatabaseModels/KubernetesCluster"
 import Label from "../../../Models/DatabaseModels/Label";
 import Monitor from "../../../Models/DatabaseModels/Monitor";
 import OnCallDutyPolicy from "../../../Models/DatabaseModels/OnCallDutyPolicy";
+import ProxmoxCluster from "../../../Models/DatabaseModels/ProxmoxCluster";
 import Service from "../../../Models/DatabaseModels/Service";
 import Includes from "../../../Types/BaseDatabase/Includes";
 import SortOrder from "../../../Types/BaseDatabase/SortOrder";
@@ -20,10 +22,12 @@ import ObjectID from "../../../Types/ObjectID";
 import ProbeMonitorResponse from "../../../Types/Probe/ProbeMonitorResponse";
 import { TelemetryQuery } from "../../../Types/Telemetry/TelemetryQuery";
 import { DisableAutomaticIncidentCreation } from "../../EnvironmentConfig";
+import CephClusterService from "../../Services/CephClusterService";
 import DockerHostService from "../../Services/DockerHostService";
 import HostService from "../../Services/HostService";
 import IncidentService from "../../Services/IncidentService";
 import KubernetesClusterService from "../../Services/KubernetesClusterService";
+import ProxmoxClusterService from "../../Services/ProxmoxClusterService";
 import ServiceService from "../../Services/ServiceService";
 import IncidentSeverityService from "../../Services/IncidentSeverityService";
 import IncidentStateTimelineService from "../../Services/IncidentStateTimelineService";
@@ -37,6 +41,9 @@ import OneUptimeDate from "../../../Types/Date";
 import MonitorEvaluationSummary from "../../../Types/Monitor/MonitorEvaluationSummary";
 import { IncidentMemberRoleAssignment } from "../../../Types/Monitor/CriteriaIncident";
 import { PerSeriesCriteriaMatch } from "../../../Types/Probe/ProbeApiIngestResponse";
+import MonitorClusterContextUtil, {
+  MonitorClusterContext,
+} from "./MonitorClusterContext";
 import SeriesResourceLabels, {
   SeriesResourceRefs,
 } from "./SeriesResourceLabels";
@@ -196,6 +203,19 @@ export default class MonitorIncident {
       return;
     }
 
+    /*
+     * Proxmox/Ceph monitors: resolve the monitored cluster once per
+     * evaluation (lookup-only, from the step config's clusterIdentifier)
+     * so every incident created below is attached to it. Series labels
+     * cannot supply this — they carry datapoint labels (`id`,
+     * `ceph_daemon`, `pool_id`), not cluster identity, and ungrouped
+     * templates have no series at all. No-op for other monitor types.
+     */
+    const clusterContext: MonitorClusterContext =
+      await MonitorClusterContextUtil.resolveClusterContextForMonitor({
+        monitor: input.monitor,
+      });
+
     /*
      * Series-less path: one incident per criteriaIncident template as
      * before. Series-aware path: one incident per (series × template).
@@ -384,6 +404,15 @@ export default class MonitorIncident {
           });
         }
 
+        /*
+         * Deterministic Proxmox/Ceph cluster link from the monitor's
+         * step config (resolved once above). Runs for both grouped and
+         * ungrouped incidents and merges with anything the series-label
+         * path resolved, so the per-cluster Activity tabs always see
+         * monitor-created incidents.
+         */
+        this.attachClusterContext({ incident, clusterContext });
+
         incident.onCallDutyPolicies =
           criteriaIncident.onCallPolicyIds?.map((id: ObjectID) => {
             const onCallPolicy: OnCallDutyPolicy = new OnCallDutyPolicy();
@@ -527,13 +556,14 @@ export default class MonitorIncident {
   }
 
   /*
-   * Pull every host / docker-host / k8s-cluster / service identifier
-   * out of the series labels and attach the matching project-scoped
-   * records to the incident. The label-key → resource-type mapping
-   * lives in SeriesResourceLabels (shared with the scheduled-maintenance
-   * suppression path so the two never disagree about which labels
-   * identify which resource). Lookups are always project-scoped so a
-   * stale or hostile stamp can't pull in a record from another tenant.
+   * Pull every host / docker-host / k8s-cluster / proxmox-cluster /
+   * ceph-cluster / service identifier out of the series labels and
+   * attach the matching project-scoped records to the incident. The
+   * label-key → resource-type mapping lives in SeriesResourceLabels
+   * (shared with the scheduled-maintenance suppression path so the two
+   * never disagree about which labels identify which resource). Lookups
+   * are always project-scoped so a stale or hostile stamp can't pull in
+   * a record from another tenant.
    */
   private static async linkResourceContextFromSeries(input: {
     incident: Incident;
@@ -549,6 +579,8 @@ export default class MonitorIncident {
       resolvedDockerHosts,
       resolvedClusters,
       resolvedServices,
+      resolvedProxmoxClusters,
+      resolvedCephClusters,
     ] = await Promise.all([
       this.resolveResourceIds({
         ids: refs.hostIds,
@@ -578,6 +610,20 @@ export default class MonitorIncident {
         projectId: input.projectId,
         findBy: ServiceService.findBy.bind(ServiceService),
       }),
+      this.resolveResourceIds({
+        ids: [],
+        names: refs.proxmoxClusterNames,
+        nameColumn: "name",
+        projectId: input.projectId,
+        findBy: ProxmoxClusterService.findBy.bind(ProxmoxClusterService),
+      }),
+      this.resolveResourceIds({
+        ids: [],
+        names: refs.cephClusterNames,
+        nameColumn: "name",
+        projectId: input.projectId,
+        findBy: CephClusterService.findBy.bind(CephClusterService),
+      }),
     ]);
 
     if (resolvedHosts.length > 0) {
@@ -612,6 +658,83 @@ export default class MonitorIncident {
         return service;
       });
     }
+    if (resolvedProxmoxClusters.length > 0) {
+      input.incident.proxmoxClusters = resolvedProxmoxClusters.map(
+        (id: string): ProxmoxCluster => {
+          const cluster: ProxmoxCluster = new ProxmoxCluster();
+          cluster._id = id;
+          return cluster;
+        },
+      );
+    }
+    if (resolvedCephClusters.length > 0) {
+      input.incident.cephClusters = resolvedCephClusters.map(
+        (id: string): CephCluster => {
+          const cluster: CephCluster = new CephCluster();
+          cluster._id = id;
+          return cluster;
+        },
+      );
+    }
+  }
+
+  /*
+   * Attach the monitor-step-resolved Proxmox/Ceph cluster ids to the
+   * incident, merging with (never overwriting) anything the
+   * series-label path already linked. Both paths can resolve the same
+   * cluster, so dedupe by id.
+   */
+  private static attachClusterContext(input: {
+    incident: Incident;
+    clusterContext: MonitorClusterContext;
+  }): void {
+    if (input.clusterContext.proxmoxClusterIds.length > 0) {
+      const existingIds: Set<string> = new Set<string>(
+        (input.incident.proxmoxClusters || []).map(
+          (cluster: ProxmoxCluster) => {
+            return String(cluster._id);
+          },
+        ),
+      );
+
+      const merged: Array<ProxmoxCluster> = [
+        ...(input.incident.proxmoxClusters || []),
+      ];
+
+      for (const id of input.clusterContext.proxmoxClusterIds) {
+        if (existingIds.has(id)) {
+          continue;
+        }
+        const cluster: ProxmoxCluster = new ProxmoxCluster();
+        cluster._id = id;
+        merged.push(cluster);
+      }
+
+      input.incident.proxmoxClusters = merged;
+    }
+
+    if (input.clusterContext.cephClusterIds.length > 0) {
+      const existingIds: Set<string> = new Set<string>(
+        (input.incident.cephClusters || []).map((cluster: CephCluster) => {
+          return String(cluster._id);
+        }),
+      );
+
+      const merged: Array<CephCluster> = [
+        ...(input.incident.cephClusters || []),
+      ];
+
+      for (const id of input.clusterContext.cephClusterIds) {
+        if (existingIds.has(id)) {
+          continue;
+        }
+        const cluster: CephCluster = new CephCluster();
+        cluster._id = id;
+        merged.push(cluster);
+      }
+
+      input.incident.cephClusters = merged;
+    }
   }
 
   /*

package/Server/Utils/Monitor/MonitorMaintenanceSuppression.ts

@@ -24,6 +24,12 @@ export interface MaintainedResourceKeys {
   hosts: ResourceKeySet;
   dockerHosts: ResourceKeySet;
   kubernetesClusters: ResourceKeySet;
+  /*
+   * Proxmox/Ceph clusters have no `oneuptime.*.id` label stamp, so only
+   * the name set is ever matched; the id set exists for shape parity.
+   */
+  proxmoxClusters: ResourceKeySet;
+  cephClusters: ResourceKeySet;
   services: ResourceKeySet;
 }
 
@@ -45,7 +51,8 @@ export interface MaintainedResourceKeys {
  * incident/alert creation loops can skip exactly those series while the
  * other 90 hosts keep alerting. It covers every resource type a
  * maintenance event can attach to AND a series can identify: Host,
- * DockerHost, KubernetesCluster, and Service.
+ * DockerHost, KubernetesCluster, ProxmoxCluster, CephCluster, and
+ * Service.
  */
 export default class MonitorMaintenanceSuppression {
   /*
@@ -113,6 +120,14 @@ export default class MonitorMaintenanceSuppression {
           refs.kubernetesClusterNames,
           input.maintained.kubernetesClusters.names,
         ) ||
+        this.intersects(
+          refs.proxmoxClusterNames,
+          input.maintained.proxmoxClusters.names,
+        ) ||
+        this.intersects(
+          refs.cephClusterNames,
+          input.maintained.cephClusters.names,
+        ) ||
         this.intersects(refs.serviceIds, input.maintained.services.ids) ||
         this.intersects(refs.serviceNames, input.maintained.services.names);
 
@@ -143,6 +158,10 @@ export default class MonitorMaintenanceSuppression {
       maintained.dockerHosts.names.size > 0 ||
       maintained.kubernetesClusters.ids.size > 0 ||
       maintained.kubernetesClusters.names.size > 0 ||
+      maintained.proxmoxClusters.ids.size > 0 ||
+      maintained.proxmoxClusters.names.size > 0 ||
+      maintained.cephClusters.ids.size > 0 ||
+      maintained.cephClusters.names.size > 0 ||
       maintained.services.ids.size > 0 ||
       maintained.services.names.size > 0
     );
@@ -150,11 +169,11 @@ export default class MonitorMaintenanceSuppression {
 
   /*
    * Collect the ids + identifiers of every Host / DockerHost /
-   * KubernetesCluster / Service attached to an ongoing maintenance event
-   * in this project. Monitors attached to the event are intentionally
-   * not collected here — those are already handled upstream by the
-   * whole-monitor disable flag, which short-circuits evaluation before
-   * we ever reach per-series creation.
+   * KubernetesCluster / ProxmoxCluster / CephCluster / Service attached
+   * to an ongoing maintenance event in this project. Monitors attached
+   * to the event are intentionally not collected here — those are
+   * already handled upstream by the whole-monitor disable flag, which
+   * short-circuits evaluation before we ever reach per-series creation.
    */
   private static async getResourcesUnderOngoingMaintenance(
     projectId: ObjectID,
@@ -163,6 +182,8 @@ export default class MonitorMaintenanceSuppression {
       hosts: { ids: new Set<string>(), names: new Set<string>() },
       dockerHosts: { ids: new Set<string>(), names: new Set<string>() },
       kubernetesClusters: { ids: new Set<string>(), names: new Set<string>() },
+      proxmoxClusters: { ids: new Set<string>(), names: new Set<string>() },
+      cephClusters: { ids: new Set<string>(), names: new Set<string>() },
       services: { ids: new Set<string>(), names: new Set<string>() },
     };
 
@@ -179,6 +200,8 @@ export default class MonitorMaintenanceSuppression {
           hosts: { _id: true, hostIdentifier: true },
           dockerHosts: { _id: true, hostIdentifier: true },
           kubernetesClusters: { _id: true, clusterIdentifier: true },
+          proxmoxClusters: { _id: true, name: true },
+          cephClusters: { _id: true, name: true },
           services: { _id: true, name: true },
         },
         skip: 0,
@@ -206,6 +229,16 @@ export default class MonitorMaintenanceSuppression {
           cluster.clusterIdentifier,
         );
       }
+      for (const proxmoxCluster of event.proxmoxClusters || []) {
+        this.addKey(
+          maintained.proxmoxClusters,
+          proxmoxCluster._id,
+          proxmoxCluster.name,
+        );
+      }
+      for (const cephCluster of event.cephClusters || []) {
+        this.addKey(maintained.cephClusters, cephCluster._id, cephCluster.name);
+      }
       for (const service of event.services || []) {
         this.addKey(maintained.services, service._id, service.name);
       }

package/Server/Utils/Monitor/MonitorTemplateUtil.ts

@@ -360,7 +360,9 @@ export default class MonitorTemplateUtil {
       if (
         data.monitorType === MonitorType.Metrics ||
         data.monitorType === MonitorType.Kubernetes ||
-        data.monitorType === MonitorType.Docker
+        data.monitorType === MonitorType.Docker ||
+        data.monitorType === MonitorType.Proxmox ||
+        data.monitorType === MonitorType.Ceph
       ) {
         const metricResponse: MetricMonitorResponse =
           data.dataToProcess as MetricMonitorResponse;

package/Server/Utils/Monitor/SeriesResourceLabels.ts

@@ -60,6 +60,29 @@ export const KubernetesClusterNameLabelKeys: ReadonlyArray<string> = [
   "k8s.cluster.name",
 ];
 
+/*
+ * Proxmox/Ceph cluster identity rides the agent-stamped resource
+ * attribute (`proxmox.cluster.name` / `ceph.cluster.name`) and its
+ * ClickHouse `resource.`-prefixed twin. Ingest keys cluster rows by
+ * name only — there is no `oneuptime.*.id` stamp for these clusters —
+ * so only name keys exist. The name maps to the cluster model's `name`
+ * column. Note the shipped Proxmox/Ceph alert templates group by
+ * datapoint labels (`id`, `ceph_daemon`, `pool_id`), so their series
+ * labels do NOT carry these keys; the deterministic cluster link for
+ * those monitors comes from the monitor step config instead (see
+ * MonitorClusterContext). These keys cover user-built monitors that
+ * group by the cluster attribute, exactly like the K8s keys above.
+ */
+export const ProxmoxClusterNameLabelKeys: ReadonlyArray<string> = [
+  "resource.proxmox.cluster.name",
+  "proxmox.cluster.name",
+];
+
+export const CephClusterNameLabelKeys: ReadonlyArray<string> = [
+  "resource.ceph.cluster.name",
+  "ceph.cluster.name",
+];
+
 /*
  * Services come from OTel-ingested telemetry. The ingest pipeline
  * auto-creates a Service row keyed by `service.name`, so any series
@@ -91,6 +114,8 @@ export interface SeriesResourceRefs {
   dockerHostNames: Array<string>;
   kubernetesClusterIds: Array<string>;
   kubernetesClusterNames: Array<string>;
+  proxmoxClusterNames: Array<string>;
+  cephClusterNames: Array<string>;
   serviceIds: Array<string>;
   serviceNames: Array<string>;
 }
@@ -149,6 +174,14 @@ export default class SeriesResourceLabels {
         seriesLabels,
         KubernetesClusterNameLabelKeys,
       ),
+      proxmoxClusterNames: this.collectLabelValues(
+        seriesLabels,
+        ProxmoxClusterNameLabelKeys,
+      ),
+      cephClusterNames: this.collectLabelValues(
+        seriesLabels,
+        CephClusterNameLabelKeys,
+      ),
       serviceIds: this.collectLabelValues(seriesLabels, ServiceIdLabelKeys),
       serviceNames: this.collectLabelValues(seriesLabels, ServiceNameLabelKeys),
     };

package/Server/Utils/Profiling.ts

@@ -1,9 +1,20 @@
-import Pyroscope from "@pyroscope/nodejs";
 import { EnableProfiling } from "../EnvironmentConfig";
 import logger, { LogAttributes } from "./Logger";
 import GracefulShutdown, { ShutdownPriority } from "./GracefulShutdown";
 
+/*
+ * Type-only import: erased at compile time, so it emits NO runtime require.
+ * @pyroscope/nodejs pulls in @datadog/pprof, whose native (node-gyp) binding loads
+ * at require() time. When no prebuilt binary exists for the running Node ABI (e.g. a
+ * newer Node major than the pprof build ships for), that require throws — so the module
+ * must never be imported at the top level, or the throw takes down the whole process
+ * before init() is even reached. We load it lazily and defensively inside init() instead.
+ */
+type PyroscopeModule = typeof import("@pyroscope/nodejs");
+
 export default class Profiling {
+  private static pyroscope: PyroscopeModule | null = null;
+
   public static init(data: { serviceName: string }): void {
     if (!EnableProfiling) {
       return;
@@ -24,6 +35,25 @@ export default class Profiling {
       return;
     }
 
+    let Pyroscope: PyroscopeModule;
+    try {
+      /*
+       * Lazy load so the native pprof binding is only required when profiling is actually
+       * enabled, and so a load failure (missing/incompatible prebuilt binary for the running
+       * Node ABI) is caught here and degrades to "profiling disabled" instead of crashing
+       * the server. Profiling is a best-effort, optional feature.
+       */
+      // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+      Pyroscope = require("@pyroscope/nodejs") as PyroscopeModule;
+    } catch (err) {
+      logger.warn(
+        "Profiling enabled but the profiler native module could not be loaded. Continuing without profiling.",
+        profilingLogAttributes,
+      );
+      logger.warn(err, profilingLogAttributes);
+      return;
+    }
+
     try {
       Pyroscope.init({
         appName: data.serviceName,
@@ -36,6 +66,8 @@ export default class Profiling {
 
       Pyroscope.start();
 
+      this.pyroscope = Pyroscope;
+
       logger.info(
         `Profiling initialized for service: ${data.serviceName} -> ${serverAddress}`,
         profilingLogAttributes,
@@ -43,6 +75,7 @@ export default class Profiling {
     } catch (err) {
       logger.error("Failed to initialize profiling:", profilingLogAttributes);
       logger.error(err, profilingLogAttributes);
+      return;
     }
 
     // Stop the profiler last (Telemetry tier), alongside the OTEL flush.
@@ -51,7 +84,9 @@ export default class Profiling {
       ShutdownPriority.Telemetry,
       async (): Promise<void> => {
         try {
-          await Pyroscope.stop();
+          if (this.pyroscope) {
+            await this.pyroscope.stop();
+          }
         } catch (err) {
           logger.error("Error stopping profiler:", profilingLogAttributes);
           logger.error(err, profilingLogAttributes);

package/Server/Utils/Telemetry/EntityRegistry.ts

@@ -60,6 +60,10 @@ export const DEFAULT_ENTITY_BUDGET: ReadonlyMap<EntityType, number> = new Map<
   [EntityType.KubernetesNamespace, 1000],
   [EntityType.KubernetesPod, 5000],
   [EntityType.KubernetesDeployment, 5000],
+  [EntityType.ProxmoxCluster, 10000],
+  [EntityType.ProxmoxNode, 1000],
+  [EntityType.ProxmoxGuest, 5000],
+  [EntityType.CephCluster, 10000],
 ]);
 
 // For types not in the map (future promotions of high-churn types).