@oneuptime/common 10.5.32 → 10.5.34

package/Models/DatabaseModels/KubernetesResource.ts

@@ -553,6 +553,43 @@ export default class KubernetesResource extends BaseModel {
   })
   public latestMemoryBytes?: number = undefined;
 
+  @ColumnAccessControl({
+    create: [],
+    read: READ_PERMISSIONS,
+    update: [],
+  })
+  @TableColumn({
+    required: false,
+    type: TableColumnType.Number,
+    canReadOnRelationQuery: true,
+    title: "Latest Memory Percent",
+    description:
+      "Most recent memory usage as a percent of the resource's node allocatable memory (Pod or Node). Stored as decimal — mirrors latestCpuPercent — so the workload/namespace list views can SUM a per-pod percentage. Null until the first metric arrives or while the node's allocatable memory is still unknown.",
+  })
+  @Column({
+    nullable: true,
+    type: ColumnType.Decimal,
+    transformer: {
+      to: (value: number | null | undefined): number | null => {
+        if (value === null || value === undefined) {
+          return null;
+        }
+        return value;
+      },
+      from: (value: string | number | null | undefined): number | null => {
+        if (value === null || value === undefined) {
+          return null;
+        }
+        if (typeof value === "number") {
+          return value;
+        }
+        const parsed: number = parseFloat(value);
+        return isNaN(parsed) ? null : parsed;
+      },
+    },
+  })
+  public latestMemoryPercent?: number = undefined;
+
   @ColumnAccessControl({
     create: [],
     read: READ_PERMISSIONS,

package/Server/API/KubernetesResourceAPI.ts

@@ -143,19 +143,24 @@ export default class KubernetesResourceAPI extends BaseAPI<
 
   /*
    * Translate a service-layer Map of aggregates into a JSON dict
-   * { name: { cpuPercent, memoryBytes } } suitable for the wire.
-   * memoryBytes is stringified so values past 2 GiB don't overflow
-   * client-side number parsing in the JSON path; the UI parses it
-   * back to a number for rendering.
+   * { name: { cpuPercent, memoryBytes, memoryPercent } } suitable for
+   * the wire. memoryBytes is stringified so values past 2 GiB don't
+   * overflow client-side number parsing in the JSON path; the UI parses
+   * it back to a number for rendering. memoryPercent is the summed
+   * per-pod "% of node allocatable memory" (parallel to cpuPercent).
    */
   private mapAggregatesToJson(
-    aggregates: Map<string, { cpuPercent: number; memoryBytes: number }>,
+    aggregates: Map<
+      string,
+      { cpuPercent: number; memoryBytes: number; memoryPercent: number }
+    >,
   ): JSONObject {
     const out: JSONObject = {};
     for (const [name, value] of aggregates.entries()) {
       out[name] = {
         cpuPercent: value.cpuPercent,
         memoryBytes: value.memoryBytes.toString(),
+        memoryPercent: value.memoryPercent,
       };
     }
     return out;
@@ -169,12 +174,14 @@ export default class KubernetesResourceAPI extends BaseAPI<
       await this.resolveClusterForRequest(req);
 
     const staleAfter: Date = new Date(Date.now() - 15 * 60 * 1000);
-    const aggregates: Map<string, { cpuPercent: number; memoryBytes: number }> =
-      await this.service.getLatestMetricsByNamespace({
-        projectId,
-        kubernetesClusterId,
-        staleAfter,
-      });
+    const aggregates: Map<
+      string,
+      { cpuPercent: number; memoryBytes: number; memoryPercent: number }
+    > = await this.service.getLatestMetricsByNamespace({
+      projectId,
+      kubernetesClusterId,
+      staleAfter,
+    });
 
     return Response.sendJsonObjectResponse(req, res, {
       aggregates: this.mapAggregatesToJson(aggregates),
@@ -210,13 +217,15 @@ export default class KubernetesResourceAPI extends BaseAPI<
       await this.resolveClusterForRequest(req);
 
     const staleAfter: Date = new Date(Date.now() - 15 * 60 * 1000);
-    const aggregates: Map<string, { cpuPercent: number; memoryBytes: number }> =
-      await this.service.getLatestMetricsByOwner({
-        projectId,
-        kubernetesClusterId,
-        ownerKind,
-        staleAfter,
-      });
+    const aggregates: Map<
+      string,
+      { cpuPercent: number; memoryBytes: number; memoryPercent: number }
+    > = await this.service.getLatestMetricsByOwner({
+      projectId,
+      kubernetesClusterId,
+      ownerKind,
+      staleAfter,
+    });
 
     return Response.sendJsonObjectResponse(req, res, {
       aggregates: this.mapAggregatesToJson(aggregates),

package/Server/Infrastructure/Postgres/SchemaMigrations/1780651429467-AddKubernetesLatestMemoryPercent.ts

@@ -0,0 +1,19 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddKubernetesLatestMemoryPercent1780651429467
+  implements MigrationInterface
+{
+  public name = "AddKubernetesLatestMemoryPercent1780651429467";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" ADD "latestMemoryPercent" numeric`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" DROP COLUMN "latestMemoryPercent"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -366,6 +366,7 @@ import { MigrationName1780382837019 } from "./1780382837019-MigrationName";
 import { MigrationName1780387560604 } from "./1780387560604-MigrationName";
 import { MigrationName1780388219225 } from "./1780388219225-MigrationName";
 import { AddMetricAndTraceSavedView1780645560183 } from "./1780645560183-AddMetricAndTraceSavedView";
+import { AddKubernetesLatestMemoryPercent1780651429467 } from "./1780651429467-AddKubernetesLatestMemoryPercent";
 
 export default [
   InitialMigration,
@@ -736,4 +737,5 @@ export default [
   MigrationName1780387560604,
   MigrationName1780388219225,
   AddMetricAndTraceSavedView1780645560183,
+  AddKubernetesLatestMemoryPercent1780651429467,
 ];

package/Server/Services/KubernetesResourceService.ts

@@ -46,6 +46,7 @@ export interface ResourceLatestMetric {
   name: string;
   cpuPercent: number | null;
   memoryBytes: number | null;
+  memoryPercent: number | null;
   observedAt: Date;
   /*
    * Optional Pod controller lineage. Read from
@@ -447,7 +448,7 @@ export class Service extends DatabaseService<Model> {
 
       for (const m of chunk) {
         valueFragments.push(
-          `($${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}::numeric, $${paramIndex++}::bigint, $${paramIndex++}::timestamptz, $${paramIndex++}, $${paramIndex++})`,
+          `($${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}::numeric, $${paramIndex++}::bigint, $${paramIndex++}::numeric, $${paramIndex++}::timestamptz, $${paramIndex++}, $${paramIndex++})`,
         );
         params.push(
           m.kind,
@@ -459,6 +460,9 @@ export class Service extends DatabaseService<Model> {
           m.memoryBytes !== null && m.memoryBytes !== undefined
             ? Math.trunc(m.memoryBytes).toString()
             : null,
+          m.memoryPercent !== null && m.memoryPercent !== undefined
+            ? m.memoryPercent
+            : null,
           m.observedAt,
           m.controllerDeploymentName ?? null,
           m.controllerCronJobName ?? null,
@@ -470,12 +474,13 @@ export class Service extends DatabaseService<Model> {
         SET
           "latestCpuPercent" = COALESCE(v."cpu", k."latestCpuPercent"),
           "latestMemoryBytes" = COALESCE(v."mem", k."latestMemoryBytes"),
+          "latestMemoryPercent" = COALESCE(v."memPct", k."latestMemoryPercent"),
           "metricsUpdatedAt" = v."observedAt",
           "controllerDeploymentName" = COALESCE(v."deployName", k."controllerDeploymentName"),
           "controllerCronJobName" = COALESCE(v."cronName", k."controllerCronJobName"),
           "updatedAt" = now()
         FROM (VALUES ${valueFragments.join(", ")})
-          AS v("kind", "ns", "name", "cpu", "mem", "observedAt", "deployName", "cronName")
+          AS v("kind", "ns", "name", "cpu", "mem", "memPct", "observedAt", "deployName", "cronName")
         WHERE
           k."projectId" = $1
           AND k."kubernetesClusterId" = $2
@@ -744,15 +749,22 @@ export class Service extends DatabaseService<Model> {
     projectId: ObjectID;
     kubernetesClusterId: ObjectID;
     staleAfter: Date;
-  }): Promise<Map<string, { cpuPercent: number; memoryBytes: number }>> {
+  }): Promise<
+    Map<
+      string,
+      { cpuPercent: number; memoryBytes: number; memoryPercent: number }
+    >
+  > {
     const rows: Array<{
       namespaceKey: string;
       cpu: string | null;
       mem: string | null;
+      memPct: string | null;
     }> = await this.getRepository().manager.query(
       `SELECT "namespaceKey",
               SUM("latestCpuPercent")::text AS cpu,
-              SUM("latestMemoryBytes")::text AS mem
+              SUM("latestMemoryBytes")::text AS mem,
+              SUM("latestMemoryPercent")::text AS "memPct"
        FROM "KubernetesResource"
        WHERE "projectId" = $1
          AND "kubernetesClusterId" = $2
@@ -768,12 +780,15 @@ export class Service extends DatabaseService<Model> {
       ],
     );
 
-    const out: Map<string, { cpuPercent: number; memoryBytes: number }> =
-      new Map();
+    const out: Map<
+      string,
+      { cpuPercent: number; memoryBytes: number; memoryPercent: number }
+    > = new Map();
     for (const row of rows) {
       out.set(row.namespaceKey || "", {
         cpuPercent: row.cpu ? parseFloat(row.cpu) || 0 : 0,
         memoryBytes: row.mem ? parseInt(row.mem, 10) || 0 : 0,
+        memoryPercent: row.memPct ? parseFloat(row.memPct) || 0 : 0,
       });
     }
     return out;
@@ -800,11 +815,17 @@ export class Service extends DatabaseService<Model> {
     kubernetesClusterId: ObjectID;
     ownerKind: string;
     staleAfter: Date;
-  }): Promise<Map<string, { cpuPercent: number; memoryBytes: number }>> {
+  }): Promise<
+    Map<
+      string,
+      { cpuPercent: number; memoryBytes: number; memoryPercent: number }
+    >
+  > {
     let rows: Array<{
       ownerName: string;
       cpu: string | null;
       mem: string | null;
+      memPct: string | null;
     }>;
 
     if (data.ownerKind === "Deployment" || data.ownerKind === "CronJob") {
@@ -816,7 +837,8 @@ export class Service extends DatabaseService<Model> {
         `SELECT
            "${column}" AS "ownerName",
            SUM("latestCpuPercent")::text AS cpu,
-           SUM("latestMemoryBytes")::text AS mem
+           SUM("latestMemoryBytes")::text AS mem,
+           SUM("latestMemoryPercent")::text AS "memPct"
          FROM "KubernetesResource"
          WHERE "projectId" = $1
            AND "kubernetesClusterId" = $2
@@ -837,7 +859,8 @@ export class Service extends DatabaseService<Model> {
         `SELECT
            (owner->>'name') AS "ownerName",
            SUM("latestCpuPercent")::text AS cpu,
-           SUM("latestMemoryBytes")::text AS mem
+           SUM("latestMemoryBytes")::text AS mem,
+           SUM("latestMemoryPercent")::text AS "memPct"
          FROM "KubernetesResource",
               jsonb_array_elements("ownerReferences"->'items') AS owner
          WHERE "projectId" = $1
@@ -858,8 +881,10 @@ export class Service extends DatabaseService<Model> {
       );
     }
 
-    const out: Map<string, { cpuPercent: number; memoryBytes: number }> =
-      new Map();
+    const out: Map<
+      string,
+      { cpuPercent: number; memoryBytes: number; memoryPercent: number }
+    > = new Map();
     for (const row of rows) {
       if (!row.ownerName) {
         continue;
@@ -867,6 +892,7 @@ export class Service extends DatabaseService<Model> {
       out.set(row.ownerName, {
         cpuPercent: row.cpu ? parseFloat(row.cpu) || 0 : 0,
         memoryBytes: row.mem ? parseInt(row.mem, 10) || 0 : 0,
+        memoryPercent: row.memPct ? parseFloat(row.memPct) || 0 : 0,
       });
     }
     return out;

package/Server/Utils/Monitor/MonitorAlert.ts

@@ -115,6 +115,13 @@ export default class MonitorAlert {
       telemetryQuery?: TelemetryQuery | undefined;
     };
     matchesPerSeries?: Array<PerSeriesCriteriaMatch> | undefined;
+    /**
+     * Series fingerprints whose underlying resource is inside an
+     * ongoing scheduled maintenance window. Alerts for these series are
+     * suppressed at creation time even though the monitor keeps
+     * evaluating. See MonitorMaintenanceSuppression.
+     */
+    suppressedSeriesFingerprints?: Set<string> | undefined;
   }): Promise<void> {
     const alertLogAttributes: LogAttributes = {
       projectId: input.monitor.projectId?.toString(),
@@ -164,6 +171,33 @@ export default class MonitorAlert {
         const seriesRootCause: string =
           seriesMatch?.rootCause || input.rootCause;
 
+        /*
+         * Per-series scheduled-maintenance suppression: skip creating an
+         * alert for a series whose resource is inside an ongoing
+         * maintenance window. Other series on the same monitor are
+         * unaffected. Only *new* creation is suppressed — existing open
+         * alerts follow the normal resolve path.
+         */
+        if (
+          seriesFingerprint &&
+          input.suppressedSeriesFingerprints?.has(seriesFingerprint)
+        ) {
+          logger.debug(
+            `${input.monitor.id?.toString()} - Skipping alert for series ${seriesFingerprint}: its resource is under an active scheduled maintenance window.`,
+            alertLogAttributes,
+          );
+
+          input.evaluationSummary?.events.push({
+            type: "alert-skipped",
+            title: "Alert suppressed by scheduled maintenance",
+            message:
+              "Skipped creating an alert because the resource for this series is under an active scheduled maintenance window.",
+            relatedCriteriaId: input.criteriaInstance.data?.id,
+            at: OneUptimeDate.getCurrentDate(),
+          });
+          continue;
+        }
+
         const alreadyOpenAlert: Alert | undefined = openAlerts.find(
           (alert: Alert) => {
             return (

package/Server/Utils/Monitor/MonitorIncident.ts

@@ -37,6 +37,9 @@ import OneUptimeDate from "../../../Types/Date";
 import MonitorEvaluationSummary from "../../../Types/Monitor/MonitorEvaluationSummary";
 import { IncidentMemberRoleAssignment } from "../../../Types/Monitor/CriteriaIncident";
 import { PerSeriesCriteriaMatch } from "../../../Types/Probe/ProbeApiIngestResponse";
+import SeriesResourceLabels, {
+  SeriesResourceRefs,
+} from "./SeriesResourceLabels";
 
 export default class MonitorIncident {
   @CaptureSpan()
@@ -143,6 +146,14 @@ export default class MonitorIncident {
      * reference `{{host.name}}` etc. via the template engine.
      */
     matchesPerSeries?: Array<PerSeriesCriteriaMatch> | undefined;
+    /**
+     * Series fingerprints whose underlying resource (host, docker host,
+     * kubernetes cluster, or service) is inside an ongoing scheduled
+     * maintenance window. The monitor itself keeps evaluating — it is
+     * not attached to the maintenance — but incidents for these series
+     * are suppressed at creation time. See MonitorMaintenanceSuppression.
+     */
+    suppressedSeriesFingerprints?: Set<string> | undefined;
   }): Promise<void> {
     const incidentLogAttributes: LogAttributes = {
       projectId: input.monitor.projectId?.toString(),
@@ -202,6 +213,37 @@ export default class MonitorIncident {
         const seriesRootCause: string =
           seriesMatch?.rootCause || input.rootCause;
 
+        /*
+         * Per-series scheduled-maintenance suppression: this series'
+         * resource is inside an ongoing maintenance window, so skip
+         * creating an incident for it. Other series on the same monitor
+         * whose resources are not under maintenance still get incidents.
+         * Note: we only suppress *new* creation — any incident already
+         * open for this series is left to the normal resolve path
+         * (checkOpenIncidentsAndCloseIfResolved still sees the full
+         * breaching set), so a real incident raised before maintenance
+         * is not silently closed.
+         */
+        if (
+          seriesFingerprint &&
+          input.suppressedSeriesFingerprints?.has(seriesFingerprint)
+        ) {
+          logger.debug(
+            `${input.monitor.id?.toString()} - Skipping incident for series ${seriesFingerprint}: its resource is under an active scheduled maintenance window.`,
+            incidentLogAttributes,
+          );
+
+          input.evaluationSummary?.events.push({
+            type: "incident-skipped",
+            title: "Incident suppressed by scheduled maintenance",
+            message:
+              "Skipped creating an incident because the resource for this series is under an active scheduled maintenance window.",
+            relatedCriteriaId: input.criteriaInstance.data?.id,
+            at: OneUptimeDate.getCurrentDate(),
+          });
+          continue;
+        }
+
         const alreadyOpenIncident: Incident | undefined = openIncidents.find(
           (incident: Incident) => {
             return (
@@ -485,97 +527,22 @@ export default class MonitorIncident {
   }
 
   /*
-   * Pull every host / docker-host / k8s-cluster identifier out of the
-   * series labels and attach the matching project-scoped records to
-   * the incident. Series labels carry both raw OTel resource
-   * attributes (`host.name`, `k8s.cluster.name`) and the OneUptime
-   * stamps added at ingest (`oneuptime.host.id`,
-   * `oneuptime.docker.host.id`, `oneuptime.kubernetes.cluster.id` and
-   * their `.name` twins). OTel resource attributes are stored under
-   * the `resource.` prefix in ClickHouse, so prefixed and unprefixed
-   * forms are both accepted — whichever the group-by query surfaced,
-   * we'll find it. Multi-value labels are flattened, so a series that
-   * groups by a multi-valued attribute attaches every matching
-   * record. Lookups are always project-scoped so a stale or hostile
-   * stamp can't pull in a record from another tenant.
-   *
-   * For Docker hosts we deliberately ignore raw `host.name`/
-   * `oneuptime.host.name`: those are the Host's territory, and the
-   * ingest pipeline stamps `oneuptime.docker.host.*` independently
-   * when the source is a docker host.
+   * Pull every host / docker-host / k8s-cluster / service identifier
+   * out of the series labels and attach the matching project-scoped
+   * records to the incident. The label-key → resource-type mapping
+   * lives in SeriesResourceLabels (shared with the scheduled-maintenance
+   * suppression path so the two never disagree about which labels
+   * identify which resource). Lookups are always project-scoped so a
+   * stale or hostile stamp can't pull in a record from another tenant.
    */
   private static async linkResourceContextFromSeries(input: {
     incident: Incident;
     seriesLabels: JSONObject;
     projectId: ObjectID;
   }): Promise<void> {
-    const collect: (keys: ReadonlyArray<string>) => Array<string> = (
-      keys: ReadonlyArray<string>,
-    ): Array<string> => {
-      const found: Set<string> = new Set<string>();
-      for (const key of keys) {
-        const value: unknown = input.seriesLabels[key];
-        if (typeof value === "string" && value.length > 0) {
-          found.add(value);
-          continue;
-        }
-        if (Array.isArray(value)) {
-          for (const item of value) {
-            if (typeof item === "string" && item.length > 0) {
-              found.add(item);
-            }
-          }
-        }
-      }
-      return Array.from(found);
-    };
-
-    const hostIds: Array<string> = collect([
-      "resource.oneuptime.host.id",
-      "oneuptime.host.id",
-    ]);
-    const hostNames: Array<string> = collect([
-      "resource.oneuptime.host.name",
-      "oneuptime.host.name",
-      "resource.host.name",
-      "host.name",
-    ]);
-
-    const dockerHostIds: Array<string> = collect([
-      "resource.oneuptime.docker.host.id",
-      "oneuptime.docker.host.id",
-    ]);
-    const dockerHostNames: Array<string> = collect([
-      "resource.oneuptime.docker.host.name",
-      "oneuptime.docker.host.name",
-    ]);
-
-    const clusterIds: Array<string> = collect([
-      "resource.oneuptime.kubernetes.cluster.id",
-      "oneuptime.kubernetes.cluster.id",
-    ]);
-    const clusterNames: Array<string> = collect([
-      "resource.oneuptime.kubernetes.cluster.name",
-      "oneuptime.kubernetes.cluster.name",
-      "resource.k8s.cluster.name",
-      "k8s.cluster.name",
-    ]);
-
-    /*
-     * Services come from OTel-ingested telemetry. The ingest pipeline
-     * auto-creates a Service row keyed by `service.name`, so any series
-     * label that carries that attribute (raw or prefixed) tells us the
-     * emitting service. We also accept the `oneuptime.service.id`
-     * stamp for callers that already resolved the ID upstream.
-     */
-    const serviceIds: Array<string> = collect([
-      "resource.oneuptime.service.id",
-      "oneuptime.service.id",
-    ]);
-    const serviceNames: Array<string> = collect([
-      "resource.service.name",
-      "service.name",
-    ]);
+    const refs: SeriesResourceRefs = SeriesResourceLabels.extractResourceRefs(
+      input.seriesLabels,
+    );
 
     const [
       resolvedHosts,
@@ -584,29 +551,29 @@ export default class MonitorIncident {
       resolvedServices,
     ] = await Promise.all([
       this.resolveResourceIds({
-        ids: hostIds,
-        names: hostNames,
+        ids: refs.hostIds,
+        names: refs.hostNames,
         nameColumn: "hostIdentifier",
         projectId: input.projectId,
         findBy: HostService.findBy.bind(HostService),
       }),
       this.resolveResourceIds({
-        ids: dockerHostIds,
-        names: dockerHostNames,
+        ids: refs.dockerHostIds,
+        names: refs.dockerHostNames,
         nameColumn: "hostIdentifier",
         projectId: input.projectId,
         findBy: DockerHostService.findBy.bind(DockerHostService),
       }),
       this.resolveResourceIds({
-        ids: clusterIds,
-        names: clusterNames,
+        ids: refs.kubernetesClusterIds,
+        names: refs.kubernetesClusterNames,
         nameColumn: "clusterIdentifier",
         projectId: input.projectId,
         findBy: KubernetesClusterService.findBy.bind(KubernetesClusterService),
       }),
       this.resolveResourceIds({
-        ids: serviceIds,
-        names: serviceNames,
+        ids: refs.serviceIds,
+        names: refs.serviceNames,
         nameColumn: "name",
         projectId: input.projectId,
         findBy: ServiceService.findBy.bind(ServiceService),