@oneuptime/common 11.7.3 → 11.7.4

package/Tests/Utils/StatusPage/ResourceUptime.test.ts

@@ -1,7 +1,11 @@
 import StatusPageResourceUptimeUtil from "../../../Utils/StatusPage/ResourceUptime";
 import { Green, Red, Yellow } from "../../../Types/BrandColors";
 import ObjectID from "../../../Types/ObjectID";
+import OneUptimeDate from "../../../Types/Date";
 import Dictionary from "../../../Types/Dictionary";
+import UptimePrecision from "../../../Types/StatusPage/UptimePrecision";
+import { UptimeWindow } from "../../../Utils/Uptime/UptimeUtil";
+import Monitor from "../../../Models/DatabaseModels/Monitor";
 import MonitorStatus from "../../../Models/DatabaseModels/MonitorStatus";
 import MonitorStatusTimeline from "../../../Models/DatabaseModels/MonitorStatusTimeline";
 import StatusPageResource from "../../../Models/DatabaseModels/StatusPageResource";
@@ -260,4 +264,411 @@ describe("StatusPageResourceUptimeUtil", () => {
       expect(result).toHaveLength(0);
     });
   });
+
+  /*
+   * Groups can be nested (Corporate Unit -> Region -> Market -> Site), and the
+   * whole point of the hierarchy is that every level reports a rolled up
+   * number. These cases pin down what "rolled up" means:
+   *
+   *   - a group's status / uptime covers its own resources AND everything in
+   *     the groups below it,
+   *   - the hierarchy is opt in: without the group list the helpers behave
+   *     exactly as they did before nesting existed,
+   *   - and the page-wide average counts each resource once, however deep it
+   *     sits (a parent already averages its subtree, so descending past it
+   *     would count its children twice).
+   *
+   * A monitor that is operational for the whole window is 100%, a monitor that
+   * is offline for the whole window is 0% - so the expected averages below are
+   * exact rather than time-dependent.
+   */
+  describe("nested groups", () => {
+    const PARENT_GROUP: ObjectID = new ObjectID(
+      "cccccccc-cccc-4ccc-8ccc-cccccccccccc",
+    );
+    const CHILD_GROUP: ObjectID = new ObjectID(
+      "dddddddd-dddd-4ddd-8ddd-dddddddddddd",
+    );
+    const OPERATIONAL_STATUS: ObjectID = new ObjectID(
+      "eeeeeeee-eeee-4eee-8eee-eeeeeeeeeeee",
+    );
+    const OFFLINE_STATUS: ObjectID = new ObjectID(
+      "ffffffff-ffff-4fff-8fff-ffffffffffff",
+    );
+
+    const uptimeWindow: UptimeWindow = {
+      startDate: OneUptimeDate.getSomeDaysAgo(10),
+      endDate: OneUptimeDate.getCurrentDate(),
+    };
+
+    function operationalStatus(): MonitorStatus {
+      const status: MonitorStatus = new MonitorStatus();
+      status._id = OPERATIONAL_STATUS.toString();
+      status.name = "Operational";
+      status.priority = 1;
+      status.color = Green;
+      status.isOperationalState = true;
+      return status;
+    }
+
+    function offlineStatus(): MonitorStatus {
+      const status: MonitorStatus = new MonitorStatus();
+      status._id = OFFLINE_STATUS.toString();
+      status.name = "Offline";
+      status.priority = 3;
+      status.color = Red;
+      return status;
+    }
+
+    // an open timeline row covering the whole reporting window.
+    function makeTimeline(data: {
+      monitorId: ObjectID;
+      status: MonitorStatus;
+    }): MonitorStatusTimeline {
+      const timeline: MonitorStatusTimeline = new MonitorStatusTimeline();
+      timeline.monitorId = data.monitorId;
+      timeline.monitorStatus = data.status;
+      timeline.monitorStatusId = data.status.id!;
+      timeline.startsAt = uptimeWindow.startDate;
+      return timeline;
+    }
+
+    function makeGroup(data: {
+      id: ObjectID;
+      name: string;
+      parentId?: ObjectID | undefined;
+      showUptimePercent: boolean;
+    }): StatusPageGroup {
+      const group: StatusPageGroup = new StatusPageGroup();
+      group._id = data.id.toString();
+      group.name = data.name;
+      group.showUptimePercent = data.showUptimePercent;
+      group.uptimePercentPrecision = UptimePrecision.TWO_DECIMAL;
+
+      if (data.parentId) {
+        group.parentStatusPageGroupId = data.parentId;
+      }
+
+      return group;
+    }
+
+    function makeResource(data: {
+      monitorId: ObjectID;
+      currentStatusId: ObjectID;
+      groupId?: ObjectID | undefined;
+    }): StatusPageResource {
+      const resource: StatusPageResource = new StatusPageResource();
+      resource.monitorId = data.monitorId;
+      resource.showUptimePercent = true;
+
+      const monitor: Monitor = new Monitor();
+      monitor._id = data.monitorId.toString();
+      monitor.currentMonitorStatusId = data.currentStatusId;
+      resource.monitor = monitor;
+
+      if (data.groupId) {
+        resource.statusPageGroupId = data.groupId;
+      }
+
+      return resource;
+    }
+
+    /*
+     * Parent (2 healthy resources) -> Child (1 offline resource).
+     * Parent on its own is 100%, the subtree is (100 + 100 + 0) / 3 = 66.66%.
+     */
+    function makeFixture(data?: {
+      parentShowsUptimePercent?: boolean | undefined;
+    }): {
+      groups: Array<StatusPageGroup>;
+      resources: Array<StatusPageResource>;
+      timelines: Array<MonitorStatusTimeline>;
+      statuses: Array<MonitorStatus>;
+    } {
+      const parent: StatusPageGroup = makeGroup({
+        id: PARENT_GROUP,
+        name: "Corporate Units",
+        showUptimePercent: data?.parentShowsUptimePercent ?? true,
+      });
+      const child: StatusPageGroup = makeGroup({
+        id: CHILD_GROUP,
+        name: "Region 1000",
+        parentId: PARENT_GROUP,
+        showUptimePercent: true,
+      });
+
+      return {
+        groups: [parent, child],
+        resources: [
+          makeResource({
+            monitorId: MONITOR_A,
+            currentStatusId: OPERATIONAL_STATUS,
+            groupId: PARENT_GROUP,
+          }),
+          makeResource({
+            monitorId: MONITOR_B,
+            currentStatusId: OPERATIONAL_STATUS,
+            groupId: PARENT_GROUP,
+          }),
+          makeResource({
+            monitorId: MONITOR_C,
+            currentStatusId: OFFLINE_STATUS,
+            groupId: CHILD_GROUP,
+          }),
+        ],
+        timelines: [
+          makeTimeline({ monitorId: MONITOR_A, status: operationalStatus() }),
+          makeTimeline({ monitorId: MONITOR_B, status: operationalStatus() }),
+          makeTimeline({ monitorId: MONITOR_C, status: offlineStatus() }),
+        ],
+        statuses: [operationalStatus(), offlineStatus()],
+      };
+    }
+
+    describe("getResourcesInStatusPageGroupAndDescendants", () => {
+      test("a parent covers its own resources and its children's", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture();
+
+        expect(
+          StatusPageResourceUptimeUtil.getResourcesInStatusPageGroupAndDescendants(
+            {
+              statusPageGroup: fixture.groups[0]!,
+              statusPageResources: fixture.resources,
+              allStatusPageGroups: fixture.groups,
+            },
+          ),
+        ).toHaveLength(3);
+      });
+
+      test("a child covers only its own resources", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture();
+
+        const resources: Array<StatusPageResource> =
+          StatusPageResourceUptimeUtil.getResourcesInStatusPageGroupAndDescendants(
+            {
+              statusPageGroup: fixture.groups[1]!,
+              statusPageResources: fixture.resources,
+              allStatusPageGroups: fixture.groups,
+            },
+          );
+
+        expect(resources).toHaveLength(1);
+        expect(resources[0]!.monitorId?.toString()).toBe(MONITOR_C.toString());
+      });
+
+      test("without the group list it falls back to the group's own resources", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture();
+
+        expect(
+          StatusPageResourceUptimeUtil.getResourcesInStatusPageGroupAndDescendants(
+            {
+              statusPageGroup: fixture.groups[0]!,
+              statusPageResources: fixture.resources,
+            },
+          ),
+        ).toHaveLength(2);
+      });
+
+      test("never picks up ungrouped resources", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture();
+
+        const ungrouped: StatusPageResource = makeResource({
+          monitorId: MONITOR_A,
+          currentStatusId: OPERATIONAL_STATUS,
+        });
+
+        expect(
+          StatusPageResourceUptimeUtil.getResourcesInStatusPageGroupAndDescendants(
+            {
+              statusPageGroup: fixture.groups[0]!,
+              statusPageResources: [...fixture.resources, ungrouped],
+              allStatusPageGroups: fixture.groups,
+            },
+          ),
+        ).toHaveLength(3);
+      });
+    });
+
+    describe("calculateAvgUptimePercentOfStatusPageGroup", () => {
+      test("a parent averages every resource in its subtree", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture();
+
+        expect(
+          StatusPageResourceUptimeUtil.calculateAvgUptimePercentOfStatusPageGroup(
+            {
+              statusPageGroup: fixture.groups[0]!,
+              monitorStatusTimelines: fixture.timelines,
+              precision: UptimePrecision.TWO_DECIMAL,
+              downtimeMonitorStatuses: [offlineStatus()],
+              statusPageResources: fixture.resources,
+              monitorsInGroup: {},
+              uptimeWindow: uptimeWindow,
+              allStatusPageGroups: fixture.groups,
+            },
+          ),
+        ).toBe(66.66);
+      });
+
+      test("a child reports only its own resources", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture();
+
+        expect(
+          StatusPageResourceUptimeUtil.calculateAvgUptimePercentOfStatusPageGroup(
+            {
+              statusPageGroup: fixture.groups[1]!,
+              monitorStatusTimelines: fixture.timelines,
+              precision: UptimePrecision.TWO_DECIMAL,
+              downtimeMonitorStatuses: [offlineStatus()],
+              statusPageResources: fixture.resources,
+              monitorsInGroup: {},
+              uptimeWindow: uptimeWindow,
+              allStatusPageGroups: fixture.groups,
+            },
+          ),
+        ).toBe(0);
+      });
+
+      test("without the group list a parent reports only its own resources", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture();
+
+        expect(
+          StatusPageResourceUptimeUtil.calculateAvgUptimePercentOfStatusPageGroup(
+            {
+              statusPageGroup: fixture.groups[0]!,
+              monitorStatusTimelines: fixture.timelines,
+              precision: UptimePrecision.TWO_DECIMAL,
+              downtimeMonitorStatuses: [offlineStatus()],
+              statusPageResources: fixture.resources,
+              monitorsInGroup: {},
+              uptimeWindow: uptimeWindow,
+            },
+          ),
+        ).toBe(100);
+      });
+
+      test("a group that does not show uptime percent reports nothing", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture({
+          parentShowsUptimePercent: false,
+        });
+
+        expect(
+          StatusPageResourceUptimeUtil.calculateAvgUptimePercentOfStatusPageGroup(
+            {
+              statusPageGroup: fixture.groups[0]!,
+              monitorStatusTimelines: fixture.timelines,
+              precision: UptimePrecision.TWO_DECIMAL,
+              downtimeMonitorStatuses: [offlineStatus()],
+              statusPageResources: fixture.resources,
+              monitorsInGroup: {},
+              uptimeWindow: uptimeWindow,
+              allStatusPageGroups: fixture.groups,
+            },
+          ),
+        ).toBeNull();
+      });
+    });
+
+    describe("getCurrentStatusPageGroupStatus", () => {
+      test("a parent takes the worst status in its subtree", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture();
+
+        const status: MonitorStatus =
+          StatusPageResourceUptimeUtil.getCurrentStatusPageGroupStatus({
+            statusPageGroup: fixture.groups[0]!,
+            monitorStatusTimelines: fixture.timelines,
+            statusPageResources: fixture.resources,
+            monitorStatuses: fixture.statuses,
+            monitorGroupCurrentStatuses: {},
+            allStatusPageGroups: fixture.groups,
+          });
+
+        expect(status.name).toBe("Offline");
+      });
+
+      test("without the group list a parent only sees its own resources", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture();
+
+        const status: MonitorStatus =
+          StatusPageResourceUptimeUtil.getCurrentStatusPageGroupStatus({
+            statusPageGroup: fixture.groups[0]!,
+            monitorStatusTimelines: fixture.timelines,
+            statusPageResources: fixture.resources,
+            monitorStatuses: fixture.statuses,
+            monitorGroupCurrentStatuses: {},
+          });
+
+        expect(status.name).toBe("Operational");
+      });
+    });
+
+    describe("calculateAvgUptimePercentageOfAllResources", () => {
+      test("counts a nested resource once, through its top most reporting ancestor", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture();
+
+        /*
+         * The parent already averages all three resources (66.66%). Averaging
+         * the parent and the child together would count the offline resource
+         * twice and land on 33.33%.
+         */
+        expect(
+          StatusPageResourceUptimeUtil.calculateAvgUptimePercentageOfAllResources(
+            {
+              monitorStatusTimelines: fixture.timelines,
+              precision: UptimePrecision.TWO_DECIMAL,
+              downtimeMonitorStatuses: [offlineStatus()],
+              statusPageResources: fixture.resources,
+              resourceGroups: fixture.groups,
+              monitorsInGroup: {},
+              uptimeWindow: uptimeWindow,
+            },
+          ),
+        ).toBe(66.66);
+      });
+
+      test("descends past a parent that does not report uptime percent", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture({
+          parentShowsUptimePercent: false,
+        });
+
+        // only the child reports, and everything in the child is offline.
+        expect(
+          StatusPageResourceUptimeUtil.calculateAvgUptimePercentageOfAllResources(
+            {
+              monitorStatusTimelines: fixture.timelines,
+              precision: UptimePrecision.TWO_DECIMAL,
+              downtimeMonitorStatuses: [offlineStatus()],
+              statusPageResources: fixture.resources,
+              resourceGroups: fixture.groups,
+              monitorsInGroup: {},
+              uptimeWindow: uptimeWindow,
+            },
+          ),
+        ).toBe(0);
+      });
+
+      test("still includes resources that are in no group at all", () => {
+        const fixture: ReturnType<typeof makeFixture> = makeFixture();
+
+        const ungrouped: StatusPageResource = makeResource({
+          monitorId: MONITOR_C,
+          currentStatusId: OFFLINE_STATUS,
+        });
+
+        // parent subtree 66.66%, ungrouped offline resource 0% -> 33.33%.
+        expect(
+          StatusPageResourceUptimeUtil.calculateAvgUptimePercentageOfAllResources(
+            {
+              monitorStatusTimelines: fixture.timelines,
+              precision: UptimePrecision.TWO_DECIMAL,
+              downtimeMonitorStatuses: [offlineStatus()],
+              statusPageResources: [...fixture.resources, ungrouped],
+              resourceGroups: fixture.groups,
+              monitorsInGroup: {},
+              uptimeWindow: uptimeWindow,
+            },
+          ),
+        ).toBe(33.33);
+      });
+    });
+  });
 });

package/Types/Date.ts

@@ -341,7 +341,10 @@ export default class OneUptimeDate {
       const local: moment.Moment = this.inCurrentTimezone(date);
       return (
         local.format(formatString) +
-        (data.onlyShowDate ? "" : " " + this.getCurrentTimezoneString())
+        (data.onlyShowDate
+          ? ""
+          : // Same DST-aware lookup the explicit-timezone branch below uses.
+            " " + this.getZoneAbbrByTimezone(this.getCurrentTimezone(), date))
       ).trim();
     }
 
@@ -1746,11 +1749,17 @@ export default class OneUptimeDate {
 
     const momentDate: moment.Moment = this.inCurrentTimezone(date);
 
-    return (
-      momentDate.format(formatstring) +
-      " " +
-      (onlyShowDate ? "" : this.getCurrentTimezoneString())
-    ).trim();
+    /*
+     * The abbreviation has to describe THIS date's offset, not today's. Asking
+     * for it without the date labelled every January timestamp "EDT" all
+     * summer — a wall clock paired with the wrong zone names a different
+     * instant, which is worse than printing no zone at all.
+     */
+    const zoneAbbr: string = onlyShowDate
+      ? ""
+      : this.getZoneAbbrByTimezone(this.getCurrentTimezone(), date);
+
+    return (momentDate.format(formatstring) + " " + zoneAbbr).trim();
   }
 
   public static getDayInSeconds(days?: number | undefined): number {

package/Types/Monitor/KubernetesAlertTemplates.ts

@@ -959,6 +959,208 @@ const nodeMemoryRequestUtilizationTemplate: KubernetesAlertTemplate = {
   },
 };
 
+/*
+ * --- Autoscaling / limit-saturation templates ---
+ *
+ * These three close the detection gap for the classic "under-resourced
+ * workload behind an autoscaler" failure: limits are set too low, so the
+ * containers sit pinned against them (OOMKilled on memory, CFS-throttled
+ * on CPU), the resulting latency/restarts drive the HPA up, and the HPA
+ * fills the cluster until nodes go NotReady.
+ *
+ * The node-side templates above catch the END of that chain (high node
+ * CPU/memory, NotReady, pending pods) — by which point the RCA is several
+ * hops from the cause. These catch the START: the HPA running out of
+ * headroom, and the containers pinned at their own limits.
+ *
+ * All three are per-series ratios, grouped by the ClickHouse-stored
+ * `resource.`-prefixed attribute (see buildKubernetesRatioMonitorConfig).
+ */
+
+const hpaAtMaxReplicasTemplate: KubernetesAlertTemplate = {
+  id: "k8s-hpa-at-max-replicas",
+  name: "HPA Saturated at Max Replicas",
+  description:
+    "Alert when a HorizontalPodAutoscaler is running at 90% or more of its maxReplicas. Computed per HPA as k8s.hpa.current_replicas ÷ k8s.hpa.max_replicas × 100. An HPA at its ceiling has no headroom left: load it cannot absorb by scaling turns straight into latency and errors, and a workload that reaches the ceiling and stays there is usually under-resourced per pod rather than genuinely at capacity.",
+  category: "Workload",
+  severity: "Critical",
+  getMonitorStep: (args: KubernetesAlertTemplateArgs): MonitorStep => {
+    const metricAlias: string = "hpa_replica_saturation";
+
+    return buildKubernetesMonitorStep({
+      kubernetesMonitor: buildKubernetesRatioMonitorConfig({
+        clusterIdentifier: args.clusterIdentifier,
+        numeratorMetricName: "k8s.hpa.current_replicas",
+        denominatorMetricName: "k8s.hpa.max_replicas",
+        groupByAttributeKey: "resource.k8s.hpa.name",
+        numeratorAlias: "current_replicas",
+        denominatorAlias: "max_replicas",
+        resultAlias: metricAlias,
+        resultLegend: "HPA Replica Saturation (%)",
+        resourceScope: KubernetesResourceScope.Workload,
+        rollingTime: RollingTime.Past5Minutes,
+        /*
+         * ONE series per HPA on both sides (they are the same object's
+         * status/spec fields), so Avg gives the representative per-minute
+         * ratio independent of scrape count. Both come from the same
+         * k8s_cluster scrape, so Sum would also cancel — but Avg stays
+         * correct across restarts and missed scrapes. See
+         * buildKubernetesRatioMonitorConfig.
+         */
+        aggregationType: MetricsAggregationType.Avg,
+      }),
+      offlineCriteriaInstance: buildOfflineCriteriaInstance({
+        offlineMonitorStatusId: args.offlineMonitorStatusId,
+        incidentSeverityId: args.defaultIncidentSeverityId,
+        alertSeverityId: args.defaultAlertSeverityId,
+        monitorName: args.monitorName,
+        metricAlias,
+        filterType: FilterType.GreaterThanOrEqualTo,
+        value: 90,
+        incidentTitle: `[K8s] HPA Saturated at Max Replicas (>=90%) - ${args.monitorName}`,
+        incidentDescription: `A HorizontalPodAutoscaler is running at 90% or more of its maxReplicas and has effectively no scaling headroom left. Any further load cannot be absorbed by scaling out, so it will surface as latency and errors instead. Check whether the workload is genuinely at capacity or whether its per-pod CPU/memory limits are set too low — an under-resourced pod gets throttled or OOMKilled, which inflates the metric the HPA scales on and drives it to the ceiling. Check the root cause for the specific HPA, its target workload, and current vs max replicas.`,
+        criteriaName: "HPA Saturation - Current/Max Replicas >= 90%",
+        criteriaDescription:
+          "Triggers when any HPA's current replica count reaches 90% or more of its configured maxReplicas.",
+      }),
+      onlineCriteriaInstance: buildOnlineCriteriaInstance({
+        onlineMonitorStatusId: args.onlineMonitorStatusId,
+        metricAlias,
+        filterType: FilterType.LessThan,
+        value: 90,
+      }),
+    });
+  },
+};
+
+const podMemoryLimitSaturationTemplate: KubernetesAlertTemplate = {
+  id: "k8s-pod-memory-limit-saturation",
+  name: "Pod Memory Saturating Container Limit",
+  description:
+    "Alert when a pod's memory usage exceeds 90% of its configured container memory limit — the state immediately preceding an OOMKill. Computed per pod as k8s.pod.memory.usage ÷ k8s.container.memory_limit × 100 (both bytes). This is the cause-side signal for CrashLoopBackOff and restart storms: a limit set too low shows up here minutes before the container is killed.",
+  category: "Workload",
+  severity: "Critical",
+  getMonitorStep: (args: KubernetesAlertTemplateArgs): MonitorStep => {
+    const metricAlias: string = "pod_memory_limit_saturation";
+
+    return buildKubernetesMonitorStep({
+      kubernetesMonitor: buildKubernetesRatioMonitorConfig({
+        clusterIdentifier: args.clusterIdentifier,
+        numeratorMetricName: "k8s.pod.memory.usage",
+        denominatorMetricName: "k8s.container.memory_limit",
+        groupByAttributeKey: "resource.k8s.pod.name",
+        numeratorAlias: "used_mem",
+        denominatorAlias: "limit_mem",
+        resultAlias: metricAlias,
+        resultLegend: "Pod Memory vs Limit (%)",
+        resourceScope: KubernetesResourceScope.Pod,
+        rollingTime: RollingTime.Past5Minutes,
+        /*
+         * Avg, deliberately — and the one case in this file where the two
+         * sides have different series shapes, so the trade-off is worth
+         * stating.
+         *
+         * Numerator (k8s.pod.memory.usage, kubeletstats) is ONE series per
+         * pod. Denominator (k8s.container.memory_limit, k8s_cluster) is one
+         * series per CONTAINER, so a multi-container pod contributes
+         * several.
+         *
+         * Sum is definitively wrong here: the two metrics ride different
+         * receivers on independent scrape cycles, so the scrape multiple
+         * would not cancel. Avg is exact for single-container pods (the
+         * overwhelming majority, and the case this template exists for).
+         *
+         * For multi-container pods Avg takes the MEAN container limit
+         * rather than their sum, so the ratio over-reports and the alert
+         * fires early. That is the safe direction for an "OOMKill is
+         * imminent" warning — a false early page beats a missed kill — but
+         * it is a real caveat, not a rounding detail. A per-container
+         * variant needs a container-scoped usage metric to pair against
+         * (`container.memory.usage`), which the shipped catalog does not
+         * carry today.
+         */
+        aggregationType: MetricsAggregationType.Avg,
+      }),
+      offlineCriteriaInstance: buildOfflineCriteriaInstance({
+        offlineMonitorStatusId: args.offlineMonitorStatusId,
+        incidentSeverityId: args.defaultIncidentSeverityId,
+        alertSeverityId: args.defaultAlertSeverityId,
+        monitorName: args.monitorName,
+        metricAlias,
+        filterType: FilterType.GreaterThan,
+        value: 90,
+        incidentTitle: `[K8s] Pod Memory Saturating Container Limit (>90%) - ${args.monitorName}`,
+        incidentDescription: `A pod's memory usage has exceeded 90% of its configured container memory limit. The kubelet OOMKills a container the moment it crosses its limit, so this is the state immediately preceding a restart — and, if the workload sits behind an autoscaler, the start of a restart/scale-up loop. Either the limit is set too low for the workload's real footprint or the workload has a memory leak. Check the root cause for the specific pod, its limit, and its usage trend.`,
+        criteriaName: "Pod Memory Saturation - Usage/Limit > 90%",
+        criteriaDescription:
+          "Triggers when any pod's memory usage exceeds 90% of its container memory limit.",
+      }),
+      onlineCriteriaInstance: buildOnlineCriteriaInstance({
+        onlineMonitorStatusId: args.onlineMonitorStatusId,
+        metricAlias,
+        filterType: FilterType.LessThanOrEqualTo,
+        value: 90,
+      }),
+    });
+  },
+};
+
+const podCpuLimitSaturationTemplate: KubernetesAlertTemplate = {
+  id: "k8s-pod-cpu-limit-saturation",
+  name: "Pod CPU Saturating Container Limit",
+  description:
+    "Alert when a pod's CPU usage exceeds 90% of its configured container CPU limit — the point at which the kernel's CFS quota starts throttling it. Computed per pod as k8s.pod.cpu.utilization ÷ k8s.container.cpu_limit × 100; both are CPU cores, so this is a true percentage (k8s.pod.cpu.utilization is a misnamed cores gauge, not a percent). A throttled pod gets slower, not louder — behind an HPA that reads CPU, throttling drives the replica count up while every pod stays equally starved.",
+  category: "Workload",
+  severity: "Warning",
+  getMonitorStep: (args: KubernetesAlertTemplateArgs): MonitorStep => {
+    const metricAlias: string = "pod_cpu_limit_saturation";
+
+    return buildKubernetesMonitorStep({
+      kubernetesMonitor: buildKubernetesRatioMonitorConfig({
+        clusterIdentifier: args.clusterIdentifier,
+        numeratorMetricName: "k8s.pod.cpu.utilization",
+        denominatorMetricName: "k8s.container.cpu_limit",
+        groupByAttributeKey: "resource.k8s.pod.name",
+        numeratorAlias: "used_cpu",
+        denominatorAlias: "limit_cpu",
+        resultAlias: metricAlias,
+        resultLegend: "Pod CPU vs Limit (%)",
+        resourceScope: KubernetesResourceScope.Pod,
+        rollingTime: RollingTime.Past5Minutes,
+        /*
+         * Avg, for the same reason as the memory template above: ONE
+         * numerator series per pod (kubeletstats) against a per-container
+         * denominator (k8s_cluster) on an independent scrape cycle. Exact
+         * for single-container pods; over-reports (fires early) for
+         * multi-container pods. See that template's note for the full
+         * trade-off.
+         */
+        aggregationType: MetricsAggregationType.Avg,
+      }),
+      offlineCriteriaInstance: buildOfflineCriteriaInstance({
+        offlineMonitorStatusId: args.offlineMonitorStatusId,
+        incidentSeverityId: args.defaultIncidentSeverityId,
+        alertSeverityId: args.defaultAlertSeverityId,
+        monitorName: args.monitorName,
+        metricAlias,
+        filterType: FilterType.GreaterThan,
+        value: 90,
+        incidentTitle: `[K8s] Pod CPU Saturating Container Limit (>90%) - ${args.monitorName}`,
+        incidentDescription: `A pod's CPU usage has exceeded 90% of its configured container CPU limit and is being throttled by the kernel's CFS quota. Throttling is silent — the pod does not crash, it just gets slower, so this usually surfaces as request latency rather than as an error. If the workload sits behind a CPU-based HorizontalPodAutoscaler, throttling also inflates the metric the HPA scales on, so the autoscaler adds replicas that are each equally starved. Check whether the CPU limit is set too low for the workload rather than adding replicas.`,
+        criteriaName: "Pod CPU Saturation - Usage/Limit > 90%",
+        criteriaDescription:
+          "Triggers when any pod's CPU usage exceeds 90% of its container CPU limit.",
+      }),
+      onlineCriteriaInstance: buildOnlineCriteriaInstance({
+        onlineMonitorStatusId: args.onlineMonitorStatusId,
+        metricAlias,
+        filterType: FilterType.LessThanOrEqualTo,
+        value: 90,
+      }),
+    });
+  },
+};
+
 export function getAllKubernetesAlertTemplates(): Array<KubernetesAlertTemplate> {
   return [
     crashLoopBackOffTemplate,
@@ -975,6 +1177,9 @@ export function getAllKubernetesAlertTemplates(): Array<KubernetesAlertTemplate>
     daemonSetUnavailableTemplate,
     nodeCpuRequestUtilizationTemplate,
     nodeMemoryRequestUtilizationTemplate,
+    hpaAtMaxReplicasTemplate,
+    podMemoryLimitSaturationTemplate,
+    podCpuLimitSaturationTemplate,
   ];
 }