@oneuptime/common 10.0.68 → 10.0.70

package/Models/DatabaseModels/KubernetesCluster.ts

@@ -73,6 +73,11 @@ import {
 })
 @CrudApiEndpoint(new Route("/kubernetes-cluster"))
 @SlugifyColumn("name", "slug")
+// Enforce one cluster row per (projectId, clusterIdentifier) at the DB level.
+// Without this, two pods emitting OTel telemetry for a new cluster at the
+// same time (e.g. when the agent is first installed or during a rolling
+// update) race in findOrCreateByClusterIdentifier and create duplicate rows.
+@Index(["projectId", "clusterIdentifier"], { unique: true })
 @TableMetadata({
   tableName: "KubernetesCluster",
   singularName: "Kubernetes Cluster",

package/Models/DatabaseModels/KubernetesResource.ts

@@ -405,6 +405,25 @@ export default class KubernetesResource extends BaseModel {
   })
   public spec?: JSONObject = undefined;
 
+  @ColumnAccessControl({
+    create: [],
+    read: READ_PERMISSIONS,
+    update: [],
+  })
+  @TableColumn({
+    required: false,
+    type: TableColumnType.Number,
+    canReadOnRelationQuery: true,
+    title: "Container Count",
+    description:
+      "For Pods: count of entries in spec.containers, cached so the overview page can SUM it without scanning JSONB. Null for non-Pod kinds.",
+  })
+  @Column({
+    nullable: true,
+    type: ColumnType.Number,
+  })
+  public containerCount?: number = undefined;
+
   @ColumnAccessControl({
     create: [],
     read: READ_PERMISSIONS,

package/Server/API/KubernetesResourceAPI.ts

@@ -122,6 +122,8 @@ export default class KubernetesResourceAPI extends BaseAPI<
       hpaCount: summary.countsByKind["HorizontalPodAutoscaler"] || 0,
       vpaCount: summary.countsByKind["VerticalPodAutoscaler"] || 0,
       containerCount: summary.containerCount,
+      degradedPods: summary.degradedPods as unknown as JSONObject,
+      degradedNodes: summary.degradedNodes as unknown as JSONObject,
     };
 
     return Response.sendJsonObjectResponse(req, res, responseBody);

package/Server/Infrastructure/Postgres/SchemaMigrations/1776865086264-MigrationName.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1776865086264 implements MigrationInterface {
+  public name: string = "MigrationName1776865086264";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" ADD "containerCount" integer`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" DROP COLUMN "containerCount"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1776881254913-DedupeKubernetesClustersAndAddUniqueIndex.ts

@@ -0,0 +1,134 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Before this migration, KubernetesCluster had an app-level
+ * @UniqueColumnBy("projectId") check and a non-unique index on
+ * clusterIdentifier, but no DB-level uniqueness. Under concurrent telemetry
+ * from multiple agent pods (happens every time the agent is installed or
+ * rolls out), findOrCreateByClusterIdentifier would race between its find
+ * and its create, and the DB accepted both inserts — producing duplicate
+ * rows with identical (projectId, clusterIdentifier).
+ *
+ * This migration:
+ *   1. Reparents all FKs that reference duplicate clusters — KubernetesResource,
+ *      KubernetesClusterOwnerUser, KubernetesClusterOwnerTeam — onto the
+ *      oldest surviving row in each duplicate group.
+ *   2. Deletes the duplicate (non-survivor) rows.
+ *   3. Creates a DB-level unique index on (projectId, clusterIdentifier) so
+ *      future races are rejected by the DB — the service's existing
+ *      catch-and-refetch in findOrCreateByClusterIdentifier then returns the
+ *      winning row instead of producing a duplicate.
+ *
+ * The auto-generator also picked up unrelated OnCallDutyPolicyScheduleLayer
+ * default-value drift. That's dev-environment drift, not the bug we're fixing;
+ * stripped from this migration.
+ */
+export class DedupeKubernetesClustersAndAddUniqueIndex1776881254913
+  implements MigrationInterface
+{
+  public name: string = "DedupeKubernetesClustersAndAddUniqueIndex1776881254913";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    // 1: reparent KubernetesResource FKs from duplicates -> survivor.
+    await queryRunner.query(`
+      WITH survivors AS (
+        SELECT DISTINCT ON ("projectId", "clusterIdentifier")
+          _id AS survivor_id,
+          "projectId",
+          "clusterIdentifier"
+        FROM "KubernetesCluster"
+        ORDER BY "projectId", "clusterIdentifier", "createdAt" ASC, _id ASC
+      ),
+      losers AS (
+        SELECT kc._id AS loser_id, s.survivor_id
+        FROM "KubernetesCluster" kc
+        JOIN survivors s
+          ON s."projectId" = kc."projectId"
+         AND s."clusterIdentifier" = kc."clusterIdentifier"
+        WHERE kc._id <> s.survivor_id
+      )
+      UPDATE "KubernetesResource" kr
+      SET "kubernetesClusterId" = l.survivor_id
+      FROM losers l
+      WHERE kr."kubernetesClusterId" = l.loser_id;
+    `);
+
+    // 2: reparent KubernetesClusterOwnerUser FKs.
+    await queryRunner.query(`
+      WITH survivors AS (
+        SELECT DISTINCT ON ("projectId", "clusterIdentifier")
+          _id AS survivor_id,
+          "projectId",
+          "clusterIdentifier"
+        FROM "KubernetesCluster"
+        ORDER BY "projectId", "clusterIdentifier", "createdAt" ASC, _id ASC
+      ),
+      losers AS (
+        SELECT kc._id AS loser_id, s.survivor_id
+        FROM "KubernetesCluster" kc
+        JOIN survivors s
+          ON s."projectId" = kc."projectId"
+         AND s."clusterIdentifier" = kc."clusterIdentifier"
+        WHERE kc._id <> s.survivor_id
+      )
+      UPDATE "KubernetesClusterOwnerUser" o
+      SET "kubernetesClusterId" = l.survivor_id
+      FROM losers l
+      WHERE o."kubernetesClusterId" = l.loser_id;
+    `);
+
+    // 3: reparent KubernetesClusterOwnerTeam FKs.
+    await queryRunner.query(`
+      WITH survivors AS (
+        SELECT DISTINCT ON ("projectId", "clusterIdentifier")
+          _id AS survivor_id,
+          "projectId",
+          "clusterIdentifier"
+        FROM "KubernetesCluster"
+        ORDER BY "projectId", "clusterIdentifier", "createdAt" ASC, _id ASC
+      ),
+      losers AS (
+        SELECT kc._id AS loser_id, s.survivor_id
+        FROM "KubernetesCluster" kc
+        JOIN survivors s
+          ON s."projectId" = kc."projectId"
+         AND s."clusterIdentifier" = kc."clusterIdentifier"
+        WHERE kc._id <> s.survivor_id
+      )
+      UPDATE "KubernetesClusterOwnerTeam" o
+      SET "kubernetesClusterId" = l.survivor_id
+      FROM losers l
+      WHERE o."kubernetesClusterId" = l.loser_id;
+    `);
+
+    // 4: delete duplicate rows now that nothing references them.
+    await queryRunner.query(`
+      WITH survivors AS (
+        SELECT DISTINCT ON ("projectId", "clusterIdentifier")
+          _id AS survivor_id,
+          "projectId",
+          "clusterIdentifier"
+        FROM "KubernetesCluster"
+        ORDER BY "projectId", "clusterIdentifier", "createdAt" ASC, _id ASC
+      )
+      DELETE FROM "KubernetesCluster" kc
+      USING survivors s
+      WHERE s."projectId" = kc."projectId"
+        AND s."clusterIdentifier" = kc."clusterIdentifier"
+        AND kc._id <> s.survivor_id;
+    `);
+
+    // 5: add the DB-level composite unique index.
+    await queryRunner.query(
+      `CREATE UNIQUE INDEX "IDX_9756988b48848f4f7532a2af0d" ON "KubernetesCluster" ("projectId", "clusterIdentifier") `,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_9756988b48848f4f7532a2af0d"`,
+    );
+    // Duplicate rows dropped in up() are lost — a down-migration cannot
+    // resurrect them (and reinstating duplicates is not desirable anyway).
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -287,6 +287,8 @@ import { MigrationName1776541018853 } from "./1776541018853-MigrationName";
 import { MigrationName1776544084793 } from "./1776544084793-MigrationName";
 import { MigrationName1776761171349 } from "./1776761171349-MigrationName";
 import { MigrationName1776801030808 } from "./1776801030808-MigrationName";
+import { MigrationName1776865086264 } from "./1776865086264-MigrationName";
+import { DedupeKubernetesClustersAndAddUniqueIndex1776881254913 } from "./1776881254913-DedupeKubernetesClustersAndAddUniqueIndex";
 export default [
   InitialMigration,
   MigrationName1717678334852,
@@ -577,4 +579,6 @@ export default [
   MigrationName1776544084793,
   MigrationName1776761171349,
   MigrationName1776801030808,
+  MigrationName1776865086264,
+  DedupeKubernetesClustersAndAddUniqueIndex1776881254913,
 ];

package/Server/Services/DatabaseService.ts

@@ -31,7 +31,6 @@ import UpdateByIDAndFetch from "../Types/Database/UpdateByIDAndFetch";
 import UpdateOneBy from "../Types/Database/UpdateOneBy";
 import Encryption from "../Utils/Encryption";
 import logger, { LogAttributes } from "../Utils/Logger";
-import AuditLogService from "./AuditLogService";
 import BaseService from "./BaseService";
 import BaseModel from "../../Models/DatabaseModels/DatabaseBaseModel/DatabaseBaseModel";
 import { WorkflowRoute } from "../../ServiceRoute";
@@ -70,6 +69,7 @@ import { FindWhere } from "../../Types/BaseDatabase/Query";
 import Realtime from "../Utils/Realtime";
 import ModelEventType from "../../Types/Realtime/ModelEventType";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import type AuditLogServiceType from "./AuditLogService";
 
 class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
   public modelType!: { new (): TBaseModel };
@@ -777,7 +777,16 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
         !createBy.props.ignoreHooks &&
         this.getModel().enableAuditLogOn?.create
       ) {
-        await AuditLogService.recordCreate({
+        /*
+         * Lazy require to avoid circular dependency between DatabaseService and
+         * AuditLogService (which depends on ProjectService/UserService, both of
+         * which extend DatabaseService). A top-level import leaves
+         * DatabaseService undefined at class-extension time for subclasses.
+         */
+        const auditLogService: typeof AuditLogServiceType =
+          // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+          require("./AuditLogService").default;
+        await auditLogService.recordCreate({
           model: this.getModel(),
           createdItem: createBy.data,
           props: createBy.props,
@@ -1224,9 +1233,12 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
       }
 
       if (this.getModel().enableAuditLogOn?.delete && items.length > 0) {
+        const auditLogService: typeof AuditLogServiceType =
+          // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+          require("./AuditLogService").default;
         for (const item of items) {
           if (item.id) {
-            await AuditLogService.recordDelete({
+            await auditLogService.recordDelete({
               model: this.getModel(),
               deletedItem: item,
               itemId: item.id,
@@ -1645,7 +1657,10 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
           !this.hasSameValues({ item, updatedItem }) &&
           item.id
         ) {
-          await AuditLogService.recordUpdate({
+          const auditLogService: typeof AuditLogServiceType =
+            // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+            require("./AuditLogService").default;
+          await auditLogService.recordUpdate({
             model: this.getModel(),
             before: item,
             updatedFields: data as JSONObject,

package/Server/Services/KubernetesResourceService.ts

@@ -22,10 +22,28 @@ import logger from "../Utils/Logger";
 
 export type { ParsedKubernetesResource };
 
+export interface DegradedPod {
+  name: string;
+  namespace: string;
+  phase: string;
+  reason: string;
+  message: string;
+}
+
+export interface DegradedNode {
+  name: string;
+  isReady: boolean;
+  hasMemoryPressure: boolean;
+  hasDiskPressure: boolean;
+  hasPidPressure: boolean;
+  reason: string;
+  message: string;
+}
+
 export interface InventorySummary {
   countsByKind: Record<string, number>;
   /*
-   * Sum of `jsonb_array_length(spec->'containers')` across all pods in
+   * Sum of the denormalized containerCount column across all pods in
    * the cluster. Containers aren't a top-level kind in the inventory,
    * so we derive the total server-side so the sidebar badge and the
    * Containers page agree.
@@ -47,6 +65,208 @@ export interface InventorySummary {
     diskPressure: number;
     pidPressure: number;
   };
+  /*
+   * Top offenders that explain a Degraded/Unhealthy cluster state. Capped
+   * so a pathological cluster can't blow up the overview payload; the
+   * dedicated Pods/Nodes pages are the source of truth for the full list.
+   */
+  degradedPods: Array<DegradedPod>;
+  degradedNodes: Array<DegradedNode>;
+}
+
+const DEGRADED_SAMPLE_LIMIT: number = 20;
+
+/*
+ * Pull the first meaningful reason/message off a pod's status block.
+ * KubernetesInventoryExtractor stores containerStatuses as an array of
+ * { name, ready, state: "running"|"waiting"|"terminated", reason, message, ... }.
+ * A waiting container with a reason (ImagePullBackOff, CrashLoopBackOff,
+ * CreateContainerConfigError, ...) is exactly what the user needs to see,
+ * so we surface that first. We fall back to terminated reasons (OOMKilled,
+ * Error, ContainerCannotRun) and then to status-level conditions.
+ */
+function buildDegradedPod(row: {
+  name: string;
+  namespaceKey: string;
+  phase: string | null;
+  status: unknown;
+}): DegradedPod {
+  const status: Record<string, unknown> =
+    row.status && typeof row.status === "object"
+      ? (row.status as Record<string, unknown>)
+      : {};
+
+  let reason: string = "";
+  let message: string = "";
+
+  const containerStatuses: Array<Record<string, unknown>> = Array.isArray(
+    status["containerStatuses"],
+  )
+    ? (status["containerStatuses"] as Array<Record<string, unknown>>)
+    : [];
+  const initContainerStatuses: Array<Record<string, unknown>> = Array.isArray(
+    status["initContainerStatuses"],
+  )
+    ? (status["initContainerStatuses"] as Array<Record<string, unknown>>)
+    : [];
+
+  const scanForReason: (
+    list: Array<Record<string, unknown>>,
+    targetState: string,
+  ) => { reason: string; message: string } | null = (
+    list: Array<Record<string, unknown>>,
+    targetState: string,
+  ) => {
+    for (const cs of list) {
+      if (cs["state"] !== targetState) {
+        continue;
+      }
+      const r: unknown = cs["reason"];
+      if (typeof r === "string" && r) {
+        const m: unknown = cs["message"];
+        return {
+          reason: r,
+          message: typeof m === "string" ? m : "",
+        };
+      }
+    }
+    return null;
+  };
+
+  const waitingHit: { reason: string; message: string } | null =
+    scanForReason(containerStatuses, "waiting") ||
+    scanForReason(initContainerStatuses, "waiting");
+  const terminatedHit: { reason: string; message: string } | null = waitingHit
+    ? null
+    : scanForReason(containerStatuses, "terminated") ||
+      scanForReason(initContainerStatuses, "terminated");
+  const hit: { reason: string; message: string } | null =
+    waitingHit || terminatedHit;
+
+  if (hit) {
+    reason = hit.reason;
+    message = hit.message;
+  } else {
+    /*
+     * Fall back to the pod-level reason/message fields set by the scheduler
+     * (e.g. "Unschedulable" with "0/3 nodes are available: ...").
+     */
+    const topReason: unknown = status["reason"];
+    const topMessage: unknown = status["message"];
+    if (typeof topReason === "string") {
+      reason = topReason;
+    }
+    if (typeof topMessage === "string") {
+      message = topMessage;
+    }
+
+    // If still nothing, pull from the first non-True condition.
+    if (!reason) {
+      const conditions: Array<Record<string, unknown>> = Array.isArray(
+        status["conditions"],
+      )
+        ? (status["conditions"] as Array<Record<string, unknown>>)
+        : [];
+      for (const cond of conditions) {
+        if (cond["status"] !== "True") {
+          const r: unknown = cond["reason"];
+          const m: unknown = cond["message"];
+          if (typeof r === "string" && r) {
+            reason = r;
+            message = typeof m === "string" ? m : "";
+            break;
+          }
+        }
+      }
+    }
+  }
+
+  return {
+    name: row.name,
+    namespace: row.namespaceKey || "",
+    phase: row.phase || "Unknown",
+    reason,
+    message,
+  };
+}
+
+/*
+ * For a Node: if isReady is false, the "Ready" condition carries the real
+ * story (e.g. "KubeletNotReady: PLEG is not healthy"). If only pressure
+ * flags are tripped, pick the tripped condition's reason/message.
+ */
+function buildDegradedNode(row: {
+  name: string;
+  isReady: boolean | null;
+  hasMemoryPressure: boolean | null;
+  hasDiskPressure: boolean | null;
+  hasPidPressure: boolean | null;
+  status: unknown;
+}): DegradedNode {
+  const status: Record<string, unknown> =
+    row.status && typeof row.status === "object"
+      ? (row.status as Record<string, unknown>)
+      : {};
+
+  const conditions: Array<Record<string, unknown>> = Array.isArray(
+    status["conditions"],
+  )
+    ? (status["conditions"] as Array<Record<string, unknown>>)
+    : [];
+
+  const findCondition: (
+    predicate: (c: Record<string, unknown>) => boolean,
+  ) => Record<string, unknown> | null = (
+    predicate: (c: Record<string, unknown>) => boolean,
+  ) => {
+    for (const c of conditions) {
+      if (predicate(c)) {
+        return c;
+      }
+    }
+    return null;
+  };
+
+  let picked: Record<string, unknown> | null = null;
+  if (row.isReady === false) {
+    picked = findCondition((c: Record<string, unknown>) => {
+      return c["type"] === "Ready" && c["status"] !== "True";
+    });
+  }
+  if (!picked && row.hasMemoryPressure === true) {
+    picked = findCondition((c: Record<string, unknown>) => {
+      return c["type"] === "MemoryPressure" && c["status"] === "True";
+    });
+  }
+  if (!picked && row.hasDiskPressure === true) {
+    picked = findCondition((c: Record<string, unknown>) => {
+      return c["type"] === "DiskPressure" && c["status"] === "True";
+    });
+  }
+  if (!picked && row.hasPidPressure === true) {
+    picked = findCondition((c: Record<string, unknown>) => {
+      return c["type"] === "PIDPressure" && c["status"] === "True";
+    });
+  }
+
+  const reason: string =
+    picked && typeof picked["reason"] === "string"
+      ? (picked["reason"] as string)
+      : "";
+  const message: string =
+    picked && typeof picked["message"] === "string"
+      ? (picked["message"] as string)
+      : "";
+
+  return {
+    name: row.name,
+    isReady: row.isReady === true,
+    hasMemoryPressure: row.hasMemoryPressure === true,
+    hasDiskPressure: row.hasDiskPressure === true,
+    hasPidPressure: row.hasPidPressure === true,
+    reason,
+    message,
+  };
 }
 
 const UPSERT_BATCH_SIZE: number = 500;
@@ -72,6 +292,7 @@ const UPSERT_COLUMNS: Array<keyof ParsedKubernetesResource | string> = [
   "annotations",
   "ownerReferences",
   "spec",
+  "containerCount",
   "status",
   "lastSeenAt",
   "resourceCreationTimestamp",
@@ -133,6 +354,7 @@ export class Service extends DatabaseService<Model> {
           r.annotations ? JSON.stringify(r.annotations) : null,
           r.ownerReferences ? JSON.stringify(r.ownerReferences) : null,
           r.spec ? JSON.stringify(r.spec) : null,
+          r.containerCount,
           r.status ? JSON.stringify(r.status) : null,
           r.lastSeenAt,
           r.resourceCreationTimestamp,
@@ -145,7 +367,7 @@ export class Service extends DatabaseService<Model> {
           "projectId", "kubernetesClusterId", "kind", "namespaceKey", "name",
           "uid", "phase", "isReady",
           "hasMemoryPressure", "hasDiskPressure", "hasPidPressure",
-          "labels", "annotations", "ownerReferences", "spec", "status",
+          "labels", "annotations", "ownerReferences", "spec", "containerCount", "status",
           "lastSeenAt", "resourceCreationTimestamp", "version"
         )
         VALUES ${valueFragments.join(", ")}
@@ -161,6 +383,7 @@ export class Service extends DatabaseService<Model> {
           "annotations" = EXCLUDED."annotations",
           "ownerReferences" = EXCLUDED."ownerReferences",
           "spec" = EXCLUDED."spec",
+          "containerCount" = EXCLUDED."containerCount",
           "status" = EXCLUDED."status",
           "lastSeenAt" = EXCLUDED."lastSeenAt",
           "resourceCreationTimestamp" = EXCLUDED."resourceCreationTimestamp",
@@ -218,7 +441,14 @@ export class Service extends DatabaseService<Model> {
     const manager: ReturnType<Service["getRepository"]>["manager"] =
       this.getRepository().manager;
 
-    const [kindRows, podRows, nodeRows, containerRows]: [
+    const [
+      kindRows,
+      podRows,
+      nodeRows,
+      containerRows,
+      degradedPodRows,
+      degradedNodeRows,
+    ]: [
       Array<{ kind: string; count: string }>,
       Array<{ phase: string | null; count: string }>,
       Array<{
@@ -229,6 +459,20 @@ export class Service extends DatabaseService<Model> {
         pidPressure: string;
       }>,
       Array<{ total: string }>,
+      Array<{
+        name: string;
+        namespaceKey: string;
+        phase: string | null;
+        status: unknown;
+      }>,
+      Array<{
+        name: string;
+        isReady: boolean | null;
+        hasMemoryPressure: boolean | null;
+        hasDiskPressure: boolean | null;
+        hasPidPressure: boolean | null;
+        status: unknown;
+      }>,
     ] = await Promise.all([
       manager.query(
         `SELECT "kind", COUNT(*)::text AS count
@@ -256,15 +500,61 @@ export class Service extends DatabaseService<Model> {
         [data.projectId.toString(), data.kubernetesClusterId.toString()],
       ),
       manager.query(
-        `SELECT COALESCE(SUM(
-           CASE WHEN jsonb_typeof("spec"->'containers') = 'array'
-                THEN jsonb_array_length("spec"->'containers')
-                ELSE 0 END
-         ), 0)::text AS total
+        /*
+         * containerCount is cached on the row during ingest
+         * (KubernetesInventoryExtractor sets it from
+         * spec.containers.length), so this is a plain int sum instead
+         * of a JSONB scan. Rows written before that ingest change may
+         * have NULL; SUM treats those as 0, which matches the old
+         * behavior.
+         */
+        `SELECT COALESCE(SUM("containerCount"), 0)::text AS total
          FROM "KubernetesResource"
          WHERE "projectId" = $1 AND "kubernetesClusterId" = $2 AND "kind" = 'Pod' AND "deletedAt" IS NULL`,
         [data.projectId.toString(), data.kubernetesClusterId.toString()],
       ),
+      /*
+       * Top-N offenders powering the "Why is this cluster degraded?" card.
+       * Failed first (hardest outage), then Pending, then Unknown, so the
+       * user sees the worst stuff first without having to sort client-side.
+       */
+      manager.query(
+        `SELECT "name", "namespaceKey", "phase", "status"
+         FROM "KubernetesResource"
+         WHERE "projectId" = $1
+           AND "kubernetesClusterId" = $2
+           AND "kind" = 'Pod'
+           AND "deletedAt" IS NULL
+           AND ("phase" IS NULL OR "phase" NOT IN ('Running', 'Succeeded'))
+         ORDER BY
+           CASE "phase"
+             WHEN 'Failed' THEN 0
+             WHEN 'Pending' THEN 1
+             ELSE 2
+           END,
+           "lastSeenAt" DESC
+         LIMIT ${DEGRADED_SAMPLE_LIMIT}`,
+        [data.projectId.toString(), data.kubernetesClusterId.toString()],
+      ),
+      manager.query(
+        `SELECT "name", "isReady", "hasMemoryPressure", "hasDiskPressure", "hasPidPressure", "status"
+         FROM "KubernetesResource"
+         WHERE "projectId" = $1
+           AND "kubernetesClusterId" = $2
+           AND "kind" = 'Node'
+           AND "deletedAt" IS NULL
+           AND (
+             "isReady" IS FALSE
+             OR "hasMemoryPressure" IS TRUE
+             OR "hasDiskPressure" IS TRUE
+             OR "hasPidPressure" IS TRUE
+           )
+         ORDER BY
+           CASE WHEN "isReady" IS FALSE THEN 0 ELSE 1 END,
+           "lastSeenAt" DESC
+         LIMIT ${DEGRADED_SAMPLE_LIMIT}`,
+        [data.projectId.toString(), data.kubernetesClusterId.toString()],
+      ),
     ]);
 
     const countsByKind: Record<string, number> = {};
@@ -308,6 +598,29 @@ export class Service extends DatabaseService<Model> {
     const containerCount: number =
       parseInt(containerRows[0]?.total || "0", 10) || 0;
 
+    const degradedPods: Array<DegradedPod> = degradedPodRows.map(
+      (row: {
+        name: string;
+        namespaceKey: string;
+        phase: string | null;
+        status: unknown;
+      }) => {
+        return buildDegradedPod(row);
+      },
+    );
+    const degradedNodes: Array<DegradedNode> = degradedNodeRows.map(
+      (row: {
+        name: string;
+        isReady: boolean | null;
+        hasMemoryPressure: boolean | null;
+        hasDiskPressure: boolean | null;
+        hasPidPressure: boolean | null;
+        status: unknown;
+      }) => {
+        return buildDegradedNode(row);
+      },
+    );
+
     return {
       countsByKind,
       containerCount,
@@ -321,6 +634,8 @@ export class Service extends DatabaseService<Model> {
         diskPressure: parseInt(nodeRow?.diskPressure || "0", 10) || 0,
         pidPressure: parseInt(nodeRow?.pidPressure || "0", 10) || 0,
       },
+      degradedPods,
+      degradedNodes,
     };
   }