@oneuptime/common 10.0.69 → 10.0.71

package/Models/DatabaseModels/KubernetesCluster.ts

@@ -73,6 +73,13 @@ import {
 })
 @CrudApiEndpoint(new Route("/kubernetes-cluster"))
 @SlugifyColumn("name", "slug")
+/*
+ * Enforce one cluster row per (projectId, clusterIdentifier) at the DB level.
+ * Without this, two pods emitting OTel telemetry for a new cluster at the
+ * same time (e.g. when the agent is first installed or during a rolling
+ * update) race in findOrCreateByClusterIdentifier and create duplicate rows.
+ */
+@Index(["projectId", "clusterIdentifier"], { unique: true })
 @TableMetadata({
   tableName: "KubernetesCluster",
   singularName: "Kubernetes Cluster",

package/Models/DatabaseModels/Project.ts

@@ -342,15 +342,15 @@ export default class Project extends TenantModel {
     update: [Permission.ProjectOwner, Permission.ManageProjectBilling],
   })
   @TableColumn({
-    type: TableColumnType.Email,
+    type: TableColumnType.LongText,
     title: "Finance / Accounting Email",
     description:
-      "Invoices, receipts and billing related notifications will be sent to this email in addition to project owner.",
-    example: "accounting@example.com",
+      "Invoices, receipts and billing related notifications will be sent to these emails in addition to project owner. Separate multiple emails with a comma.",
+    example: "accounting@example.com, finance@example.com",
   })
   @Column({
-    type: ColumnType.Email,
-    length: ColumnLength.Email,
+    type: ColumnType.LongText,
+    length: ColumnLength.LongText,
     nullable: true,
     unique: false,
   })

package/Server/Infrastructure/Postgres/SchemaMigrations/1776865086264-MigrationName.ts

@@ -1,14 +1,17 @@
 import { MigrationInterface, QueryRunner } from "typeorm";
 
 export class MigrationName1776865086264 implements MigrationInterface {
-    name = 'MigrationName1776865086264'
+  public name: string = "MigrationName1776865086264";
 
-    public async up(queryRunner: QueryRunner): Promise<void> {
-        await queryRunner.query(`ALTER TABLE "KubernetesResource" ADD "containerCount" integer`);
-    }
-
-    public async down(queryRunner: QueryRunner): Promise<void> {
-        await queryRunner.query(`ALTER TABLE "KubernetesResource" DROP COLUMN "containerCount"`);
-    }
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" ADD "containerCount" integer`,
+    );
+  }
 
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "KubernetesResource" DROP COLUMN "containerCount"`,
+    );
+  }
 }

package/Server/Infrastructure/Postgres/SchemaMigrations/1776881254913-DedupeKubernetesClustersAndAddUniqueIndex.ts

@@ -0,0 +1,137 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Before this migration, KubernetesCluster had an app-level
+ * @UniqueColumnBy("projectId") check and a non-unique index on
+ * clusterIdentifier, but no DB-level uniqueness. Under concurrent telemetry
+ * from multiple agent pods (happens every time the agent is installed or
+ * rolls out), findOrCreateByClusterIdentifier would race between its find
+ * and its create, and the DB accepted both inserts — producing duplicate
+ * rows with identical (projectId, clusterIdentifier).
+ *
+ * This migration:
+ *   1. Reparents all FKs that reference duplicate clusters — KubernetesResource,
+ *      KubernetesClusterOwnerUser, KubernetesClusterOwnerTeam — onto the
+ *      oldest surviving row in each duplicate group.
+ *   2. Deletes the duplicate (non-survivor) rows.
+ *   3. Creates a DB-level unique index on (projectId, clusterIdentifier) so
+ *      future races are rejected by the DB — the service's existing
+ *      catch-and-refetch in findOrCreateByClusterIdentifier then returns the
+ *      winning row instead of producing a duplicate.
+ *
+ * The auto-generator also picked up unrelated OnCallDutyPolicyScheduleLayer
+ * default-value drift. That's dev-environment drift, not the bug we're fixing;
+ * stripped from this migration.
+ */
+export class DedupeKubernetesClustersAndAddUniqueIndex1776881254913
+  implements MigrationInterface
+{
+  public name: string =
+    "DedupeKubernetesClustersAndAddUniqueIndex1776881254913";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    // 1: reparent KubernetesResource FKs from duplicates -> survivor.
+    await queryRunner.query(`
+      WITH survivors AS (
+        SELECT DISTINCT ON ("projectId", "clusterIdentifier")
+          _id AS survivor_id,
+          "projectId",
+          "clusterIdentifier"
+        FROM "KubernetesCluster"
+        ORDER BY "projectId", "clusterIdentifier", "createdAt" ASC, _id ASC
+      ),
+      losers AS (
+        SELECT kc._id AS loser_id, s.survivor_id
+        FROM "KubernetesCluster" kc
+        JOIN survivors s
+          ON s."projectId" = kc."projectId"
+         AND s."clusterIdentifier" = kc."clusterIdentifier"
+        WHERE kc._id <> s.survivor_id
+      )
+      UPDATE "KubernetesResource" kr
+      SET "kubernetesClusterId" = l.survivor_id
+      FROM losers l
+      WHERE kr."kubernetesClusterId" = l.loser_id;
+    `);
+
+    // 2: reparent KubernetesClusterOwnerUser FKs.
+    await queryRunner.query(`
+      WITH survivors AS (
+        SELECT DISTINCT ON ("projectId", "clusterIdentifier")
+          _id AS survivor_id,
+          "projectId",
+          "clusterIdentifier"
+        FROM "KubernetesCluster"
+        ORDER BY "projectId", "clusterIdentifier", "createdAt" ASC, _id ASC
+      ),
+      losers AS (
+        SELECT kc._id AS loser_id, s.survivor_id
+        FROM "KubernetesCluster" kc
+        JOIN survivors s
+          ON s."projectId" = kc."projectId"
+         AND s."clusterIdentifier" = kc."clusterIdentifier"
+        WHERE kc._id <> s.survivor_id
+      )
+      UPDATE "KubernetesClusterOwnerUser" o
+      SET "kubernetesClusterId" = l.survivor_id
+      FROM losers l
+      WHERE o."kubernetesClusterId" = l.loser_id;
+    `);
+
+    // 3: reparent KubernetesClusterOwnerTeam FKs.
+    await queryRunner.query(`
+      WITH survivors AS (
+        SELECT DISTINCT ON ("projectId", "clusterIdentifier")
+          _id AS survivor_id,
+          "projectId",
+          "clusterIdentifier"
+        FROM "KubernetesCluster"
+        ORDER BY "projectId", "clusterIdentifier", "createdAt" ASC, _id ASC
+      ),
+      losers AS (
+        SELECT kc._id AS loser_id, s.survivor_id
+        FROM "KubernetesCluster" kc
+        JOIN survivors s
+          ON s."projectId" = kc."projectId"
+         AND s."clusterIdentifier" = kc."clusterIdentifier"
+        WHERE kc._id <> s.survivor_id
+      )
+      UPDATE "KubernetesClusterOwnerTeam" o
+      SET "kubernetesClusterId" = l.survivor_id
+      FROM losers l
+      WHERE o."kubernetesClusterId" = l.loser_id;
+    `);
+
+    // 4: delete duplicate rows now that nothing references them.
+    await queryRunner.query(`
+      WITH survivors AS (
+        SELECT DISTINCT ON ("projectId", "clusterIdentifier")
+          _id AS survivor_id,
+          "projectId",
+          "clusterIdentifier"
+        FROM "KubernetesCluster"
+        ORDER BY "projectId", "clusterIdentifier", "createdAt" ASC, _id ASC
+      )
+      DELETE FROM "KubernetesCluster" kc
+      USING survivors s
+      WHERE s."projectId" = kc."projectId"
+        AND s."clusterIdentifier" = kc."clusterIdentifier"
+        AND kc._id <> s.survivor_id;
+    `);
+
+    // 5: add the DB-level composite unique index.
+    await queryRunner.query(
+      `CREATE UNIQUE INDEX "IDX_9756988b48848f4f7532a2af0d" ON "KubernetesCluster" ("projectId", "clusterIdentifier") `,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `DROP INDEX "public"."IDX_9756988b48848f4f7532a2af0d"`,
+    );
+    /*
+     * Duplicate rows dropped in up() are lost — a down-migration cannot
+     * resurrect them (and reinstating duplicates is not desirable anyway).
+     */
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/1776886248361-MigrationName.ts

@@ -0,0 +1,17 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class MigrationName1776886248361 implements MigrationInterface {
+  public name = "MigrationName1776886248361";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" ALTER COLUMN "financeAccountingEmail" TYPE character varying(500)`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(
+      `ALTER TABLE "Project" ALTER COLUMN "financeAccountingEmail" TYPE character varying(100)`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -288,6 +288,7 @@ import { MigrationName1776544084793 } from "./1776544084793-MigrationName";
 import { MigrationName1776761171349 } from "./1776761171349-MigrationName";
 import { MigrationName1776801030808 } from "./1776801030808-MigrationName";
 import { MigrationName1776865086264 } from "./1776865086264-MigrationName";
+import { DedupeKubernetesClustersAndAddUniqueIndex1776881254913 } from "./1776881254913-DedupeKubernetesClustersAndAddUniqueIndex";
 export default [
   InitialMigration,
   MigrationName1717678334852,
@@ -579,4 +580,5 @@ export default [
   MigrationName1776761171349,
   MigrationName1776801030808,
   MigrationName1776865086264,
+  DedupeKubernetesClustersAndAddUniqueIndex1776881254913,
 ];

package/Server/Services/AIBillingService.ts

@@ -94,8 +94,8 @@ export class AIBillingService extends BaseService {
         amountInUSD,
         {
           sendInvoiceByEmail: project.sendInvoicesByEmail || false,
-          recipientEmail: project.financeAccountingEmail
-            ? new Email(project.financeAccountingEmail)
+          recipientEmails: project.financeAccountingEmail
+            ? Email.parseList(project.financeAccountingEmail)
             : undefined,
           projectId: project.id || undefined,
         },

package/Server/Services/BillingService.ts

@@ -138,13 +138,37 @@ export class BillingService extends BaseService {
       line2 = rest.substring(0, 200);
     }
 
+    /*
+     * financeAccountingEmail may be a comma/semicolon-separated list. Parse the first
+     * valid address for Stripe (which stores a single email), and keep the full list
+     * in metadata so OneUptime can fan out invoice emails to every recipient.
+     */
+    const parsedEmails: Array<Email> = financeAccountingEmail
+      ? (() => {
+          try {
+            return Email.parseList(financeAccountingEmail);
+          } catch (err) {
+            logger.error(
+              `[Invoice Email] Failed to parse financeAccountingEmail for customer ${id}: ${err}`,
+            );
+            return [];
+          }
+        })()
+      : [];
+
+    const normalizedEmailList: string = parsedEmails
+      .map((e: Email): string => {
+        return e.toString();
+      })
+      .join(", ");
+
     const metadata: Record<string, string> = {
       business_details_full: businessDetails.substring(0, 5000),
     };
-    if (financeAccountingEmail) {
-      metadata["finance_accounting_email"] = financeAccountingEmail.substring(
+    if (normalizedEmailList) {
+      metadata["finance_accounting_email"] = normalizedEmailList.substring(
         0,
-        200,
+        500,
       );
     } else {
       // Remove if cleared
@@ -165,11 +189,11 @@ export class BillingService extends BaseService {
     };
 
     /*
-     * If finance / accounting email provided, set it as the customer email so Stripe sends
-     * invoices / receipts there. (Stripe only supports a single email via API currently.)
+     * Stripe's customer.email only accepts one address. Use the first parsed email;
+     * the rest are delivered by OneUptime's own webhook-driven invoice email path.
      */
-    if (financeAccountingEmail && financeAccountingEmail.trim().length > 0) {
-      updateParams.email = financeAccountingEmail.trim();
+    if (parsedEmails.length > 0) {
+      updateParams.email = parsedEmails[0]!.toString();
     }
 
     if (line1) {
@@ -956,11 +980,16 @@ export class BillingService extends BaseService {
   @CaptureSpan()
   public async sendInvoiceByEmail(
     invoiceId: string,
-    recipientEmail?: Email,
+    recipientEmails?: Array<Email>,
     projectId?: ObjectID,
   ): Promise<void> {
+    const recipientsForLog: string = (recipientEmails || [])
+      .map((e: Email): string => {
+        return e.toString();
+      })
+      .join(", ");
     logger.debug(
-      `[Invoice Email] sendInvoiceByEmail called for invoice: ${invoiceId}, recipientEmail: ${recipientEmail?.toString()}`,
+      `[Invoice Email] sendInvoiceByEmail called for invoice: ${invoiceId}, recipients: ${recipientsForLog}`,
     );
 
     if (!this.isBillingEnabled()) {
@@ -986,13 +1015,22 @@ export class BillingService extends BaseService {
         return;
       }
 
-      // Determine recipient email
-      let toEmail: Email | undefined = recipientEmail;
-      if (!toEmail && stripeInvoice.customer_email) {
-        toEmail = new Email(stripeInvoice.customer_email);
+      /*
+       * Determine recipient list. Fall back to the invoice's customer email when no
+       * explicit recipients were supplied by the caller.
+       */
+      let toEmails: Array<Email> = recipientEmails || [];
+      if (toEmails.length === 0 && stripeInvoice.customer_email) {
+        try {
+          toEmails = Email.parseList(stripeInvoice.customer_email);
+        } catch (err) {
+          logger.error(
+            `[Invoice Email] Failed to parse Stripe customer_email "${stripeInvoice.customer_email}" for invoice ${invoiceId}: ${err}`,
+          );
+        }
       }
 
-      if (!toEmail) {
+      if (toEmails.length === 0) {
         logger.error(
           `[Invoice Email] No recipient email found for invoice ${invoiceId}`,
           { projectId: projectId?.toString() } as LogAttributes,
@@ -1021,33 +1059,41 @@ export class BillingService extends BaseService {
         dashboardLink = `${DashboardClientUrl.toString()}/dashboard/${projectId.toString()}/settings/billing`;
       }
 
-      logger.debug(
-        `[Invoice Email] Sending invoice email to ${toEmail.toString()} - Invoice #${invoiceNumber}, Amount: ${amount}`,
-      );
-
-      // Send email via OneUptime MailService
-      await MailService.sendMail(
-        {
-          toEmail: toEmail,
-          templateType: EmailTemplateType.Invoice,
-          vars: {
-            invoiceNumber: invoiceNumber,
-            invoiceDate: invoiceDate,
-            amount: amount,
-            description: description || "",
-            invoicePdfUrl: invoicePdfUrl || "",
-            dashboardLink: dashboardLink || "",
-          },
-          subject: `Invoice #${invoiceNumber} from OneUptime`,
-        },
-        {
-          projectId: projectId,
-        },
-      );
+      for (const toEmail of toEmails) {
+        logger.debug(
+          `[Invoice Email] Sending invoice email to ${toEmail.toString()} - Invoice #${invoiceNumber}, Amount: ${amount}`,
+        );
+        try {
+          await MailService.sendMail(
+            {
+              toEmail: toEmail,
+              templateType: EmailTemplateType.Invoice,
+              vars: {
+                invoiceNumber: invoiceNumber,
+                invoiceDate: invoiceDate,
+                amount: amount,
+                description: description || "",
+                invoicePdfUrl: invoicePdfUrl || "",
+                dashboardLink: dashboardLink || "",
+              },
+              subject: `Invoice #${invoiceNumber} from OneUptime`,
+            },
+            {
+              projectId: projectId,
+            },
+          );
 
-      logger.debug(
-        `[Invoice Email] Successfully sent invoice ${invoiceId} email to ${toEmail.toString()}`,
-      );
+          logger.debug(
+            `[Invoice Email] Successfully sent invoice ${invoiceId} email to ${toEmail.toString()}`,
+          );
+        } catch (perRecipientErr) {
+          // Keep going for the remaining recipients if one fails.
+          logger.error(
+            `[Invoice Email] Failed to send invoice ${invoiceId} to ${toEmail.toString()}: ${perRecipientErr}`,
+            { projectId: projectId?.toString() } as LogAttributes,
+          );
+        }
+      }
     } catch (err) {
       logger.error(
         `[Invoice Email] Failed to send invoice ${invoiceId} by email: ${err}`,
@@ -1108,16 +1154,21 @@ export class BillingService extends BaseService {
     amountInUsd: number,
     options?: {
       sendInvoiceByEmail?: boolean | undefined;
-      recipientEmail?: Email | undefined;
+      recipientEmails?: Array<Email> | undefined;
       projectId?: ObjectID | undefined;
     },
   ): Promise<void> {
     const sendInvoiceByEmail: boolean = options?.sendInvoiceByEmail || false;
-    const recipientEmail: Email | undefined = options?.recipientEmail;
+    const recipientEmails: Array<Email> | undefined = options?.recipientEmails;
     const projectId: ObjectID | undefined = options?.projectId;
 
+    const recipientsForLog: string = (recipientEmails || [])
+      .map((e: Email): string => {
+        return e.toString();
+      })
+      .join(", ");
     logger.debug(
-      `[Invoice Email] generateInvoiceAndChargeCustomer called - customer: ${customerId}, amount: $${amountInUsd}, sendInvoiceByEmail: ${sendInvoiceByEmail}, recipientEmail: ${recipientEmail?.toString()}, projectId: ${projectId?.toString()}`,
+      `[Invoice Email] generateInvoiceAndChargeCustomer called - customer: ${customerId}, amount: $${amountInUsd}, sendInvoiceByEmail: ${sendInvoiceByEmail}, recipients: ${recipientsForLog}, projectId: ${projectId?.toString()}`,
     );
 
     const invoice: Stripe.Invoice = await this.stripe.invoices.create({
@@ -1161,7 +1212,7 @@ export class BillingService extends BaseService {
         logger.debug(
           `[Invoice Email] sendInvoiceByEmail is true, sending invoice ${invoice.id} by email`,
         );
-        await this.sendInvoiceByEmail(invoice.id!, recipientEmail, projectId);
+        await this.sendInvoiceByEmail(invoice.id!, recipientEmails, projectId);
       } else {
         logger.debug(
           `[Invoice Email] sendInvoiceByEmail is false, skipping email for invoice ${invoice.id}`,
@@ -1356,16 +1407,33 @@ export class BillingService extends BaseService {
             },
           });
 
-          let recipientEmail: Email | undefined = undefined;
+          let recipientEmails: Array<Email> | undefined = undefined;
           let projectId: ObjectID | undefined = undefined;
 
           if (project) {
             projectId = project.id || undefined;
             if (project.financeAccountingEmail) {
-              recipientEmail = new Email(project.financeAccountingEmail);
+              try {
+                recipientEmails = Email.parseList(
+                  project.financeAccountingEmail,
+                );
+              } catch (parseErr) {
+                /*
+                 * Log and fall back to Stripe customer email rather than dropping
+                 * the invoice delivery entirely due to a malformed address.
+                 */
+                logger.error(
+                  `[Invoice Email] Failed to parse financeAccountingEmail for project ${projectId?.toString()}: ${parseErr}`,
+                );
+              }
             }
+            const parsedForLog: string = (recipientEmails || [])
+              .map((e: Email): string => {
+                return e.toString();
+              })
+              .join(", ");
             logger.debug(
-              `[Invoice Email] Found project ${projectId?.toString()}, financeAccountingEmail: ${recipientEmail?.toString()}`,
+              `[Invoice Email] Found project ${projectId?.toString()}, financeAccountingEmail recipients: ${parsedForLog}`,
             );
           } else {
             logger.debug(
@@ -1376,7 +1444,7 @@ export class BillingService extends BaseService {
           logger.debug(
             `[Invoice Email] Sending invoice ${invoice.id} by email`,
           );
-          await this.sendInvoiceByEmail(invoice.id, recipientEmail, projectId);
+          await this.sendInvoiceByEmail(invoice.id, recipientEmails, projectId);
           logger.debug(
             `[Invoice Email] Successfully processed invoice.finalized - sent invoice ${invoice.id} by email`,
           );

package/Server/Services/DatabaseService.ts

@@ -69,6 +69,7 @@ import { FindWhere } from "../../Types/BaseDatabase/Query";
 import Realtime from "../Utils/Realtime";
 import ModelEventType from "../../Types/Realtime/ModelEventType";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import type AuditLogServiceType from "./AuditLogService";
 
 class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
   public modelType!: { new (): TBaseModel };
@@ -782,14 +783,9 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
          * which extend DatabaseService). A top-level import leaves
          * DatabaseService undefined at class-extension time for subclasses.
          */
-        // eslint-disable-next-line @typescript-eslint/no-var-requires
-        const auditLogService: {
-          recordCreate: (data: {
-            model: TBaseModel;
-            createdItem: TBaseModel;
-            props: DatabaseCommonInteractionProps;
-          }) => Promise<void>;
-        } = require("./AuditLogService").default;
+        const auditLogService: typeof AuditLogServiceType =
+          // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+          require("./AuditLogService").default;
         await auditLogService.recordCreate({
           model: this.getModel(),
           createdItem: createBy.data,
@@ -1237,15 +1233,9 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
       }
 
       if (this.getModel().enableAuditLogOn?.delete && items.length > 0) {
-        // eslint-disable-next-line @typescript-eslint/no-var-requires
-        const auditLogService: {
-          recordDelete: (args: {
-            model: TBaseModel;
-            deletedItem: TBaseModel;
-            itemId: ObjectID;
-            props: DatabaseCommonInteractionProps;
-          }) => Promise<void>;
-        } = require("./AuditLogService").default;
+        const auditLogService: typeof AuditLogServiceType =
+          // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+          require("./AuditLogService").default;
         for (const item of items) {
           if (item.id) {
             await auditLogService.recordDelete({
@@ -1667,16 +1657,9 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
           !this.hasSameValues({ item, updatedItem }) &&
           item.id
         ) {
-          // eslint-disable-next-line @typescript-eslint/no-var-requires
-          const auditLogService: {
-            recordUpdate: (args: {
-              model: TBaseModel;
-              before: TBaseModel;
-              updatedFields: JSONObject;
-              itemId: ObjectID;
-              props: DatabaseCommonInteractionProps;
-            }) => Promise<void>;
-          } = require("./AuditLogService").default;
+          const auditLogService: typeof AuditLogServiceType =
+            // eslint-disable-next-line @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires
+            require("./AuditLogService").default;
           await auditLogService.recordUpdate({
             model: this.getModel(),
             before: item,

package/Server/Services/KubernetesResourceService.ts

@@ -113,7 +113,10 @@ function buildDegradedPod(row: {
   const scanForReason: (
     list: Array<Record<string, unknown>>,
     targetState: string,
-  ) => { reason: string; message: string } | null = (list, targetState) => {
+  ) => { reason: string; message: string } | null = (
+    list: Array<Record<string, unknown>>,
+    targetState: string,
+  ) => {
     for (const cs of list) {
       if (cs["state"] !== targetState) {
         continue;
@@ -144,8 +147,10 @@ function buildDegradedPod(row: {
     reason = hit.reason;
     message = hit.message;
   } else {
-    // Fall back to the pod-level reason/message fields set by the scheduler
-    // (e.g. "Unschedulable" with "0/3 nodes are available: ...").
+    /*
+     * Fall back to the pod-level reason/message fields set by the scheduler
+     * (e.g. "Unschedulable" with "0/3 nodes are available: ...").
+     */
     const topReason: unknown = status["reason"];
     const topMessage: unknown = status["message"];
     if (typeof topReason === "string") {
@@ -211,7 +216,9 @@ function buildDegradedNode(row: {
 
   const findCondition: (
     predicate: (c: Record<string, unknown>) => boolean,
-  ) => Record<string, unknown> | null = (predicate) => {
+  ) => Record<string, unknown> | null = (
+    predicate: (c: Record<string, unknown>) => boolean,
+  ) => {
     for (const c of conditions) {
       if (predicate(c)) {
         return c;
@@ -591,12 +598,28 @@ export class Service extends DatabaseService<Model> {
     const containerCount: number =
       parseInt(containerRows[0]?.total || "0", 10) || 0;
 
-    const degradedPods: Array<DegradedPod> = degradedPodRows.map((row) => {
-      return buildDegradedPod(row);
-    });
-    const degradedNodes: Array<DegradedNode> = degradedNodeRows.map((row) => {
-      return buildDegradedNode(row);
-    });
+    const degradedPods: Array<DegradedPod> = degradedPodRows.map(
+      (row: {
+        name: string;
+        namespaceKey: string;
+        phase: string | null;
+        status: unknown;
+      }) => {
+        return buildDegradedPod(row);
+      },
+    );
+    const degradedNodes: Array<DegradedNode> = degradedNodeRows.map(
+      (row: {
+        name: string;
+        isReady: boolean | null;
+        hasMemoryPressure: boolean | null;
+        hasDiskPressure: boolean | null;
+        hasPidPressure: boolean | null;
+        status: unknown;
+      }) => {
+        return buildDegradedNode(row);
+      },
+    );
 
     return {
       countsByKind,

package/Server/Services/NotificationService.ts

@@ -90,8 +90,8 @@ export class NotificationService extends BaseService {
         amountInUSD,
         {
           sendInvoiceByEmail: project.sendInvoicesByEmail || false,
-          recipientEmail: project.financeAccountingEmail
-            ? new Email(project.financeAccountingEmail)
+          recipientEmails: project.financeAccountingEmail
+            ? Email.parseList(project.financeAccountingEmail)
             : undefined,
           projectId: project.id || undefined,
         },