@oneuptime/common 10.0.70 → 10.0.72

package/Server/Services/BillingService.ts

@@ -138,13 +138,37 @@ export class BillingService extends BaseService {
       line2 = rest.substring(0, 200);
     }
 
+    /*
+     * financeAccountingEmail may be a comma/semicolon-separated list. Parse the first
+     * valid address for Stripe (which stores a single email), and keep the full list
+     * in metadata so OneUptime can fan out invoice emails to every recipient.
+     */
+    const parsedEmails: Array<Email> = financeAccountingEmail
+      ? (() => {
+          try {
+            return Email.parseList(financeAccountingEmail);
+          } catch (err) {
+            logger.error(
+              `[Invoice Email] Failed to parse financeAccountingEmail for customer ${id}: ${err}`,
+            );
+            return [];
+          }
+        })()
+      : [];
+
+    const normalizedEmailList: string = parsedEmails
+      .map((e: Email): string => {
+        return e.toString();
+      })
+      .join(", ");
+
     const metadata: Record<string, string> = {
       business_details_full: businessDetails.substring(0, 5000),
     };
-    if (financeAccountingEmail) {
-      metadata["finance_accounting_email"] = financeAccountingEmail.substring(
+    if (normalizedEmailList) {
+      metadata["finance_accounting_email"] = normalizedEmailList.substring(
         0,
-        200,
+        500,
       );
     } else {
       // Remove if cleared
@@ -165,11 +189,11 @@ export class BillingService extends BaseService {
     };
 
     /*
-     * If finance / accounting email provided, set it as the customer email so Stripe sends
-     * invoices / receipts there. (Stripe only supports a single email via API currently.)
+     * Stripe's customer.email only accepts one address. Use the first parsed email;
+     * the rest are delivered by OneUptime's own webhook-driven invoice email path.
      */
-    if (financeAccountingEmail && financeAccountingEmail.trim().length > 0) {
-      updateParams.email = financeAccountingEmail.trim();
+    if (parsedEmails.length > 0) {
+      updateParams.email = parsedEmails[0]!.toString();
     }
 
     if (line1) {
@@ -956,11 +980,16 @@ export class BillingService extends BaseService {
   @CaptureSpan()
   public async sendInvoiceByEmail(
     invoiceId: string,
-    recipientEmail?: Email,
+    recipientEmails?: Array<Email>,
     projectId?: ObjectID,
   ): Promise<void> {
+    const recipientsForLog: string = (recipientEmails || [])
+      .map((e: Email): string => {
+        return e.toString();
+      })
+      .join(", ");
     logger.debug(
-      `[Invoice Email] sendInvoiceByEmail called for invoice: ${invoiceId}, recipientEmail: ${recipientEmail?.toString()}`,
+      `[Invoice Email] sendInvoiceByEmail called for invoice: ${invoiceId}, recipients: ${recipientsForLog}`,
     );
 
     if (!this.isBillingEnabled()) {
@@ -986,13 +1015,22 @@ export class BillingService extends BaseService {
         return;
       }
 
-      // Determine recipient email
-      let toEmail: Email | undefined = recipientEmail;
-      if (!toEmail && stripeInvoice.customer_email) {
-        toEmail = new Email(stripeInvoice.customer_email);
+      /*
+       * Determine recipient list. Fall back to the invoice's customer email when no
+       * explicit recipients were supplied by the caller.
+       */
+      let toEmails: Array<Email> = recipientEmails || [];
+      if (toEmails.length === 0 && stripeInvoice.customer_email) {
+        try {
+          toEmails = Email.parseList(stripeInvoice.customer_email);
+        } catch (err) {
+          logger.error(
+            `[Invoice Email] Failed to parse Stripe customer_email "${stripeInvoice.customer_email}" for invoice ${invoiceId}: ${err}`,
+          );
+        }
       }
 
-      if (!toEmail) {
+      if (toEmails.length === 0) {
         logger.error(
           `[Invoice Email] No recipient email found for invoice ${invoiceId}`,
           { projectId: projectId?.toString() } as LogAttributes,
@@ -1021,33 +1059,41 @@ export class BillingService extends BaseService {
         dashboardLink = `${DashboardClientUrl.toString()}/dashboard/${projectId.toString()}/settings/billing`;
       }
 
-      logger.debug(
-        `[Invoice Email] Sending invoice email to ${toEmail.toString()} - Invoice #${invoiceNumber}, Amount: ${amount}`,
-      );
-
-      // Send email via OneUptime MailService
-      await MailService.sendMail(
-        {
-          toEmail: toEmail,
-          templateType: EmailTemplateType.Invoice,
-          vars: {
-            invoiceNumber: invoiceNumber,
-            invoiceDate: invoiceDate,
-            amount: amount,
-            description: description || "",
-            invoicePdfUrl: invoicePdfUrl || "",
-            dashboardLink: dashboardLink || "",
-          },
-          subject: `Invoice #${invoiceNumber} from OneUptime`,
-        },
-        {
-          projectId: projectId,
-        },
-      );
+      for (const toEmail of toEmails) {
+        logger.debug(
+          `[Invoice Email] Sending invoice email to ${toEmail.toString()} - Invoice #${invoiceNumber}, Amount: ${amount}`,
+        );
+        try {
+          await MailService.sendMail(
+            {
+              toEmail: toEmail,
+              templateType: EmailTemplateType.Invoice,
+              vars: {
+                invoiceNumber: invoiceNumber,
+                invoiceDate: invoiceDate,
+                amount: amount,
+                description: description || "",
+                invoicePdfUrl: invoicePdfUrl || "",
+                dashboardLink: dashboardLink || "",
+              },
+              subject: `Invoice #${invoiceNumber} from OneUptime`,
+            },
+            {
+              projectId: projectId,
+            },
+          );
 
-      logger.debug(
-        `[Invoice Email] Successfully sent invoice ${invoiceId} email to ${toEmail.toString()}`,
-      );
+          logger.debug(
+            `[Invoice Email] Successfully sent invoice ${invoiceId} email to ${toEmail.toString()}`,
+          );
+        } catch (perRecipientErr) {
+          // Keep going for the remaining recipients if one fails.
+          logger.error(
+            `[Invoice Email] Failed to send invoice ${invoiceId} to ${toEmail.toString()}: ${perRecipientErr}`,
+            { projectId: projectId?.toString() } as LogAttributes,
+          );
+        }
+      }
     } catch (err) {
       logger.error(
         `[Invoice Email] Failed to send invoice ${invoiceId} by email: ${err}`,
@@ -1108,16 +1154,21 @@ export class BillingService extends BaseService {
     amountInUsd: number,
     options?: {
       sendInvoiceByEmail?: boolean | undefined;
-      recipientEmail?: Email | undefined;
+      recipientEmails?: Array<Email> | undefined;
       projectId?: ObjectID | undefined;
     },
   ): Promise<void> {
     const sendInvoiceByEmail: boolean = options?.sendInvoiceByEmail || false;
-    const recipientEmail: Email | undefined = options?.recipientEmail;
+    const recipientEmails: Array<Email> | undefined = options?.recipientEmails;
     const projectId: ObjectID | undefined = options?.projectId;
 
+    const recipientsForLog: string = (recipientEmails || [])
+      .map((e: Email): string => {
+        return e.toString();
+      })
+      .join(", ");
     logger.debug(
-      `[Invoice Email] generateInvoiceAndChargeCustomer called - customer: ${customerId}, amount: $${amountInUsd}, sendInvoiceByEmail: ${sendInvoiceByEmail}, recipientEmail: ${recipientEmail?.toString()}, projectId: ${projectId?.toString()}`,
+      `[Invoice Email] generateInvoiceAndChargeCustomer called - customer: ${customerId}, amount: $${amountInUsd}, sendInvoiceByEmail: ${sendInvoiceByEmail}, recipients: ${recipientsForLog}, projectId: ${projectId?.toString()}`,
     );
 
     const invoice: Stripe.Invoice = await this.stripe.invoices.create({
@@ -1161,7 +1212,7 @@ export class BillingService extends BaseService {
         logger.debug(
           `[Invoice Email] sendInvoiceByEmail is true, sending invoice ${invoice.id} by email`,
         );
-        await this.sendInvoiceByEmail(invoice.id!, recipientEmail, projectId);
+        await this.sendInvoiceByEmail(invoice.id!, recipientEmails, projectId);
       } else {
         logger.debug(
           `[Invoice Email] sendInvoiceByEmail is false, skipping email for invoice ${invoice.id}`,
@@ -1356,16 +1407,33 @@ export class BillingService extends BaseService {
             },
           });
 
-          let recipientEmail: Email | undefined = undefined;
+          let recipientEmails: Array<Email> | undefined = undefined;
           let projectId: ObjectID | undefined = undefined;
 
           if (project) {
             projectId = project.id || undefined;
             if (project.financeAccountingEmail) {
-              recipientEmail = new Email(project.financeAccountingEmail);
+              try {
+                recipientEmails = Email.parseList(
+                  project.financeAccountingEmail,
+                );
+              } catch (parseErr) {
+                /*
+                 * Log and fall back to Stripe customer email rather than dropping
+                 * the invoice delivery entirely due to a malformed address.
+                 */
+                logger.error(
+                  `[Invoice Email] Failed to parse financeAccountingEmail for project ${projectId?.toString()}: ${parseErr}`,
+                );
+              }
             }
+            const parsedForLog: string = (recipientEmails || [])
+              .map((e: Email): string => {
+                return e.toString();
+              })
+              .join(", ");
             logger.debug(
-              `[Invoice Email] Found project ${projectId?.toString()}, financeAccountingEmail: ${recipientEmail?.toString()}`,
+              `[Invoice Email] Found project ${projectId?.toString()}, financeAccountingEmail recipients: ${parsedForLog}`,
             );
           } else {
             logger.debug(
@@ -1376,7 +1444,7 @@ export class BillingService extends BaseService {
           logger.debug(
             `[Invoice Email] Sending invoice ${invoice.id} by email`,
           );
-          await this.sendInvoiceByEmail(invoice.id, recipientEmail, projectId);
+          await this.sendInvoiceByEmail(invoice.id, recipientEmails, projectId);
           logger.debug(
             `[Invoice Email] Successfully processed invoice.finalized - sent invoice ${invoice.id} by email`,
           );

package/Server/Services/NotificationService.ts

@@ -90,8 +90,8 @@ export class NotificationService extends BaseService {
         amountInUSD,
         {
           sendInvoiceByEmail: project.sendInvoicesByEmail || false,
-          recipientEmail: project.financeAccountingEmail
-            ? new Email(project.financeAccountingEmail)
+          recipientEmails: project.financeAccountingEmail
+            ? Email.parseList(project.financeAccountingEmail)
             : undefined,
           projectId: project.id || undefined,
         },

package/Server/Types/Database/QueryUtil.ts

@@ -212,8 +212,10 @@ export default class QueryUtil {
               relationColumnName: manyToManyMeta.relationColumnName,
             });
 
-            // Remove the relation-based filter so TypeORM does not create a
-            // JOIN that would yield OR semantics.
+            /*
+             * Remove the relation-based filter so TypeORM does not create a
+             * JOIN that would yield OR semantics.
+             */
             delete query[key];
 
             const existingIdFilter: any = (query as any)._id;
@@ -445,11 +447,13 @@ export default class QueryUtil {
       return null;
     }
 
-    // Only the owning side of a many-to-many has join/inverse columns. Follow
-    // the inverse relation when needed.
+    /*
+     * Only the owning side of a many-to-many has join/inverse columns. Follow
+     * the inverse relation when needed.
+     */
     const owningRelation: RelationMetadata = relation.isOwning
       ? relation
-      : (relation.inverseRelation ?? relation);
+      : relation.inverseRelation ?? relation;
 
     const joinTableName: string | undefined =
       owningRelation.junctionEntityMetadata?.tableName;
@@ -458,8 +462,10 @@ export default class QueryUtil {
       return null;
     }
 
-    // When `modelType` is the owning side, its id lives on joinColumns. When
-    // it is the inverse side, its id lives on inverseJoinColumns.
+    /*
+     * When `modelType` is the owning side, its id lives on joinColumns. When
+     * it is the inverse side, its id lives on inverseJoinColumns.
+     */
     const ownerColumns: Array<any> = relation.isOwning
       ? owningRelation.joinColumns
       : owningRelation.inverseJoinColumns;

package/Server/Utils/Monitor/Criteria/MetricMonitorCriteria.ts

@@ -8,6 +8,7 @@ import MetricCriteriaContext, {
   MetricComponent,
   MetricComponentValue,
 } from "../../../../Types/Monitor/MetricMonitor/MetricCriteriaContext";
+import MetricSeriesResult from "../../../../Types/Monitor/MetricMonitor/MetricSeriesResult";
 import MonitorStep from "../../../../Types/Monitor/MonitorStep";
 import { JSONObject } from "../../../../Types/JSON";
 import DataToProcess from "../DataToProcess";
@@ -23,6 +24,20 @@ import CaptureSpan from "../../Telemetry/CaptureSpan";
 import MetricUnitUtil from "../../../../Utils/MetricUnitUtil";
 import MetricFormulaEvaluator from "../../../../Utils/Metrics/MetricFormulaEvaluator";
 
+/**
+ * Result of evaluating a single criteria filter against a single metric
+ * series. `rootCause` is null when the filter did not match; otherwise
+ * it's the human-readable comparison message. `context` always reflects
+ * the metric identity for this series (used to render the metric
+ * details + breaching samples section of the incident root cause).
+ */
+export interface MetricSeriesEvaluationResult {
+  fingerprint: string | undefined;
+  labels: JSONObject;
+  rootCause: string | null;
+  context: MetricCriteriaContext;
+}
+
 export default class MetricMonitorCriteria {
   @CaptureSpan()
   public static async isMonitorInstanceCriteriaFilterMet(input: {
@@ -30,8 +45,50 @@ export default class MetricMonitorCriteria {
     criteriaFilter: CriteriaFilter;
     monitorStep: MonitorStep;
   }): Promise<string | null> {
-    // Metric Monitoring Check
+    const evaluations: Array<MetricSeriesEvaluationResult> =
+      MetricMonitorCriteria.evaluateAllSeries(input);
+
+    /*
+     * Backwards-compat: the scalar entrypoint collapses per-series
+     * evaluation down to the first matching series so existing callers
+     * (single-incident path) keep working. The per-series code path uses
+     * `evaluateAllSeries` directly.
+     */
+    const match: MetricSeriesEvaluationResult | undefined = evaluations.find(
+      (e: MetricSeriesEvaluationResult) => {
+        return e.rootCause !== null;
+      },
+    );
+
+    /*
+     * Always populate the legacy single-context field so the root-cause
+     * renderer can still read metric identity from the criteria filter
+     * even when nothing matched. Pick the first evaluation's context.
+     */
+    if (evaluations.length > 0) {
+      input.criteriaFilter.metricCriteriaContext = (
+        match || evaluations[0]!
+      ).context;
+    }
 
+    return match ? match.rootCause : null;
+  }
+
+  /**
+   * Evaluate a single criteria filter against every series produced by
+   * the monitor. For monitors without group-by, this returns a single
+   * evaluation covering all aggregated results (legacy behavior). For
+   * monitors with group-by attributes set, it returns one evaluation
+   * per unique series fingerprint — each with its own
+   * `MetricCriteriaContext` carrying that series' breaching samples
+   * and labels. The caller fans this out into one incident per
+   * breaching series.
+   */
+  public static evaluateAllSeries(input: {
+    dataToProcess: DataToProcess;
+    criteriaFilter: CriteriaFilter;
+    monitorStep: MonitorStep;
+  }): Array<MetricSeriesEvaluationResult> {
     if (
       input.criteriaFilter.metricMonitorOptions &&
       !input.criteriaFilter.metricMonitorOptions.metricAggregationType
@@ -41,20 +98,14 @@ export default class MetricMonitorCriteria {
     }
 
     if (input.criteriaFilter.checkOn !== CheckOn.MetricValue) {
-      return null;
+      return [];
     }
 
-    const rawThreshold: number | null = CompareCriteria.convertToNumber(
-      input.criteriaFilter.value,
-    );
-
-    const metricAlias: string =
-      input.criteriaFilter.metricMonitorOptions?.metricAlias || "";
-
     const metricResponse: MetricMonitorResponse =
       input.dataToProcess as MetricMonitorResponse;
-    const metricAggregatedResult: Array<AggregatedResult> =
-      metricResponse.metricResult || [];
+
+    const seriesBreakdown: Array<MetricSeriesResult> | undefined =
+      metricResponse.seriesBreakdown;
 
     const queryConfigs: Array<MetricQueryConfigData> =
       input.monitorStep.data?.metricMonitor?.metricViewConfig?.queryConfigs ||
@@ -63,6 +114,60 @@ export default class MetricMonitorCriteria {
       input.monitorStep.data?.metricMonitor?.metricViewConfig?.formulaConfigs ||
       [];
 
+    /*
+     * Series-less path: one synthetic "all-series" evaluation over the
+     * flat metricResult. Preserves the pre-group-by behavior exactly.
+     */
+    if (!seriesBreakdown || seriesBreakdown.length === 0) {
+      const result: MetricSeriesEvaluationResult =
+        MetricMonitorCriteria.evaluateOneSeries({
+          criteriaFilter: input.criteriaFilter,
+          aggregatedResults: metricResponse.metricResult || [],
+          queryConfigs,
+          formulaConfigs,
+          seriesFingerprint: undefined,
+          seriesLabels: {},
+        });
+      return [result];
+    }
+
+    return seriesBreakdown.map((series: MetricSeriesResult) => {
+      return MetricMonitorCriteria.evaluateOneSeries({
+        criteriaFilter: input.criteriaFilter,
+        aggregatedResults: series.aggregatedResults,
+        queryConfigs,
+        formulaConfigs,
+        seriesFingerprint: series.fingerprint,
+        seriesLabels: series.labels,
+      });
+    });
+  }
+
+  /**
+   * Core evaluation loop: compare the samples for one metric series
+   * against the criteria threshold. Builds the metric identity context,
+   * identifies breaching samples, and assembles the human-readable
+   * root-cause message. Factored out so `evaluateAllSeries` can invoke
+   * it once per series without duplicating logic.
+   */
+  private static evaluateOneSeries(input: {
+    criteriaFilter: CriteriaFilter;
+    aggregatedResults: Array<AggregatedResult>;
+    queryConfigs: Array<MetricQueryConfigData>;
+    formulaConfigs: Array<MetricFormulaConfigData>;
+    seriesFingerprint: string | undefined;
+    seriesLabels: JSONObject;
+  }): MetricSeriesEvaluationResult {
+    const rawThreshold: number | null = CompareCriteria.convertToNumber(
+      input.criteriaFilter.value,
+    );
+
+    const metricAlias: string =
+      input.criteriaFilter.metricMonitorOptions?.metricAlias || "";
+
+    const metricAggregatedResult: Array<AggregatedResult> =
+      input.aggregatedResults;
+
     /*
      * Resolve which query/formula the alias refers to. Use explicit index
      * checks (not `findIndex() || -1`, which incorrectly falls back to -1
@@ -73,25 +178,25 @@ export default class MetricMonitorCriteria {
     let aliasIndex: number = -1;
 
     if (metricAlias) {
-      const qIdx: number = queryConfigs.findIndex(
+      const qIdx: number = input.queryConfigs.findIndex(
         (q: MetricQueryConfigData) => {
           return q.metricAliasData?.metricVariable === metricAlias;
         },
       );
 
       if (qIdx >= 0) {
-        matchedQuery = queryConfigs[qIdx] || null;
+        matchedQuery = input.queryConfigs[qIdx] || null;
         aliasIndex = qIdx;
       } else {
-        const fIdx: number = formulaConfigs.findIndex(
+        const fIdx: number = input.formulaConfigs.findIndex(
           (f: MetricFormulaConfigData) => {
             return f.metricAliasData?.metricVariable === metricAlias;
           },
         );
 
         if (fIdx >= 0) {
-          matchedFormula = formulaConfigs[fIdx] || null;
-          aliasIndex = queryConfigs.length + fIdx;
+          matchedFormula = input.formulaConfigs[fIdx] || null;
+          aliasIndex = input.queryConfigs.length + fIdx;
         }
       }
     }
@@ -105,8 +210,8 @@ export default class MetricMonitorCriteria {
         ? metricAggregatedResult[aliasIndex]
         : metricAggregatedResult[0];
 
-    if (!matchedQuery && !matchedFormula && queryConfigs[0]) {
-      matchedQuery = queryConfigs[0];
+    if (!matchedQuery && !matchedFormula && input.queryConfigs[0]) {
+      matchedQuery = input.queryConfigs[0];
     }
 
     /*
@@ -119,14 +224,24 @@ export default class MetricMonitorCriteria {
         matchedFormula,
         metricAlias,
         criteriaFilter: input.criteriaFilter,
-        queryConfigs,
-        formulaConfigs,
+        queryConfigs: input.queryConfigs,
+        formulaConfigs: input.formulaConfigs,
       });
 
-    input.criteriaFilter.metricCriteriaContext = metricContext;
+    if (input.seriesFingerprint) {
+      metricContext.seriesFingerprint = input.seriesFingerprint;
+    }
+    if (input.seriesLabels && Object.keys(input.seriesLabels).length > 0) {
+      metricContext.seriesLabels = input.seriesLabels;
+    }
 
     if (rawThreshold === null) {
-      return null;
+      return {
+        fingerprint: input.seriesFingerprint,
+        labels: input.seriesLabels,
+        rootCause: null,
+        context: metricContext,
+      };
     }
 
     /*
@@ -180,11 +295,21 @@ export default class MetricMonitorCriteria {
         NoDataPolicy.Ignore;
 
       if (policy === NoDataPolicy.Ignore) {
-        return null;
+        return {
+          fingerprint: input.seriesFingerprint,
+          labels: input.seriesLabels,
+          rootCause: null,
+          context: metricContext,
+        };
       }
 
       if (policy === NoDataPolicy.Trigger) {
-        return `No data received for ${metricContext.metricName} in the evaluation window — triggering per no-data policy.`;
+        return {
+          fingerprint: input.seriesFingerprint,
+          labels: input.seriesLabels,
+          rootCause: `No data received for ${metricContext.metricName} in the evaluation window — triggering per no-data policy.`,
+          context: metricContext,
+        };
       }
 
       // TreatAsZero: fall through to the comparator with value 0.
@@ -206,7 +331,12 @@ export default class MetricMonitorCriteria {
       });
 
     if (!comparisonMessage) {
-      return null;
+      return {
+        fingerprint: input.seriesFingerprint,
+        labels: input.seriesLabels,
+        rootCause: null,
+        context: metricContext,
+      };
     }
 
     /*
@@ -232,8 +362,8 @@ export default class MetricMonitorCriteria {
       matchedFormula
         ? MetricMonitorCriteria.buildComponentValueLookup({
             components: metricContext.components || [],
-            queryConfigs,
-            formulaConfigs,
+            queryConfigs: input.queryConfigs,
+            formulaConfigs: input.formulaConfigs,
             metricAggregatedResult,
           })
         : null;
@@ -276,7 +406,12 @@ export default class MetricMonitorCriteria {
       metricContext.breachingSamples = breachingSamples;
     }
 
-    return comparisonMessage;
+    return {
+      fingerprint: input.seriesFingerprint,
+      labels: input.seriesLabels,
+      rootCause: comparisonMessage,
+      context: metricContext,
+    };
   }
 
   private static buildComponentValueLookup(input: {
@@ -438,6 +573,17 @@ export default class MetricMonitorCriteria {
       ? Object.keys(q.metricQueryData.groupBy as Record<string, unknown>)
       : [];
 
+    /*
+     * Include user-selected attribute keys as part of the groupBy view
+     * so the root-cause block shows "Grouped By: host.name" not just the
+     * raw columns ClickHouse was asked to partition on.
+     */
+    const groupByAttributeKeys: Array<string> =
+      q?.metricQueryData?.groupByAttributeKeys || [];
+    const allGroupBy: Array<string> = Array.from(
+      new Set([...groupBy, ...groupByAttributeKeys]),
+    );
+
     const components: Array<MetricComponent> | undefined = f
       ? MetricMonitorCriteria.buildFormulaComponents({
           formulaConfig: f,
@@ -454,7 +600,7 @@ export default class MetricMonitorCriteria {
       isFormula: Boolean(f),
       formulaExpression: f?.metricFormulaData?.metricFormula,
       filterAttributes,
-      groupBy,
+      groupBy: allGroupBy,
       timeWindowMinutes:
         input.criteriaFilter.evaluateOverTimeOptions?.timeValueInMinutes,
       ...(components && components.length > 0 ? { components } : {}),