@oneuptime/common 8.0.5440 → 8.0.5466

package/Server/Services/StatusPageService.ts

@@ -1150,5 +1150,122 @@ export class Service extends DatabaseService<StatusPage> {
       },
     );
   }
+
+  @CaptureSpan()
+  public async getMonitorGroupCurrentStatuses(data: {
+    statusPageResources: Array<StatusPageResource>;
+    monitorStatuses: Array<MonitorStatus>;
+  }): Promise<Dictionary<ObjectID>> {
+    const monitorGroupCurrentStatuses: Dictionary<ObjectID> = {};
+
+    for (const resource of data.statusPageResources) {
+      if (resource.monitorGroupId) {
+        const monitorGroupResources: Array<MonitorGroupResource> =
+          await MonitorGroupResourceService.findBy({
+            query: {
+              monitorGroupId: resource.monitorGroupId,
+            },
+            select: {
+              monitorId: true,
+              monitor: {
+                currentMonitorStatusId: true,
+              },
+            },
+            skip: 0,
+            limit: LIMIT_PER_PROJECT,
+            props: {
+              isRoot: true,
+            },
+          });
+
+        const statuses: Array<ObjectID> = monitorGroupResources
+          .filter((item: MonitorGroupResource) => {
+            return (
+              item.monitor &&
+              item.monitor.currentMonitorStatusId &&
+              item.monitorId
+            );
+          })
+          .map((item: MonitorGroupResource) => {
+            return item.monitor!.currentMonitorStatusId!;
+          });
+
+        let worstStatus: MonitorStatus | null = null;
+
+        for (const statusId of statuses) {
+          const status: MonitorStatus | undefined = data.monitorStatuses.find(
+            (status: MonitorStatus) => {
+              return status._id?.toString() === statusId.toString();
+            },
+          );
+
+          if (
+            status &&
+            (!worstStatus || status.priority! < worstStatus.priority!)
+          ) {
+            worstStatus = status;
+          }
+        }
+
+        if (worstStatus && worstStatus._id) {
+          monitorGroupCurrentStatuses[resource.monitorGroupId.toString()] =
+            new ObjectID(worstStatus._id);
+        }
+      }
+    }
+
+    return monitorGroupCurrentStatuses;
+  }
+
+  @CaptureSpan()
+  public getOverallMonitorStatus(data: {
+    statusPageResources: Array<StatusPageResource>;
+    monitorStatuses: Array<MonitorStatus>;
+    monitorGroupCurrentStatuses: Dictionary<ObjectID>;
+  }): MonitorStatus | null {
+    let currentStatus: MonitorStatus | null =
+      data.monitorStatuses.length > 0 && data.monitorStatuses[0]
+        ? data.monitorStatuses[0]
+        : null;
+
+    const dict: Dictionary<number> = {};
+
+    for (const resource of data.statusPageResources) {
+      if (resource.monitor?.currentMonitorStatusId) {
+        if (
+          !Object.keys(dict).includes(
+            resource.monitor?.currentMonitorStatusId.toString() || "",
+          )
+        ) {
+          dict[resource.monitor?.currentMonitorStatusId?.toString()] = 1;
+        } else {
+          dict[resource.monitor!.currentMonitorStatusId!.toString()]!++;
+        }
+      }
+    }
+
+    // check status of monitor groups.
+
+    for (const groupId in data.monitorGroupCurrentStatuses) {
+      const statusId: ObjectID | undefined =
+        data.monitorGroupCurrentStatuses[groupId];
+
+      if (statusId) {
+        if (!Object.keys(dict).includes(statusId.toString() || "")) {
+          dict[statusId.toString()] = 1;
+        } else {
+          dict[statusId.toString()]!++;
+        }
+      }
+    }
+
+    for (const monitorStatus of data.monitorStatuses) {
+      if (monitorStatus._id && dict[monitorStatus._id]) {
+        currentStatus = monitorStatus;
+      }
+    }
+
+    return currentStatus;
+  }
 }
 export default new Service();

package/Server/Services/TelemetryUsageBillingService.ts

@@ -1,16 +1,33 @@
 import { MeteredPlanUtil } from "../Types/Billing/MeteredPlan/AllMeteredPlans";
 import TelemetryMeteredPlan from "../Types/Billing/MeteredPlan/TelemetryMeteredPlan";
-import QueryHelper from "../Types/Database/QueryHelper";
 import DatabaseService from "./DatabaseService";
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
-import LIMIT_MAX from "../../Types/Database/LimitMax";
+import LIMIT_MAX, { LIMIT_INFINITY } from "../../Types/Database/LimitMax";
 import OneUptimeDate from "../../Types/Date";
 import Decimal from "../../Types/Decimal";
 import BadDataException from "../../Types/Exception/BadDataException";
 import ProductType from "../../Types/MeteredPlan/ProductType";
 import ObjectID from "../../Types/ObjectID";
-import Model from "../../Models/DatabaseModels/TelemetryUsageBilling";
-import { IsBillingEnabled } from "../EnvironmentConfig";
+import Model, {
+  DEFAULT_RETENTION_IN_DAYS,
+} from "../../Models/DatabaseModels/TelemetryUsageBilling";
+import TelemetryServiceService from "./TelemetryServiceService";
+import SpanService from "./SpanService";
+import LogService from "./LogService";
+import MetricService from "./MetricService";
+import ExceptionInstanceService from "./ExceptionInstanceService";
+import AnalyticsQueryHelper from "../Types/AnalyticsDatabase/QueryHelper";
+import DiskSize from "../../Types/DiskSize";
+import logger from "../Utils/Logger";
+import PositiveNumber from "../../Types/PositiveNumber";
+import TelemetryServiceModel from "../../Models/DatabaseModels/TelemetryService";
+import {
+  AverageSpanRowSizeInBytes,
+  AverageLogRowSizeInBytes,
+  AverageMetricRowSizeInBytes,
+  AverageExceptionRowSizeInBytes,
+  IsBillingEnabled,
+} from "../EnvironmentConfig";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 
 export class Service extends DatabaseService<Model> {
@@ -31,9 +48,6 @@ export class Service extends DatabaseService<Model> {
         projectId: data.projectId,
         productType: data.productType,
         isReportedToBillingProvider: false,
-        createdAt: QueryHelper.lessThan(
-          OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), -1),
-        ), // we need to get everything that's not today.
       },
       skip: 0,
       limit: LIMIT_MAX, /// because a project can have MANY telemetry services.
@@ -47,6 +61,203 @@ export class Service extends DatabaseService<Model> {
     });
   }
 
+  @CaptureSpan()
+  public async stageTelemetryUsageForProject(data: {
+    projectId: ObjectID;
+    productType: ProductType;
+    usageDate?: Date;
+  }): Promise<void> {
+    if (!IsBillingEnabled) {
+      return;
+    }
+
+    const usageDate: Date = data.usageDate
+      ? OneUptimeDate.fromString(data.usageDate)
+      : OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), -1);
+
+    const averageRowSizeInBytes: number = this.getAverageRowSizeForProduct(
+      data.productType,
+    );
+    const averageExceptionRowSizeInBytes: number =
+      this.getAverageExceptionRowSize();
+
+    if (data.productType !== ProductType.Traces && averageRowSizeInBytes <= 0) {
+      return;
+    }
+
+    if (
+      data.productType === ProductType.Traces &&
+      averageRowSizeInBytes <= 0 &&
+      averageExceptionRowSizeInBytes <= 0
+    ) {
+      return;
+    }
+
+    const usageDayString: string = OneUptimeDate.getDateString(usageDate);
+    const startOfDay: Date = OneUptimeDate.getStartOfDay(usageDate);
+    const endOfDay: Date = OneUptimeDate.getEndOfDay(usageDate);
+
+    const telemetryServices: Array<TelemetryServiceModel> =
+      await TelemetryServiceService.findBy({
+        query: {
+          projectId: data.projectId,
+        },
+        select: {
+          _id: true,
+          retainTelemetryDataForDays: true,
+        },
+        skip: 0,
+        limit: LIMIT_MAX,
+        props: {
+          isRoot: true,
+        },
+      });
+
+    if (!telemetryServices || telemetryServices.length === 0) {
+      return;
+    }
+
+    for (const telemetryService of telemetryServices) {
+      if (!telemetryService?.id) {
+        continue;
+      }
+
+      const existingEntry: Model | null = await this.findOneBy({
+        query: {
+          projectId: data.projectId,
+          productType: data.productType,
+          telemetryServiceId: telemetryService.id,
+          day: usageDayString,
+        },
+        select: {
+          _id: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (existingEntry) {
+        continue;
+      }
+
+      let estimatedBytes: number = 0;
+
+      try {
+        if (data.productType === ProductType.Traces) {
+          const spanCount: PositiveNumber = await SpanService.countBy({
+            query: {
+              projectId: data.projectId,
+              serviceId: telemetryService.id,
+              startTime: AnalyticsQueryHelper.inBetween(startOfDay, endOfDay),
+            },
+            skip: 0,
+            limit: LIMIT_INFINITY,
+            props: {
+              isRoot: true,
+            },
+          });
+
+          const exceptionCount: PositiveNumber =
+            await ExceptionInstanceService.countBy({
+              query: {
+                projectId: data.projectId,
+                serviceId: telemetryService.id,
+                time: AnalyticsQueryHelper.inBetween(startOfDay, endOfDay),
+              },
+              skip: 0,
+              limit: LIMIT_INFINITY,
+              props: {
+                isRoot: true,
+              },
+            });
+
+          const totalSpanCount: number = spanCount.toNumber();
+          const totalExceptionCount: number = exceptionCount.toNumber();
+
+          if (totalSpanCount <= 0 && totalExceptionCount <= 0) {
+            continue;
+          }
+
+          estimatedBytes =
+            totalSpanCount * averageRowSizeInBytes +
+            totalExceptionCount * averageExceptionRowSizeInBytes;
+        } else if (data.productType === ProductType.Logs) {
+          const count: PositiveNumber = await LogService.countBy({
+            query: {
+              projectId: data.projectId,
+              serviceId: telemetryService.id,
+              time: AnalyticsQueryHelper.inBetween(startOfDay, endOfDay),
+            },
+            skip: 0,
+            limit: LIMIT_INFINITY,
+            props: {
+              isRoot: true,
+            },
+          });
+
+          const totalRowCount: number = count.toNumber();
+
+          if (totalRowCount <= 0) {
+            continue;
+          }
+
+          estimatedBytes = totalRowCount * averageRowSizeInBytes;
+        } else if (data.productType === ProductType.Metrics) {
+          const count: PositiveNumber = await MetricService.countBy({
+            query: {
+              projectId: data.projectId,
+              serviceId: telemetryService.id,
+              time: AnalyticsQueryHelper.inBetween(startOfDay, endOfDay),
+            },
+            skip: 0,
+            limit: LIMIT_INFINITY,
+            props: {
+              isRoot: true,
+            },
+          });
+
+          const totalRowCount: number = count.toNumber();
+
+          if (totalRowCount <= 0) {
+            continue;
+          }
+
+          estimatedBytes = totalRowCount * averageRowSizeInBytes;
+        }
+      } catch (error) {
+        logger.error(
+          `Failed to compute telemetry usage for service ${telemetryService.id?.toString()}:`,
+        );
+        logger.error(error as Error);
+        continue;
+      }
+
+      if (estimatedBytes <= 0) {
+        continue;
+      }
+
+      const estimatedGigabytes: number = DiskSize.byteSizeToGB(estimatedBytes);
+
+      if (!Number.isFinite(estimatedGigabytes) || estimatedGigabytes <= 0) {
+        continue;
+      }
+
+      const dataRetentionInDays: number =
+        telemetryService.retainTelemetryDataForDays ||
+        DEFAULT_RETENTION_IN_DAYS;
+
+      await this.updateUsageBilling({
+        projectId: data.projectId,
+        productType: data.productType,
+        telemetryServiceId: telemetryService.id,
+        dataIngestedInGB: estimatedGigabytes,
+        retentionInDays: dataRetentionInDays,
+        usageDate: usageDate,
+      });
+    }
+  }
+
   @CaptureSpan()
   public async updateUsageBilling(data: {
     projectId: ObjectID;
@@ -54,6 +265,7 @@ export class Service extends DatabaseService<Model> {
     telemetryServiceId: ObjectID;
     dataIngestedInGB: number;
     retentionInDays: number;
+    usageDate?: Date;
   }): Promise<void> {
     if (
       data.productType !== ProductType.Traces &&
@@ -70,6 +282,12 @@ export class Service extends DatabaseService<Model> {
         data.productType,
       ) as TelemetryMeteredPlan;
 
+    const usageDate: Date = data.usageDate
+      ? OneUptimeDate.fromString(data.usageDate)
+      : OneUptimeDate.getCurrentDate();
+
+    const usageDayString: string = OneUptimeDate.getDateString(usageDate);
+
     const totalCostOfThisOperationInUSD: number =
       serverMeteredPlan.getTotalCostInUSD({
         dataIngestedInGB: data.dataIngestedInGB,
@@ -82,10 +300,7 @@ export class Service extends DatabaseService<Model> {
         productType: data.productType,
         telemetryServiceId: data.telemetryServiceId,
         isReportedToBillingProvider: false,
-        createdAt: QueryHelper.inBetween(
-          OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), -1),
-          OneUptimeDate.getCurrentDate(),
-        ),
+        day: usageDayString,
       },
       select: {
         _id: true,
@@ -135,11 +350,9 @@ export class Service extends DatabaseService<Model> {
       usageBilling.telemetryServiceId = data.telemetryServiceId;
       usageBilling.retainTelemetryDataForDays = data.retentionInDays;
       usageBilling.isReportedToBillingProvider = false;
-      usageBilling.createdAt = OneUptimeDate.getCurrentDate();
+      usageBilling.createdAt = usageDate;
 
-      usageBilling.day = OneUptimeDate.getDateString(
-        OneUptimeDate.getCurrentDate(),
-      );
+      usageBilling.day = usageDayString;
 
       usageBilling.totalCostInUSD = new Decimal(totalCostOfThisOperationInUSD);
 
@@ -151,6 +364,46 @@ export class Service extends DatabaseService<Model> {
       });
     }
   }
+
+  private getAverageRowSizeForProduct(productType: ProductType): number {
+    const fallbackSize: number = 1024;
+
+    // Narrow to telemetry product types before indexing to satisfy TypeScript
+    if (
+      productType !== ProductType.Traces &&
+      productType !== ProductType.Logs &&
+      productType !== ProductType.Metrics
+    ) {
+      return fallbackSize;
+    }
+
+    const value: number =
+      {
+        [ProductType.Traces]: AverageSpanRowSizeInBytes,
+        [ProductType.Logs]: AverageLogRowSizeInBytes,
+        [ProductType.Metrics]: AverageMetricRowSizeInBytes,
+      }[productType] ?? fallbackSize;
+
+    if (!Number.isFinite(value) || value <= 0) {
+      return fallbackSize;
+    }
+
+    return value;
+  }
+
+  private getAverageExceptionRowSize(): number {
+    const fallbackSize: number = 1024;
+
+    if (!Number.isFinite(AverageExceptionRowSizeInBytes)) {
+      return fallbackSize;
+    }
+
+    if (AverageExceptionRowSizeInBytes <= 0) {
+      return fallbackSize;
+    }
+
+    return AverageExceptionRowSizeInBytes;
+  }
 }
 
 export default new Service();

package/Server/Types/Billing/MeteredPlan/TelemetryMeteredPlan.ts

@@ -56,6 +56,11 @@ export default class TelemetryMeteredPlan extends ServerMeteredPlan {
   ): Promise<void> {
     // get all unreported logs
 
+    await TelemetryUsageBillingService.stageTelemetryUsageForProject({
+      projectId: projectId,
+      productType: this.productType,
+    });
+
     const usageBillings: Array<TelemetryUsageBilling> =
       await TelemetryUsageBillingService.getUnreportedUsageBilling({
         projectId: projectId,