@oneuptime/common 8.0.5438 → 8.0.5462

package/Server/Services/TelemetryUsageBillingService.ts

@@ -1,16 +1,31 @@
 import { MeteredPlanUtil } from "../Types/Billing/MeteredPlan/AllMeteredPlans";
 import TelemetryMeteredPlan from "../Types/Billing/MeteredPlan/TelemetryMeteredPlan";
-import QueryHelper from "../Types/Database/QueryHelper";
 import DatabaseService from "./DatabaseService";
 import SortOrder from "../../Types/BaseDatabase/SortOrder";
-import LIMIT_MAX from "../../Types/Database/LimitMax";
+import LIMIT_MAX, { LIMIT_INFINITY } from "../../Types/Database/LimitMax";
 import OneUptimeDate from "../../Types/Date";
 import Decimal from "../../Types/Decimal";
 import BadDataException from "../../Types/Exception/BadDataException";
 import ProductType from "../../Types/MeteredPlan/ProductType";
 import ObjectID from "../../Types/ObjectID";
-import Model from "../../Models/DatabaseModels/TelemetryUsageBilling";
-import { IsBillingEnabled } from "../EnvironmentConfig";
+import Model, {
+  DEFAULT_RETENTION_IN_DAYS,
+} from "../../Models/DatabaseModels/TelemetryUsageBilling";
+import TelemetryServiceService from "./TelemetryServiceService";
+import SpanService from "./SpanService";
+import LogService from "./LogService";
+import MetricService from "./MetricService";
+import AnalyticsQueryHelper from "../Types/AnalyticsDatabase/QueryHelper";
+import DiskSize from "../../Types/DiskSize";
+import logger from "../Utils/Logger";
+import PositiveNumber from "../../Types/PositiveNumber";
+import TelemetryServiceModel from "../../Models/DatabaseModels/TelemetryService";
+import {
+  AverageSpanRowSizeInBytes,
+  AverageLogRowSizeInBytes,
+  AverageMetricRowSizeInBytes,
+  IsBillingEnabled,
+} from "../EnvironmentConfig";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 
 export class Service extends DatabaseService<Model> {
@@ -31,9 +46,6 @@ export class Service extends DatabaseService<Model> {
         projectId: data.projectId,
         productType: data.productType,
         isReportedToBillingProvider: false,
-        createdAt: QueryHelper.lessThan(
-          OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), -1),
-        ), // we need to get everything that's not today.
       },
       skip: 0,
       limit: LIMIT_MAX, /// because a project can have MANY telemetry services.
@@ -47,6 +59,159 @@ export class Service extends DatabaseService<Model> {
     });
   }
 
+  @CaptureSpan()
+  public async stageTelemetryUsageForProject(data: {
+    projectId: ObjectID;
+    productType: ProductType;
+    usageDate?: Date;
+  }): Promise<void> {
+    if (!IsBillingEnabled) {
+      return;
+    }
+
+    const usageDate: Date = data.usageDate
+      ? OneUptimeDate.fromString(data.usageDate)
+      : OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), -1);
+
+    const averageRowSizeInBytes: number = this.getAverageRowSizeForProduct(
+      data.productType,
+    );
+
+    if (averageRowSizeInBytes <= 0) {
+      return;
+    }
+
+    const usageDayString: string = OneUptimeDate.getDateString(usageDate);
+    const startOfDay: Date = OneUptimeDate.getStartOfDay(usageDate);
+    const endOfDay: Date = OneUptimeDate.getEndOfDay(usageDate);
+
+    const telemetryServices: Array<TelemetryServiceModel> =
+      await TelemetryServiceService.findBy({
+        query: {
+          projectId: data.projectId,
+        },
+        select: {
+          _id: true,
+          retainTelemetryDataForDays: true,
+        },
+        skip: 0,
+        limit: LIMIT_MAX,
+        props: {
+          isRoot: true,
+        },
+      });
+
+    if (!telemetryServices || telemetryServices.length === 0) {
+      return;
+    }
+
+    for (const telemetryService of telemetryServices) {
+      if (!telemetryService?.id) {
+        continue;
+      }
+
+      const existingEntry: Model | null = await this.findOneBy({
+        query: {
+          projectId: data.projectId,
+          productType: data.productType,
+          telemetryServiceId: telemetryService.id,
+          day: usageDayString,
+        },
+        select: {
+          _id: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (existingEntry) {
+        continue;
+      }
+
+      let rowCount: number = 0;
+
+      try {
+        if (data.productType === ProductType.Traces) {
+          const count: PositiveNumber = await SpanService.countBy({
+            query: {
+              projectId: data.projectId,
+              serviceId: telemetryService.id,
+              startTime: AnalyticsQueryHelper.inBetween(startOfDay, endOfDay),
+            },
+            skip: 0,
+            limit: LIMIT_INFINITY,
+            props: {
+              isRoot: true,
+            },
+          });
+
+          rowCount = count.toNumber();
+        } else if (data.productType === ProductType.Logs) {
+          const count: PositiveNumber = await LogService.countBy({
+            query: {
+              projectId: data.projectId,
+              serviceId: telemetryService.id,
+              time: AnalyticsQueryHelper.inBetween(startOfDay, endOfDay),
+            },
+            skip: 0,
+            limit: LIMIT_INFINITY,
+            props: {
+              isRoot: true,
+            },
+          });
+
+          rowCount = count.toNumber();
+        } else if (data.productType === ProductType.Metrics) {
+          const count: PositiveNumber = await MetricService.countBy({
+            query: {
+              projectId: data.projectId,
+              serviceId: telemetryService.id,
+              time: AnalyticsQueryHelper.inBetween(startOfDay, endOfDay),
+            },
+            skip: 0,
+            limit: LIMIT_INFINITY,
+            props: {
+              isRoot: true,
+            },
+          });
+
+          rowCount = count.toNumber();
+        }
+      } catch (error) {
+        logger.error(
+          `Failed to compute telemetry usage for service ${telemetryService.id?.toString()}:`,
+        );
+        logger.error(error as Error);
+        continue;
+      }
+
+      if (rowCount <= 0) {
+        continue;
+      }
+
+      const estimatedBytes: number = rowCount * averageRowSizeInBytes;
+      const estimatedGigabytes: number = DiskSize.byteSizeToGB(estimatedBytes);
+
+      if (!Number.isFinite(estimatedGigabytes) || estimatedGigabytes <= 0) {
+        continue;
+      }
+
+      const dataRetentionInDays: number =
+        telemetryService.retainTelemetryDataForDays ||
+        DEFAULT_RETENTION_IN_DAYS;
+
+      await this.updateUsageBilling({
+        projectId: data.projectId,
+        productType: data.productType,
+        telemetryServiceId: telemetryService.id,
+        dataIngestedInGB: estimatedGigabytes,
+        retentionInDays: dataRetentionInDays,
+        usageDate: usageDate,
+      });
+    }
+  }
+
   @CaptureSpan()
   public async updateUsageBilling(data: {
     projectId: ObjectID;
@@ -54,6 +219,7 @@ export class Service extends DatabaseService<Model> {
     telemetryServiceId: ObjectID;
     dataIngestedInGB: number;
     retentionInDays: number;
+    usageDate?: Date;
   }): Promise<void> {
     if (
       data.productType !== ProductType.Traces &&
@@ -70,6 +236,12 @@ export class Service extends DatabaseService<Model> {
         data.productType,
       ) as TelemetryMeteredPlan;
 
+    const usageDate: Date = data.usageDate
+      ? OneUptimeDate.fromString(data.usageDate)
+      : OneUptimeDate.getCurrentDate();
+
+    const usageDayString: string = OneUptimeDate.getDateString(usageDate);
+
     const totalCostOfThisOperationInUSD: number =
       serverMeteredPlan.getTotalCostInUSD({
         dataIngestedInGB: data.dataIngestedInGB,
@@ -82,10 +254,7 @@ export class Service extends DatabaseService<Model> {
         productType: data.productType,
         telemetryServiceId: data.telemetryServiceId,
         isReportedToBillingProvider: false,
-        createdAt: QueryHelper.inBetween(
-          OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), -1),
-          OneUptimeDate.getCurrentDate(),
-        ),
+        day: usageDayString,
       },
       select: {
         _id: true,
@@ -135,11 +304,9 @@ export class Service extends DatabaseService<Model> {
       usageBilling.telemetryServiceId = data.telemetryServiceId;
       usageBilling.retainTelemetryDataForDays = data.retentionInDays;
       usageBilling.isReportedToBillingProvider = false;
-      usageBilling.createdAt = OneUptimeDate.getCurrentDate();
+      usageBilling.createdAt = usageDate;
 
-      usageBilling.day = OneUptimeDate.getDateString(
-        OneUptimeDate.getCurrentDate(),
-      );
+      usageBilling.day = usageDayString;
 
       usageBilling.totalCostInUSD = new Decimal(totalCostOfThisOperationInUSD);
 
@@ -151,6 +318,32 @@ export class Service extends DatabaseService<Model> {
       });
     }
   }
+
+  private getAverageRowSizeForProduct(productType: ProductType): number {
+    const fallbackSize: number = 1024;
+
+    // Narrow to telemetry product types before indexing to satisfy TypeScript
+    if (
+      productType !== ProductType.Traces &&
+      productType !== ProductType.Logs &&
+      productType !== ProductType.Metrics
+    ) {
+      return fallbackSize;
+    }
+
+    const value: number =
+      {
+        [ProductType.Traces]: AverageSpanRowSizeInBytes,
+        [ProductType.Logs]: AverageLogRowSizeInBytes,
+        [ProductType.Metrics]: AverageMetricRowSizeInBytes,
+      }[productType] ?? fallbackSize;
+
+    if (!Number.isFinite(value) || value <= 0) {
+      return fallbackSize;
+    }
+
+    return value;
+  }
 }
 
 export default new Service();

package/Server/Types/Billing/MeteredPlan/TelemetryMeteredPlan.ts

@@ -56,6 +56,11 @@ export default class TelemetryMeteredPlan extends ServerMeteredPlan {
   ): Promise<void> {
     // get all unreported logs
 
+    await TelemetryUsageBillingService.stageTelemetryUsageForProject({
+      projectId: projectId,
+      productType: this.productType,
+    });
+
     const usageBillings: Array<TelemetryUsageBilling> =
       await TelemetryUsageBillingService.getUnreportedUsageBilling({
         projectId: projectId,

package/Server/Utils/Telemetry/Telemetry.ts

@@ -143,10 +143,10 @@ export default class TelemetryUtil {
     let { prefixKeysWithString } = data;
 
     if (prefixKeysWithString) {
-      prefixKeysWithString = prefixKeysWithString + ".";
+      prefixKeysWithString = `${prefixKeysWithString}.`;
     }
 
-    let finalObj: Dictionary<AttributeType | Array<AttributeType>> = {};
+    const finalObj: Dictionary<AttributeType | Array<AttributeType>> = {};
     const attributes: JSONArray = items;
 
     if (!attributes) {
@@ -154,25 +154,40 @@ export default class TelemetryUtil {
     }
 
     for (const attribute of attributes) {
-      if (attribute["key"] && typeof attribute["key"] === "string") {
-        const keyWithPrefix: string = `${prefixKeysWithString}${attribute["key"]}`;
-
-        const value:
-          | AttributeType
-          | Dictionary<AttributeType>
-          | Array<AttributeType> = this.getAttributeValues(
-          keyWithPrefix,
-          attribute["value"],
-        );
-
-        if (Array.isArray(value)) {
-          finalObj = { ...finalObj, [keyWithPrefix]: value };
-        } else if (typeof value === "object" && value !== null) {
-          finalObj = { ...finalObj, ...(value as Dictionary<AttributeType>) };
-        } else {
-          finalObj[keyWithPrefix] = value;
+      if (!attribute["key"] || typeof attribute["key"] !== "string") {
+        continue;
+      }
+
+      const keyWithPrefix: string = `${prefixKeysWithString}${attribute["key"]}`;
+
+      const value = this.getAttributeValues(
+        keyWithPrefix,
+        attribute["value"],
+      );
+
+      if (value === null) {
+        finalObj[keyWithPrefix] = null;
+        continue;
+      }
+
+      if (Array.isArray(value)) {
+        finalObj[keyWithPrefix] = value;
+        continue;
+      }
+
+      if (typeof value === "object") {
+        for (const [nestedKey, nestedValue] of Object.entries(
+          value as Dictionary<AttributeType | Array<AttributeType>>,
+        )) {
+          finalObj[nestedKey] = nestedValue as
+            | AttributeType
+            | Array<AttributeType>;
         }
+
+        continue;
       }
+
+      finalObj[keyWithPrefix] = value as AttributeType;
     }
 
     return finalObj;
@@ -181,73 +196,106 @@ export default class TelemetryUtil {
   public static getAttributeValues(
     prefixKeysWithString: string,
     value: JSONValue,
-  ): AttributeType | Dictionary<AttributeType> | Array<AttributeType> {
+  ):
+    | AttributeType
+    | Dictionary<AttributeType | Array<AttributeType>>
+    | Array<AttributeType>
+    | null {
     let finalObj:
-      | Dictionary<AttributeType>
+      | Dictionary<AttributeType | Array<AttributeType>>
       | AttributeType
-      | Array<AttributeType> = null;
-    value = value as JSONObject;
-
-    if (value["stringValue"]) {
-      finalObj = value["stringValue"] as string;
-    } else if (value["intValue"]) {
-      finalObj = value["intValue"] as number;
-    } else if (value["doubleValue"]) {
-      finalObj = value["doubleValue"] as number;
-    } else if (value["boolValue"]) {
-      finalObj = value["boolValue"] as boolean;
-    } else if (
-      value["arrayValue"] &&
-      (value["arrayValue"] as JSONObject)["values"]
-    ) {
-      const values: JSONArray = (value["arrayValue"] as JSONObject)[
-        "values"
-      ] as JSONArray;
-      finalObj = values.map((v: JSONObject) => {
-        return this.getAttributeValues(
-          prefixKeysWithString,
-          v,
-        ) as AttributeType;
-      }) as Array<AttributeType>;
-    } else if (
-      value["mapValue"] &&
-      (value["mapValue"] as JSONObject)["fields"]
-    ) {
-      const fields: JSONObject = (value["mapValue"] as JSONObject)?.[
-        "fields"
-      ] as JSONObject;
-
-      const flattenedFields: Dictionary<AttributeType> = {};
-      for (const key in fields) {
-        const prefixKey: string = `${prefixKeysWithString}.${key}`;
-        const nestedValue: AttributeType | Dictionary<AttributeType> =
-          this.getAttributeValues(prefixKey, fields[key]) as AttributeType;
-        if (typeof nestedValue === "object" && nestedValue !== null) {
-          for (const nestedKey in nestedValue as Dictionary<AttributeType>) {
-            flattenedFields[`${prefixKey}.${nestedKey}`] = (
-              nestedValue as Dictionary<AttributeType>
-            )[nestedKey] as AttributeType;
+      | Array<AttributeType>
+      | null = null;
+    const jsonValue: JSONObject = value as JSONObject;
+
+    if (jsonValue && typeof jsonValue === "object") {
+      if (Object.prototype.hasOwnProperty.call(jsonValue, "stringValue")) {
+        const stringValue: JSONValue = jsonValue["stringValue"];
+        finalObj =
+          stringValue !== undefined && stringValue !== null
+            ? (stringValue as string)
+            : "";
+      } else if (Object.prototype.hasOwnProperty.call(jsonValue, "intValue")) {
+        const intValue: JSONValue = jsonValue["intValue"];
+        if (intValue !== undefined && intValue !== null) {
+          finalObj = intValue as number;
+        }
+      } else if (
+        Object.prototype.hasOwnProperty.call(jsonValue, "doubleValue")
+      ) {
+        const doubleValue: JSONValue = jsonValue["doubleValue"];
+        if (doubleValue !== undefined && doubleValue !== null) {
+          finalObj = doubleValue as number;
+        }
+      } else if (Object.prototype.hasOwnProperty.call(jsonValue, "boolValue")) {
+        finalObj = jsonValue["boolValue"] as boolean;
+      } else if (
+        jsonValue["arrayValue"] &&
+        (jsonValue["arrayValue"] as JSONObject)["values"]
+      ) {
+        const values: JSONArray = (jsonValue["arrayValue"] as JSONObject)[
+          "values"
+        ] as JSONArray;
+        finalObj = values.map((v: JSONObject) => {
+          return this.getAttributeValues(prefixKeysWithString, v) as AttributeType;
+        }) as Array<AttributeType>;
+      } else if (
+        jsonValue["mapValue"] &&
+        (jsonValue["mapValue"] as JSONObject)["fields"]
+      ) {
+        const fields: JSONObject = (jsonValue["mapValue"] as JSONObject)[
+          "fields"
+        ] as JSONObject;
+
+        const flattenedFields: Dictionary<AttributeType | Array<AttributeType>> =
+          {};
+        for (const key in fields) {
+          const nestedPrefix: string = `${prefixKeysWithString}.${key}`;
+          const nestedValue = this.getAttributeValues(
+            nestedPrefix,
+            fields[key],
+          );
+
+          if (nestedValue === null) {
+            flattenedFields[nestedPrefix] = null;
+            continue;
           }
-        } else {
-          flattenedFields[prefixKey] = nestedValue;
+
+          if (Array.isArray(nestedValue)) {
+            flattenedFields[nestedPrefix] = nestedValue;
+            continue;
+          }
+
+          if (typeof nestedValue === "object") {
+            for (const [nestedKey, nestedEntry] of Object.entries(
+              nestedValue as Dictionary<AttributeType | Array<AttributeType>>,
+            )) {
+              flattenedFields[nestedKey] = nestedEntry as
+                | AttributeType
+                | Array<AttributeType>;
+            }
+
+            continue;
+          }
+
+          flattenedFields[nestedPrefix] = nestedValue as AttributeType;
         }
+
+        finalObj = flattenedFields;
+      } else if (
+        jsonValue["kvlistValue"] &&
+        (jsonValue["kvlistValue"] as JSONObject)["values"]
+      ) {
+        const values: JSONArray = (jsonValue["kvlistValue"] as JSONObject)[
+          "values"
+        ] as JSONArray;
+        finalObj = this.getAttributes({
+          prefixKeysWithString,
+          items: values,
+        });
+      } else if ("nullValue" in jsonValue) {
+        finalObj = null;
       }
-      finalObj = flattenedFields;
-    }
-    // kvlistValue
-    else if (
-      value["kvlistValue"] &&
-      (value["kvlistValue"] as JSONObject)["values"]
-    ) {
-      const values: JSONArray = (value["kvlistValue"] as JSONObject)[
-        "values"
-      ] as JSONArray;
-      finalObj = this.getAttributes({
-        prefixKeysWithString,
-        items: values,
-      }) as Dictionary<AttributeType>;
-    } else if (value["nullValue"]) {
-      finalObj = null;
     }
 
     return finalObj;

package/Server/Utils/VM/VMRunner.ts

@@ -48,10 +48,9 @@ export default class VMRunner {
 
     vm.createContext(sandbox); // Contextify the object.
 
-    const script: string =
-      `(async()=>{
-            ${code}
-        })()` || "";
+    const script: string = `(async()=>{
+        ${code}
+      })()`;
 
     const returnVal: any = await vm.runInContext(script, sandbox, {
       timeout: options.timeout || 5000,

package/UI/Components/Dictionary/Dictionary.tsx

@@ -148,6 +148,7 @@ const DictionaryForm: FunctionComponent<ComponentProps> = (
                     title="Key"
                     required={true}
                     hideOptionalLabel={true}
+                    className="block text-xs text-gray-500 font-normal flex justify-between"
                   />
                 </div>
                 <AutocompleteTextInput
@@ -173,6 +174,7 @@ const DictionaryForm: FunctionComponent<ComponentProps> = (
                       title="Type"
                       hideOptionalLabel={true}
                       required={true}
+                      className="block text-xs text-gray-500 font-normal flex justify-between"
                     />
                   </div>
                   <Dropdown
@@ -206,6 +208,7 @@ const DictionaryForm: FunctionComponent<ComponentProps> = (
                     title="Value"
                     hideOptionalLabel={true}
                     required={true}
+                    className="block text-xs text-gray-500 font-normal flex justify-between"
                   />
                 </div>
                 {item.type === ValueType.Text && (

package/UI/Components/Forms/Fields/FieldLabel.tsx

@@ -9,6 +9,7 @@ export interface ComponentProps {
   description?: string | ReactElement | undefined;
   isHeading?: boolean | undefined;
   hideOptionalLabel?: boolean | undefined;
+  className?: string | undefined;
 }
 
 const FieldLabelElement: FunctionComponent<ComponentProps> = (
@@ -17,9 +18,12 @@ const FieldLabelElement: FunctionComponent<ComponentProps> = (
   return (
     <>
       <label
-        className={`block ${
-          props.isHeading ? "text-lg" : "text-sm"
-        }  font-medium text-gray-700 flex justify-between`}
+        className={
+          props.className ||
+          `block ${
+            props.isHeading ? "text-lg" : "text-sm"
+          }  font-medium text-gray-700 flex justify-between`
+        }
       >
         <span>
           {props.title}{" "}

package/UI/Components/LogsViewer/LogItem.tsx

@@ -265,7 +265,7 @@ const LogItem: FunctionComponent<ComponentProps> = (
   if (isCollapsed) {
     return (
       <div
-        className={`group relative text-slate-200 flex items-center gap-2 cursor-pointer hover:bg-slate-800/40 px-2 py-0.5 border border-transparent border-l ${leftBorderColor} rounded-sm transition-colors duration-100`}
+        className={`group relative text-slate-200 flex items-center gap-2 cursor-pointer hover:bg-slate-800/40 px-2 py-0.5 border border-transparent border-l ${leftBorderColor} rounded-sm transition-colors duration-100 font-mono`}
         onClick={toggleCollapsed}
         role="button"
         aria-expanded={!isCollapsed}
@@ -311,11 +311,14 @@ const LogItem: FunctionComponent<ComponentProps> = (
           className={`${bodyColor} font-mono text-[13px] md:text-sm leading-5 tracking-tight subpixel-antialiased flex-1 min-w-0`}
         >
           {isBodyInJSON ? (
-            <div className="truncate" title={logBodyMinified}>
+            <div className="truncate font-mono" title={logBodyMinified}>
               {logBodyMinified}
             </div>
           ) : (
-            <div className="truncate" title={props.log.body?.toString()}>
+            <div
+              className="truncate font-mono"
+              title={props.log.body?.toString()}
+            >
               {props.log.body?.toString()}
             </div>
           )}
@@ -339,7 +342,12 @@ const LogItem: FunctionComponent<ComponentProps> = (
       className={`group relative text-slate-200 bg-slate-950/70 border ${leftBorderColor} border-l border-slate-900 rounded-sm p-2 hover:border-slate-700 transition-colors`}
     >
       {/* Header with Service Name and Close Indicator */}
-      <div className="flex items-center justify-between mb-1 pb-1 border-b border-slate-800/80">
+      <div
+        className="flex items-center justify-between mb-1 pb-1 border-b border-slate-800/80"
+        onClick={() => {
+          toggleCollapsed();
+        }}
+      >
         {serviceName && (
           <div
             className="text-[13px] font-semibold"