@oneuptime/common 13.0.0 → 13.0.1

package/Server/Services/ProjectService.ts

@@ -113,6 +113,7 @@ import DatabaseCommonInteractionProps from "../../Types/BaseDatabase/DatabaseCom
 import PositiveNumber from "../../Types/PositiveNumber";
 import Semaphore, { SemaphoreMutex } from "../Infrastructure/Semaphore";
 import InMemoryTTLCache from "../Infrastructure/InMemoryTTLCache";
+import { IsNull, UpdateResult } from "typeorm";
 
 export interface CurrentPlan {
   plan: PlanType | null;
@@ -2704,6 +2705,41 @@ These are no longer recorded against the project and have to be cancelled by han
     }
   }
 
+  /**
+   * Record seat synchronization state only for the subscription that was read.
+   * The normal update path locates rows first and then writes by id, which can
+   * overwrite a replacement subscription's state during a concurrent plan change.
+   */
+  @CaptureSpan()
+  public async updateSubscriptionSeats(data: {
+    projectId: ObjectID;
+    subscriptionId: string;
+    planId: string;
+    seats: number | null;
+  }): Promise<number> {
+    const result: UpdateResult = await this.getRepository().update(
+      {
+        _id: data.projectId.toString(),
+        paymentProviderSubscriptionId: data.subscriptionId,
+        paymentProviderPlanId: data.planId,
+        deletedAt: IsNull(),
+      },
+      {
+        paymentProviderSubscriptionSeats:
+          data.seats === null
+            ? () => {
+                return "NULL";
+              }
+            : data.seats,
+        version: () => {
+          return '"version" + 1';
+        },
+      },
+    );
+
+    return result.affected || 0;
+  }
+
   @CaptureSpan()
   public async reactiveSubscription(projectId: ObjectID): Promise<void> {
     logger.debug("Reactivating subscription for project " + projectId, {
@@ -2744,7 +2780,15 @@ These are no longer recorded against the project and have to be cancelled by han
       );
     }
 
-    if (!project.paymentProviderSubscriptionSeats) {
+    /*
+     * A pending seat synchronization leaves the acknowledged count empty.
+     * Reactivation can still use the memberships already stored in the project.
+     */
+    const seats: number =
+      project.paymentProviderSubscriptionSeats ??
+      (await TeamMemberService.getUniqueTeamMemberCountInProject(projectId));
+
+    if (!seats) {
       throw new BadDataException(
         "Payment Provider subscription seats not found",
       );
@@ -2792,7 +2836,7 @@ These are no longer recorded against the project and have to be cancelled by han
       meteredSubscriptionId: project.paymentProviderMeteredSubscriptionId,
       serverMeteredPlans: AllMeteredPlans,
       newPlan: subscriptionPlan,
-      quantity: project.paymentProviderSubscriptionSeats,
+      quantity: seats,
       isYearly: SubscriptionPlan.isYearlyPlan(project.paymentProviderPlanId),
       endTrialAt: endTrialAt,
     });

package/Server/Services/ServerlessFunctionInstanceService.ts

@@ -2,6 +2,8 @@ import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/ServerlessFunctionInstance";
 import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
+import QueryHelper from "../Types/Database/QueryHelper";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import logger from "../Utils/Logger";
 
@@ -58,6 +60,75 @@ export class Service extends DatabaseService<Model> {
       );
     }
   }
+
+  /**
+   * Hard-delete every instance of one serverless function whose
+   * lastSeenAt is older than olderThan. Returns the number of rows
+   * removed.
+   *
+   * faas.instance identifies a warm execution environment, and FaaS
+   * platforms recycle those constantly — every cold start mints a new
+   * one and nothing ever reports the old one as retired; it simply
+   * stops sending. Without this sweep the Instances tab counts every
+   * environment that ever ran the function. Called only by the
+   * Serverless:CleanupStaleResources worker, which anchors olderThan
+   * to the parent function's own lastSeenAt (see the job header for
+   * why wall-clock now is the wrong anchor).
+   */
+  @CaptureSpan()
+  public async deleteStaleForFunction(data: {
+    serverlessFunctionId: ObjectID;
+    olderThan: Date;
+  }): Promise<number> {
+    return await this.deleteBy({
+      query: {
+        serverlessFunctionId: data.serverlessFunctionId,
+        lastSeenAt: QueryHelper.lessThan(data.olderThan),
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: { isRoot: true },
+    });
+  }
+
+  /**
+   * Helper for the cleanup worker: ingest-cadence aware cutoff.
+   * 3x the 5-minute OTel ingest maintenance fence by default. Tune via
+   * SERVERLESS_INSTANCE_STALE_MINUTES (min 10).
+   */
+  public getStaleThresholdDate(nowOverride?: Date): Date {
+    const minutes: number = this.getStaleThresholdMinutes();
+    return OneUptimeDate.addRemoveMinutes(
+      nowOverride || OneUptimeDate.getCurrentDate(),
+      -minutes,
+    );
+  }
+
+  /*
+   * Threshold must stay well above the 5-minute OTel ingest maintenance
+   * fence (MAINTENANCE_FENCE_TTL_SECONDS in OtelIngestBaseService).
+   * recordInstance sits behind that fence per (function, instance), so
+   * a live instance's lastSeenAt is legitimately up to ~5 minutes stale
+   * during continuous telemetry; a threshold at or below the fence
+   * would prune live instances between refreshes. 15 minutes gives 3x
+   * headroom and matches ServerlessFunctionService.markDisconnectedFunctions.
+   * The floor is 10 minutes, not the fence's nominal 5: the fence key is
+   * armed with jitter (300-375 s) behind a 30-second negative memo, so a
+   * live row's lastSeenAt can legitimately be close to 7 minutes old. A
+   * floor at the nominal TTL would let an override prune live rows.
+   * Anything unparseable falls back to the default.
+   */
+  public getStaleThresholdMinutes(): number {
+    const raw: string | undefined =
+      process.env["SERVERLESS_INSTANCE_STALE_MINUTES"];
+    if (raw) {
+      const parsed: number = parseInt(raw, 10);
+      if (!isNaN(parsed) && parsed >= 10) {
+        return parsed;
+      }
+    }
+    return 15;
+  }
 }
 
 export default new Service();

package/Server/Services/TeamMemberService.ts

@@ -1,5 +1,6 @@
 import DatabaseConfig from "../DatabaseConfig";
 import { IsBillingEnabled } from "../EnvironmentConfig";
+import Semaphore, { SemaphoreMutex } from "../Infrastructure/Semaphore";
 import CreateBy from "../Types/Database/CreateBy";
 import DeleteBy from "../Types/Database/DeleteBy";
 import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
@@ -174,18 +175,26 @@ export class TeamMemberService extends DatabaseService<TeamMember> {
         id: createBy.data.projectId!,
         select: {
           seatLimit: true,
-          paymentProviderSubscriptionSeats: true,
         },
         props: {
           isRoot: true,
         },
       });
 
+      /*
+       * Billing can lag after a provider outage. Admission limits must use
+       * persisted memberships, including pending invitations.
+       */
+      const numberOfMembers: number =
+        project &&
+        (project.seatLimit || createBy.props.currentPlan === PlanType.Free)
+          ? await this.getUniqueTeamMemberCountInProject(projectId)
+          : 0;
+
       if (
         project &&
         project.seatLimit &&
-        project.paymentProviderSubscriptionSeats &&
-        project.paymentProviderSubscriptionSeats >= project.seatLimit
+        numberOfMembers >= project.seatLimit
       ) {
         throw new BadDataException(Errors.TeamMemberService.LIMIT_REACHED);
       }
@@ -193,8 +202,7 @@ export class TeamMemberService extends DatabaseService<TeamMember> {
       if (
         createBy.props.currentPlan === PlanType.Free &&
         project &&
-        project.paymentProviderSubscriptionSeats &&
-        project.paymentProviderSubscriptionSeats >= 1
+        numberOfMembers >= 1
       ) {
         throw new BadDataException(
           Errors.TeamMemberService.LIMIT_REACHED_FOR_FREE_PLAN,
@@ -535,7 +543,7 @@ export class TeamMemberService extends DatabaseService<TeamMember> {
       onCreate.createBy.data.projectId!,
     );
 
-    await this.updateSubscriptionSeatsByUniqueTeamMembersInProject(
+    await this.syncSubscriptionSeatsAfterMembershipChange(
       onCreate.createBy.data.projectId!,
     );
 
@@ -726,9 +734,7 @@ export class TeamMemberService extends DatabaseService<TeamMember> {
 
     for (const item of onDelete.carryForward as Array<TeamMember>) {
       await this.refreshTokens(item.userId!, item.projectId!);
-      await this.updateSubscriptionSeatsByUniqueTeamMembersInProject(
-        item.projectId!,
-      );
+      await this.syncSubscriptionSeatsAfterMembershipChange(item.projectId!);
 
       /*
        * Before the notification settings go: the "removed from on-call
@@ -1269,6 +1275,27 @@ export class TeamMemberService extends DatabaseService<TeamMember> {
     });
   }
 
+  /**
+   * The membership write has already committed. A payment-provider outage
+   * must not report a failed invite or prevent removal cleanup. The scheduled
+   * seat reconciliation retries projects whose acknowledged count is stale.
+   */
+  @CaptureSpan()
+  private async syncSubscriptionSeatsAfterMembershipChange(
+    projectId: ObjectID,
+  ): Promise<void> {
+    try {
+      await this.updateSubscriptionSeatsByUniqueTeamMembersInProject(projectId);
+    } catch (err) {
+      logger.error(
+        err as Error,
+        {
+          projectId: projectId.toString(),
+        } as LogAttributes,
+      );
+    }
+  }
+
   @CaptureSpan()
   public async updateSubscriptionSeatsByUniqueTeamMembersInProject(
     projectId: ObjectID,
@@ -1277,47 +1304,80 @@ export class TeamMemberService extends DatabaseService<TeamMember> {
       return;
     }
 
-    const numberOfMembers: number =
-      await this.getUniqueTeamMemberCountInProject(projectId);
-    const project: Project | null = await ProjectService.findOneById({
-      id: projectId,
-      select: {
-        paymentProviderSubscriptionId: true,
-        paymentProviderPlanId: true,
-      },
-      props: {
-        isRoot: true,
-      },
+    /*
+     * Serialize the request and worker paths, then read the latest count so
+     * an older synchronization cannot overwrite a newer seat quantity.
+     */
+    const mutex: SemaphoreMutex = await Semaphore.lock({
+      key: projectId.toString(),
+      namespace: "team-member-subscription-seats",
+      lockTimeout: 60000,
+      acquireTimeout: 5000,
     });
 
-    if (
-      project &&
-      project.paymentProviderSubscriptionId &&
-      project?.paymentProviderPlanId
-    ) {
+    try {
+      const project: Project | null = await ProjectService.findOneById({
+        id: projectId,
+        select: {
+          paymentProviderSubscriptionId: true,
+          paymentProviderPlanId: true,
+          paymentProviderSubscriptionSeats: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (
+        !project?.paymentProviderSubscriptionId ||
+        !project.paymentProviderPlanId
+      ) {
+        return;
+      }
+
       const plan: SubscriptionPlan | undefined =
-        SubscriptionPlan.getSubscriptionPlanById(
-          project?.paymentProviderPlanId,
-        );
+        SubscriptionPlan.getSubscriptionPlanById(project.paymentProviderPlanId);
 
       if (!plan) {
         return;
       }
 
+      const numberOfMembers: number =
+        await this.getUniqueTeamMemberCountInProject(projectId);
+
+      if (project.paymentProviderSubscriptionSeats === numberOfMembers) {
+        return;
+      }
+
+      /*
+       * The provider may apply the quantity and then time out. Persist that
+       * uncertainty before calling it, so even a later return to the previous
+       * member count cannot make reconciliation mistake the seats for synced.
+       */
+      const invalidated: number = await ProjectService.updateSubscriptionSeats({
+        projectId: projectId,
+        subscriptionId: project.paymentProviderSubscriptionId,
+        planId: project.paymentProviderPlanId,
+        seats: null,
+      });
+
+      if (invalidated === 0) {
+        return;
+      }
+
       await BillingService.changeQuantity(
         project.paymentProviderSubscriptionId,
         numberOfMembers,
       );
 
-      await ProjectService.updateOneById({
-        id: projectId,
-        data: {
-          paymentProviderSubscriptionSeats: numberOfMembers,
-        },
-        props: {
-          isRoot: true,
-        },
+      await ProjectService.updateSubscriptionSeats({
+        projectId: projectId,
+        subscriptionId: project.paymentProviderSubscriptionId,
+        planId: project.paymentProviderPlanId,
+        seats: numberOfMembers,
       });
+    } finally {
+      await Semaphore.release(mutex);
     }
   }
 

package/Server/Services/TelemetryIngestionKeyService.ts

@@ -4,6 +4,8 @@ import Query from "../Types/Database/Query";
 import UpdateBy from "../Types/Database/UpdateBy";
 import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
 import DatabaseService from "./DatabaseService";
+import PayAsYouGoBillingService from "./PayAsYouGoBillingService";
+import { IsBillingEnabled } from "../EnvironmentConfig";
 import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import Model from "../../Models/DatabaseModels/TelemetryIngestionKey";
@@ -189,6 +191,19 @@ export class Service extends DatabaseService<Model> {
       data["expiresAt"] = this.validateExpiresAt(data["expiresAt"]);
     }
 
+    if (IsBillingEnabled) {
+      const projectId: ObjectID | undefined =
+        createBy.props.tenantId || createBy.data.projectId;
+
+      if (!projectId) {
+        throw new BadDataException(
+          "ProjectId required to create a telemetry ingestion key.",
+        );
+      }
+
+      await PayAsYouGoBillingService.requirePayAsYouGo(projectId);
+    }
+
     return { createBy, carryForward: null };
   }
 
@@ -252,9 +267,8 @@ export class Service extends DatabaseService<Model> {
       /*
        * Only an EMPTY new list needs to know what kind of keys are being
        * updated, so the extra read is paid for only then. A non-empty list is
-       * valid on both key types, and the overwhelmingly common update (rename,
-       * toggle isEnabled, set an expiry) does not touch allowedOrigins at all
-       * and therefore reads nothing extra.
+       * valid on both key types. Renaming, disabling, or setting an expiry
+       * does not need this lookup; re-enabling has a separate billing check.
        */
       if (allowedOrigins.length === 0) {
         await this.assertNoBrowserKeyIsBeingUpdated(updateBy);
@@ -279,6 +293,33 @@ export class Service extends DatabaseService<Model> {
       data["expiresAt"] = this.validateExpiresAt(data["expiresAt"]);
     }
 
+    if (IsBillingEnabled && data["isEnabled"] === true) {
+      const keys: Array<Model> = await this.findBy({
+        query:
+          !updateBy.props.isRoot && updateBy.props.tenantId
+            ? { ...updateBy.query, projectId: updateBy.props.tenantId }
+            : updateBy.query,
+        select: { projectId: true },
+        limit: updateBy.limit,
+        skip: updateBy.skip,
+        props: { isRoot: true, ignoreHooks: true },
+      });
+      const checkedProjects: Set<string> = new Set<string>();
+
+      for (const key of keys) {
+        if (!key.projectId) {
+          throw new BadDataException(
+            "ProjectId required to enable a telemetry ingestion key.",
+          );
+        }
+
+        if (!checkedProjects.has(key.projectId.toString())) {
+          await PayAsYouGoBillingService.requirePayAsYouGo(key.projectId);
+          checkedProjects.add(key.projectId.toString());
+        }
+      }
+    }
+
     /*
      * Same reasoning as onBeforeDelete. The cached policy now carries the kill
      * switch, the expiry, the origin allowlist and the rate limit, so almost
@@ -296,9 +337,9 @@ export class Service extends DatabaseService<Model> {
    * Postgres. Returns null for unknown or malformed tokens (also cached, for a
    * shorter TTL).
    *
-   * A returned policy is NOT a decision: a disabled or expired key still
-   * resolves. Enforcement belongs to the caller, which knows which ingest
-   * surface it is and can therefore say *why* a request was refused.
+   * A disabled or expired key still resolves so the caller can explain that
+   * refusal. An otherwise usable key requires payment setup here, so every
+   * transport and validation endpoint shares the same billing admission rule.
    */
   @CaptureSpan()
   public async getPolicyFromSecretKey(
@@ -316,7 +357,9 @@ export class Service extends DatabaseService<Model> {
     const cached: CachedIngestionKeyPolicy | null | undefined =
       this.policyCache.get(cacheKey);
     if (cached !== undefined) {
-      return cached === null ? null : this.hydratePolicy(cached);
+      return cached === null
+        ? null
+        : this.requireBillingForPolicy(this.hydratePolicy(cached));
     }
 
     let secretKeyObjectId: ObjectID;
@@ -358,14 +401,41 @@ export class Service extends DatabaseService<Model> {
     );
 
     this.policyCache.set(cacheKey, snapshot, POSITIVE_TTL_MS);
-    return this.hydratePolicy(snapshot);
+    return this.requireBillingForPolicy(this.hydratePolicy(snapshot));
+  }
+
+  private async requireBillingForPolicy(
+    policy: TelemetryIngestionKeyPolicy,
+  ): Promise<TelemetryIngestionKeyPolicy> {
+    if (
+      policy.isEnabled &&
+      (!policy.expiresAt || policy.expiresAt.getTime() > Date.now())
+    ) {
+      /*
+       * Check even on key-cache hits. Billing eligibility has its own bounded
+       * cache, and must not remain enabled for the lifetime of a live key.
+       *
+       * This runs for every ingested batch, so it is the one caller that may
+       * answer from a recently cached denial rather than re-reading the
+       * payment provider. Without that, an unpaid project ingesting telemetry
+       * turns every batch into provider reads and holds the whole Stripe
+       * account at its rate limit. The client retries, so a few seconds
+       * between adding a card and ingest being admitted costs nothing - and
+       * BillingPaymentMethodService clears the entry as soon as the card is
+       * read, which is what the billing page does right after one is added.
+       */
+      await PayAsYouGoBillingService.requirePayAsYouGo(policy.projectId, {
+        allowStaleDenial: true,
+      });
+    }
+
+    return policy;
   }
 
   /**
    * Resolve an ingestion token to its projectId.
    *
-   * Unchanged in signature and in meaning: it is a LOOKUP, and it answers
-   * "which project owns this token" for any token that exists. It deliberately
+   * Shares billing admission with getPolicyFromSecretKey. It deliberately
    * does NOT enforce the kill switch or the expiry.
    *
    * That restraint is the point. Several callers (gRPC, MQTT, session replay,

package/Server/Services/TelemetryUsageBillingService.ts

@@ -30,6 +30,7 @@ import DockerHostService from "./DockerHostService";
 import PodmanHostService from "./PodmanHostService";
 import KubernetesClusterService from "./KubernetesClusterService";
 import ProxmoxClusterService from "./ProxmoxClusterService";
+import VMwareVCenterService from "./VMwareVCenterService";
 import CephClusterService from "./CephClusterService";
 import IoTFleetService from "./IoTFleetService";
 import Host from "../../Models/DatabaseModels/Host";
@@ -37,6 +38,7 @@ import DockerHost from "../../Models/DatabaseModels/DockerHost";
 import PodmanHost from "../../Models/DatabaseModels/PodmanHost";
 import KubernetesCluster from "../../Models/DatabaseModels/KubernetesCluster";
 import ProxmoxCluster from "../../Models/DatabaseModels/ProxmoxCluster";
+import VMwareVCenter from "../../Models/DatabaseModels/VMwareVCenter";
 import IoTFleet from "../../Models/DatabaseModels/IoTFleet";
 import CephCluster from "../../Models/DatabaseModels/CephCluster";
 import ServiceType from "../../Types/Telemetry/ServiceType";
@@ -52,6 +54,10 @@ import {
   IsBillingEnabled,
 } from "../EnvironmentConfig";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import QueryHelper from "../Types/Database/QueryHelper";
+import PayAsYouGoBillingService, {
+  LiveUsageAuthorization,
+} from "./PayAsYouGoBillingService";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
@@ -71,6 +77,7 @@ export class Service extends DatabaseService<Model> {
         projectId: data.projectId,
         productType: data.productType,
         isReportedToBillingProvider: false,
+        totalCostInUSD: QueryHelper.greaterThan(0),
       },
       skip: 0,
       limit: LIMIT_MAX, /// because a project can have MANY telemetry services.
@@ -84,20 +91,77 @@ export class Service extends DatabaseService<Model> {
     });
   }
 
+  public async waiveUnreportedUsageBilling(data: {
+    projectId: ObjectID;
+    productType: ProductType;
+    before?: Date;
+  }): Promise<void> {
+    let updated: number;
+    do {
+      updated = await this.updateBy({
+        query: {
+          projectId: data.projectId,
+          productType: data.productType,
+          isReportedToBillingProvider: false,
+          totalCostInUSD: QueryHelper.greaterThan(0),
+          ...(data.before
+            ? { createdAt: QueryHelper.lessThan(data.before) }
+            : {}),
+        },
+        data: { totalCostInUSD: new Decimal(0) },
+        skip: 0,
+        limit: LIMIT_MAX,
+        props: { isRoot: true, ignoreHooks: true },
+      });
+    } while (updated === LIMIT_MAX);
+  }
+
   @CaptureSpan()
   public async stageTelemetryUsageForProject(data: {
     projectId: ObjectID;
     productType: ProductType;
     usageDate?: Date;
+    liveAuthorization?: LiveUsageAuthorization | undefined;
   }): Promise<void> {
     if (!IsBillingEnabled) {
       return;
     }
 
+    /*
+     * A report that has just authorized this project live passes that answer
+     * down instead of having it read again. Anything else - a direct call, a
+     * token made for another project, one that has aged out - checks live,
+     * exactly as every staging call did before.
+     */
+    if (
+      !PayAsYouGoBillingService.isLiveAuthorizationFor(
+        data.liveAuthorization,
+        data.projectId,
+      ) &&
+      !(await PayAsYouGoBillingService.canUsePayAsYouGo(data.projectId, {
+        useCache: false,
+      }))
+    ) {
+      await this.waiveUnreportedUsageBilling(data);
+      return;
+    }
+
     const usageDate: Date = data.usageDate
       ? OneUptimeDate.fromString(data.usageDate)
       : OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), -1);
 
+    const billingStartsAt: Date | undefined =
+      await PayAsYouGoBillingService.getTelemetryBillingStartDate(
+        data.projectId,
+      );
+    if (
+      billingStartsAt &&
+      OneUptimeDate.getStartOfDay(usageDate, "UTC").getTime() <
+        billingStartsAt.getTime()
+    ) {
+      return;
+    }
+
     const averageRowSizeInBytes: number = this.getAverageRowSizeForProduct(
       data.productType,
     );
@@ -138,9 +202,9 @@ export class Service extends DatabaseService<Model> {
       return;
     }
 
-    const usageDayString: string = OneUptimeDate.getDateString(usageDate);
-    const startOfDay: Date = OneUptimeDate.getStartOfDay(usageDate);
-    const endOfDay: Date = OneUptimeDate.getEndOfDay(usageDate);
+    const usageDayString: string = this.getUsageDayString(usageDate);
+    const startOfDay: Date = OneUptimeDate.getStartOfDay(usageDate, "UTC");
+    const endOfDay: Date = OneUptimeDate.getEndOfDay(usageDate, "UTC");
 
     /*
      * Enumerate usage from ClickHouse by (primaryEntityId, primaryEntityType) in a
@@ -390,7 +454,8 @@ export class Service extends DatabaseService<Model> {
   /*
    * Map of resourceId -> retainTelemetryDataForDays for every resource in
    * the project that can own telemetry (Service, Host, DockerHost,
-   * KubernetesCluster, ProxmoxCluster, CephCluster). Used to scale billed
+   * KubernetesCluster, ProxmoxCluster, VMwareVCenter, CephCluster). Used to
+   * scale billed
    * cost by the actual retention applied to each resource's telemetry.
    * Resources without an override (and the unattributed bucket) fall back
    * to the project default.
@@ -499,6 +564,23 @@ export class Service extends DatabaseService<Model> {
       }
     }
 
+    const vmwareVCenters: Array<VMwareVCenter> =
+      await VMwareVCenterService.findBy({
+        query: { projectId: projectId },
+        select: { _id: true, retainTelemetryDataForDays: true },
+        skip: 0,
+        limit: LIMIT_MAX,
+        props: { isRoot: true },
+      });
+    for (const vmwareVCenter of vmwareVCenters) {
+      if (vmwareVCenter.id && vmwareVCenter.retainTelemetryDataForDays) {
+        retentionByServiceId.set(
+          vmwareVCenter.id.toString(),
+          vmwareVCenter.retainTelemetryDataForDays,
+        );
+      }
+    }
+
     const cephClusters: Array<CephCluster> = await CephClusterService.findBy({
       query: { projectId: projectId },
       select: { _id: true, retainTelemetryDataForDays: true },
@@ -580,7 +662,7 @@ export class Service extends DatabaseService<Model> {
       ? OneUptimeDate.fromString(data.usageDate)
       : OneUptimeDate.getCurrentDate();
 
-    const usageDayString: string = OneUptimeDate.getDateString(usageDate);
+    const usageDayString: string = this.getUsageDayString(usageDate);
 
     const totalCostOfThisOperationInUSD: number =
       serverMeteredPlan.getTotalCostInUSD({
@@ -662,6 +744,14 @@ export class Service extends DatabaseService<Model> {
     }
   }
 
+  private getUsageDayString(usageDate: Date): string {
+    return OneUptimeDate.getDateAsCustomFormattedStringInTimezone({
+      date: usageDate,
+      timezone: "UTC",
+      format: "MMM DD, YYYY",
+    });
+  }
+
   private getAverageRowSizeForProduct(productType: ProductType): number {
     const fallbackSize: number = 1024;