@oneuptime/common 12.0.34 → 13.0.1

package/Server/Services/TeamMemberService.ts

@@ -1,5 +1,6 @@
 import DatabaseConfig from "../DatabaseConfig";
 import { IsBillingEnabled } from "../EnvironmentConfig";
+import Semaphore, { SemaphoreMutex } from "../Infrastructure/Semaphore";
 import CreateBy from "../Types/Database/CreateBy";
 import DeleteBy from "../Types/Database/DeleteBy";
 import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
@@ -174,18 +175,26 @@ export class TeamMemberService extends DatabaseService<TeamMember> {
         id: createBy.data.projectId!,
         select: {
           seatLimit: true,
-          paymentProviderSubscriptionSeats: true,
         },
         props: {
           isRoot: true,
         },
       });
 
+      /*
+       * Billing can lag after a provider outage. Admission limits must use
+       * persisted memberships, including pending invitations.
+       */
+      const numberOfMembers: number =
+        project &&
+        (project.seatLimit || createBy.props.currentPlan === PlanType.Free)
+          ? await this.getUniqueTeamMemberCountInProject(projectId)
+          : 0;
+
       if (
         project &&
         project.seatLimit &&
-        project.paymentProviderSubscriptionSeats &&
-        project.paymentProviderSubscriptionSeats >= project.seatLimit
+        numberOfMembers >= project.seatLimit
       ) {
         throw new BadDataException(Errors.TeamMemberService.LIMIT_REACHED);
       }
@@ -193,8 +202,7 @@ export class TeamMemberService extends DatabaseService<TeamMember> {
       if (
         createBy.props.currentPlan === PlanType.Free &&
         project &&
-        project.paymentProviderSubscriptionSeats &&
-        project.paymentProviderSubscriptionSeats >= 1
+        numberOfMembers >= 1
       ) {
         throw new BadDataException(
           Errors.TeamMemberService.LIMIT_REACHED_FOR_FREE_PLAN,
@@ -535,7 +543,7 @@ export class TeamMemberService extends DatabaseService<TeamMember> {
       onCreate.createBy.data.projectId!,
     );
 
-    await this.updateSubscriptionSeatsByUniqueTeamMembersInProject(
+    await this.syncSubscriptionSeatsAfterMembershipChange(
       onCreate.createBy.data.projectId!,
     );
 
@@ -726,9 +734,7 @@ export class TeamMemberService extends DatabaseService<TeamMember> {
 
     for (const item of onDelete.carryForward as Array<TeamMember>) {
       await this.refreshTokens(item.userId!, item.projectId!);
-      await this.updateSubscriptionSeatsByUniqueTeamMembersInProject(
-        item.projectId!,
-      );
+      await this.syncSubscriptionSeatsAfterMembershipChange(item.projectId!);
 
       /*
        * Before the notification settings go: the "removed from on-call
@@ -1269,6 +1275,27 @@ export class TeamMemberService extends DatabaseService<TeamMember> {
     });
   }
 
+  /**
+   * The membership write has already committed. A payment-provider outage
+   * must not report a failed invite or prevent removal cleanup. The scheduled
+   * seat reconciliation retries projects whose acknowledged count is stale.
+   */
+  @CaptureSpan()
+  private async syncSubscriptionSeatsAfterMembershipChange(
+    projectId: ObjectID,
+  ): Promise<void> {
+    try {
+      await this.updateSubscriptionSeatsByUniqueTeamMembersInProject(projectId);
+    } catch (err) {
+      logger.error(
+        err as Error,
+        {
+          projectId: projectId.toString(),
+        } as LogAttributes,
+      );
+    }
+  }
+
   @CaptureSpan()
   public async updateSubscriptionSeatsByUniqueTeamMembersInProject(
     projectId: ObjectID,
@@ -1277,47 +1304,80 @@ export class TeamMemberService extends DatabaseService<TeamMember> {
       return;
     }
 
-    const numberOfMembers: number =
-      await this.getUniqueTeamMemberCountInProject(projectId);
-    const project: Project | null = await ProjectService.findOneById({
-      id: projectId,
-      select: {
-        paymentProviderSubscriptionId: true,
-        paymentProviderPlanId: true,
-      },
-      props: {
-        isRoot: true,
-      },
+    /*
+     * Serialize the request and worker paths, then read the latest count so
+     * an older synchronization cannot overwrite a newer seat quantity.
+     */
+    const mutex: SemaphoreMutex = await Semaphore.lock({
+      key: projectId.toString(),
+      namespace: "team-member-subscription-seats",
+      lockTimeout: 60000,
+      acquireTimeout: 5000,
     });
 
-    if (
-      project &&
-      project.paymentProviderSubscriptionId &&
-      project?.paymentProviderPlanId
-    ) {
+    try {
+      const project: Project | null = await ProjectService.findOneById({
+        id: projectId,
+        select: {
+          paymentProviderSubscriptionId: true,
+          paymentProviderPlanId: true,
+          paymentProviderSubscriptionSeats: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (
+        !project?.paymentProviderSubscriptionId ||
+        !project.paymentProviderPlanId
+      ) {
+        return;
+      }
+
       const plan: SubscriptionPlan | undefined =
-        SubscriptionPlan.getSubscriptionPlanById(
-          project?.paymentProviderPlanId,
-        );
+        SubscriptionPlan.getSubscriptionPlanById(project.paymentProviderPlanId);
 
       if (!plan) {
         return;
       }
 
+      const numberOfMembers: number =
+        await this.getUniqueTeamMemberCountInProject(projectId);
+
+      if (project.paymentProviderSubscriptionSeats === numberOfMembers) {
+        return;
+      }
+
+      /*
+       * The provider may apply the quantity and then time out. Persist that
+       * uncertainty before calling it, so even a later return to the previous
+       * member count cannot make reconciliation mistake the seats for synced.
+       */
+      const invalidated: number = await ProjectService.updateSubscriptionSeats({
+        projectId: projectId,
+        subscriptionId: project.paymentProviderSubscriptionId,
+        planId: project.paymentProviderPlanId,
+        seats: null,
+      });
+
+      if (invalidated === 0) {
+        return;
+      }
+
       await BillingService.changeQuantity(
         project.paymentProviderSubscriptionId,
         numberOfMembers,
       );
 
-      await ProjectService.updateOneById({
-        id: projectId,
-        data: {
-          paymentProviderSubscriptionSeats: numberOfMembers,
-        },
-        props: {
-          isRoot: true,
-        },
+      await ProjectService.updateSubscriptionSeats({
+        projectId: projectId,
+        subscriptionId: project.paymentProviderSubscriptionId,
+        planId: project.paymentProviderPlanId,
+        seats: numberOfMembers,
       });
+    } finally {
+      await Semaphore.release(mutex);
     }
   }
 

package/Server/Services/TelemetryIngestionKeyService.ts

@@ -4,6 +4,8 @@ import Query from "../Types/Database/Query";
 import UpdateBy from "../Types/Database/UpdateBy";
 import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
 import DatabaseService from "./DatabaseService";
+import PayAsYouGoBillingService from "./PayAsYouGoBillingService";
+import { IsBillingEnabled } from "../EnvironmentConfig";
 import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import Model from "../../Models/DatabaseModels/TelemetryIngestionKey";
@@ -189,6 +191,19 @@ export class Service extends DatabaseService<Model> {
       data["expiresAt"] = this.validateExpiresAt(data["expiresAt"]);
     }
 
+    if (IsBillingEnabled) {
+      const projectId: ObjectID | undefined =
+        createBy.props.tenantId || createBy.data.projectId;
+
+      if (!projectId) {
+        throw new BadDataException(
+          "ProjectId required to create a telemetry ingestion key.",
+        );
+      }
+
+      await PayAsYouGoBillingService.requirePayAsYouGo(projectId);
+    }
+
     return { createBy, carryForward: null };
   }
 
@@ -252,9 +267,8 @@ export class Service extends DatabaseService<Model> {
       /*
        * Only an EMPTY new list needs to know what kind of keys are being
        * updated, so the extra read is paid for only then. A non-empty list is
-       * valid on both key types, and the overwhelmingly common update (rename,
-       * toggle isEnabled, set an expiry) does not touch allowedOrigins at all
-       * and therefore reads nothing extra.
+       * valid on both key types. Renaming, disabling, or setting an expiry
+       * does not need this lookup; re-enabling has a separate billing check.
        */
       if (allowedOrigins.length === 0) {
         await this.assertNoBrowserKeyIsBeingUpdated(updateBy);
@@ -279,6 +293,33 @@ export class Service extends DatabaseService<Model> {
       data["expiresAt"] = this.validateExpiresAt(data["expiresAt"]);
     }
 
+    if (IsBillingEnabled && data["isEnabled"] === true) {
+      const keys: Array<Model> = await this.findBy({
+        query:
+          !updateBy.props.isRoot && updateBy.props.tenantId
+            ? { ...updateBy.query, projectId: updateBy.props.tenantId }
+            : updateBy.query,
+        select: { projectId: true },
+        limit: updateBy.limit,
+        skip: updateBy.skip,
+        props: { isRoot: true, ignoreHooks: true },
+      });
+      const checkedProjects: Set<string> = new Set<string>();
+
+      for (const key of keys) {
+        if (!key.projectId) {
+          throw new BadDataException(
+            "ProjectId required to enable a telemetry ingestion key.",
+          );
+        }
+
+        if (!checkedProjects.has(key.projectId.toString())) {
+          await PayAsYouGoBillingService.requirePayAsYouGo(key.projectId);
+          checkedProjects.add(key.projectId.toString());
+        }
+      }
+    }
+
     /*
      * Same reasoning as onBeforeDelete. The cached policy now carries the kill
      * switch, the expiry, the origin allowlist and the rate limit, so almost
@@ -296,9 +337,9 @@ export class Service extends DatabaseService<Model> {
    * Postgres. Returns null for unknown or malformed tokens (also cached, for a
    * shorter TTL).
    *
-   * A returned policy is NOT a decision: a disabled or expired key still
-   * resolves. Enforcement belongs to the caller, which knows which ingest
-   * surface it is and can therefore say *why* a request was refused.
+   * A disabled or expired key still resolves so the caller can explain that
+   * refusal. An otherwise usable key requires payment setup here, so every
+   * transport and validation endpoint shares the same billing admission rule.
    */
   @CaptureSpan()
   public async getPolicyFromSecretKey(
@@ -316,7 +357,9 @@ export class Service extends DatabaseService<Model> {
     const cached: CachedIngestionKeyPolicy | null | undefined =
       this.policyCache.get(cacheKey);
     if (cached !== undefined) {
-      return cached === null ? null : this.hydratePolicy(cached);
+      return cached === null
+        ? null
+        : this.requireBillingForPolicy(this.hydratePolicy(cached));
     }
 
     let secretKeyObjectId: ObjectID;
@@ -358,14 +401,41 @@ export class Service extends DatabaseService<Model> {
     );
 
     this.policyCache.set(cacheKey, snapshot, POSITIVE_TTL_MS);
-    return this.hydratePolicy(snapshot);
+    return this.requireBillingForPolicy(this.hydratePolicy(snapshot));
+  }
+
+  private async requireBillingForPolicy(
+    policy: TelemetryIngestionKeyPolicy,
+  ): Promise<TelemetryIngestionKeyPolicy> {
+    if (
+      policy.isEnabled &&
+      (!policy.expiresAt || policy.expiresAt.getTime() > Date.now())
+    ) {
+      /*
+       * Check even on key-cache hits. Billing eligibility has its own bounded
+       * cache, and must not remain enabled for the lifetime of a live key.
+       *
+       * This runs for every ingested batch, so it is the one caller that may
+       * answer from a recently cached denial rather than re-reading the
+       * payment provider. Without that, an unpaid project ingesting telemetry
+       * turns every batch into provider reads and holds the whole Stripe
+       * account at its rate limit. The client retries, so a few seconds
+       * between adding a card and ingest being admitted costs nothing - and
+       * BillingPaymentMethodService clears the entry as soon as the card is
+       * read, which is what the billing page does right after one is added.
+       */
+      await PayAsYouGoBillingService.requirePayAsYouGo(policy.projectId, {
+        allowStaleDenial: true,
+      });
+    }
+
+    return policy;
   }
 
   /**
    * Resolve an ingestion token to its projectId.
    *
-   * Unchanged in signature and in meaning: it is a LOOKUP, and it answers
-   * "which project owns this token" for any token that exists. It deliberately
+   * Shares billing admission with getPolicyFromSecretKey. It deliberately
    * does NOT enforce the kill switch or the expiry.
    *
    * That restraint is the point. Several callers (gRPC, MQTT, session replay,

package/Server/Services/TelemetryUsageBillingService.ts

@@ -30,6 +30,7 @@ import DockerHostService from "./DockerHostService";
 import PodmanHostService from "./PodmanHostService";
 import KubernetesClusterService from "./KubernetesClusterService";
 import ProxmoxClusterService from "./ProxmoxClusterService";
+import VMwareVCenterService from "./VMwareVCenterService";
 import CephClusterService from "./CephClusterService";
 import IoTFleetService from "./IoTFleetService";
 import Host from "../../Models/DatabaseModels/Host";
@@ -37,6 +38,7 @@ import DockerHost from "../../Models/DatabaseModels/DockerHost";
 import PodmanHost from "../../Models/DatabaseModels/PodmanHost";
 import KubernetesCluster from "../../Models/DatabaseModels/KubernetesCluster";
 import ProxmoxCluster from "../../Models/DatabaseModels/ProxmoxCluster";
+import VMwareVCenter from "../../Models/DatabaseModels/VMwareVCenter";
 import IoTFleet from "../../Models/DatabaseModels/IoTFleet";
 import CephCluster from "../../Models/DatabaseModels/CephCluster";
 import ServiceType from "../../Types/Telemetry/ServiceType";
@@ -52,6 +54,10 @@ import {
   IsBillingEnabled,
 } from "../EnvironmentConfig";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import QueryHelper from "../Types/Database/QueryHelper";
+import PayAsYouGoBillingService, {
+  LiveUsageAuthorization,
+} from "./PayAsYouGoBillingService";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
@@ -71,6 +77,7 @@ export class Service extends DatabaseService<Model> {
         projectId: data.projectId,
         productType: data.productType,
         isReportedToBillingProvider: false,
+        totalCostInUSD: QueryHelper.greaterThan(0),
       },
       skip: 0,
       limit: LIMIT_MAX, /// because a project can have MANY telemetry services.
@@ -84,20 +91,77 @@ export class Service extends DatabaseService<Model> {
     });
   }
 
+  public async waiveUnreportedUsageBilling(data: {
+    projectId: ObjectID;
+    productType: ProductType;
+    before?: Date;
+  }): Promise<void> {
+    let updated: number;
+    do {
+      updated = await this.updateBy({
+        query: {
+          projectId: data.projectId,
+          productType: data.productType,
+          isReportedToBillingProvider: false,
+          totalCostInUSD: QueryHelper.greaterThan(0),
+          ...(data.before
+            ? { createdAt: QueryHelper.lessThan(data.before) }
+            : {}),
+        },
+        data: { totalCostInUSD: new Decimal(0) },
+        skip: 0,
+        limit: LIMIT_MAX,
+        props: { isRoot: true, ignoreHooks: true },
+      });
+    } while (updated === LIMIT_MAX);
+  }
+
   @CaptureSpan()
   public async stageTelemetryUsageForProject(data: {
     projectId: ObjectID;
     productType: ProductType;
     usageDate?: Date;
+    liveAuthorization?: LiveUsageAuthorization | undefined;
   }): Promise<void> {
     if (!IsBillingEnabled) {
       return;
     }
 
+    /*
+     * A report that has just authorized this project live passes that answer
+     * down instead of having it read again. Anything else - a direct call, a
+     * token made for another project, one that has aged out - checks live,
+     * exactly as every staging call did before.
+     */
+    if (
+      !PayAsYouGoBillingService.isLiveAuthorizationFor(
+        data.liveAuthorization,
+        data.projectId,
+      ) &&
+      !(await PayAsYouGoBillingService.canUsePayAsYouGo(data.projectId, {
+        useCache: false,
+      }))
+    ) {
+      await this.waiveUnreportedUsageBilling(data);
+      return;
+    }
+
     const usageDate: Date = data.usageDate
       ? OneUptimeDate.fromString(data.usageDate)
       : OneUptimeDate.addRemoveDays(OneUptimeDate.getCurrentDate(), -1);
 
+    const billingStartsAt: Date | undefined =
+      await PayAsYouGoBillingService.getTelemetryBillingStartDate(
+        data.projectId,
+      );
+    if (
+      billingStartsAt &&
+      OneUptimeDate.getStartOfDay(usageDate, "UTC").getTime() <
+        billingStartsAt.getTime()
+    ) {
+      return;
+    }
+
     const averageRowSizeInBytes: number = this.getAverageRowSizeForProduct(
       data.productType,
     );
@@ -138,9 +202,9 @@ export class Service extends DatabaseService<Model> {
       return;
     }
 
-    const usageDayString: string = OneUptimeDate.getDateString(usageDate);
-    const startOfDay: Date = OneUptimeDate.getStartOfDay(usageDate);
-    const endOfDay: Date = OneUptimeDate.getEndOfDay(usageDate);
+    const usageDayString: string = this.getUsageDayString(usageDate);
+    const startOfDay: Date = OneUptimeDate.getStartOfDay(usageDate, "UTC");
+    const endOfDay: Date = OneUptimeDate.getEndOfDay(usageDate, "UTC");
 
     /*
      * Enumerate usage from ClickHouse by (primaryEntityId, primaryEntityType) in a
@@ -390,7 +454,8 @@ export class Service extends DatabaseService<Model> {
   /*
    * Map of resourceId -> retainTelemetryDataForDays for every resource in
    * the project that can own telemetry (Service, Host, DockerHost,
-   * KubernetesCluster, ProxmoxCluster, CephCluster). Used to scale billed
+   * KubernetesCluster, ProxmoxCluster, VMwareVCenter, CephCluster). Used to
+   * scale billed
    * cost by the actual retention applied to each resource's telemetry.
    * Resources without an override (and the unattributed bucket) fall back
    * to the project default.
@@ -499,6 +564,23 @@ export class Service extends DatabaseService<Model> {
       }
     }
 
+    const vmwareVCenters: Array<VMwareVCenter> =
+      await VMwareVCenterService.findBy({
+        query: { projectId: projectId },
+        select: { _id: true, retainTelemetryDataForDays: true },
+        skip: 0,
+        limit: LIMIT_MAX,
+        props: { isRoot: true },
+      });
+    for (const vmwareVCenter of vmwareVCenters) {
+      if (vmwareVCenter.id && vmwareVCenter.retainTelemetryDataForDays) {
+        retentionByServiceId.set(
+          vmwareVCenter.id.toString(),
+          vmwareVCenter.retainTelemetryDataForDays,
+        );
+      }
+    }
+
     const cephClusters: Array<CephCluster> = await CephClusterService.findBy({
       query: { projectId: projectId },
       select: { _id: true, retainTelemetryDataForDays: true },
@@ -580,7 +662,7 @@ export class Service extends DatabaseService<Model> {
       ? OneUptimeDate.fromString(data.usageDate)
       : OneUptimeDate.getCurrentDate();
 
-    const usageDayString: string = OneUptimeDate.getDateString(usageDate);
+    const usageDayString: string = this.getUsageDayString(usageDate);
 
     const totalCostOfThisOperationInUSD: number =
       serverMeteredPlan.getTotalCostInUSD({
@@ -662,6 +744,14 @@ export class Service extends DatabaseService<Model> {
     }
   }
 
+  private getUsageDayString(usageDate: Date): string {
+    return OneUptimeDate.getDateAsCustomFormattedStringInTimezone({
+      date: usageDate,
+      timezone: "UTC",
+      format: "MMM DD, YYYY",
+    });
+  }
+
   private getAverageRowSizeForProduct(productType: ProductType): number {
     const fallbackSize: number = 1024;
 

package/Server/Services/TraceAggregationService.ts

@@ -244,6 +244,7 @@ export class TraceAggregationService {
       ["podmanHostId", ServiceType.PodmanHost],
       ["kubernetesClusterId", ServiceType.KubernetesCluster],
       ["proxmoxClusterId", ServiceType.ProxmoxCluster],
+      ["vmwareVCenterId", ServiceType.VMwareVCenter],
       ["cephClusterId", ServiceType.CephCluster],
       ["serverlessFunctionId", ServiceType.ServerlessFunction],
       ["cloudResourceId", ServiceType.CloudResource],
@@ -1395,10 +1396,10 @@ export class TraceAggregationService {
     /*
      * Both memberships are required, and the two sets genuinely differ:
      *
-     * - `proxmoxClusterId` / `cephClusterId` are resource dimensions here
-     *   but have no branch in ResourceFacetResolver.resolveOne, which
-     *   returns [] for them. Asking anyway would cost a round-trip and
-     *   still leave the cell raw.
+     * - `proxmoxClusterId` / `vmwareVCenterId` / `cephClusterId` are
+     *   resource dimensions here but have no branch in
+     *   ResourceFacetResolver.resolveOne, which returns [] for them. Asking
+     *   anyway would cost a round-trip and still leave the cell raw.
      * - `primaryEntityId` / `serviceId` are resolvable but are NOT resource
      *   dimensions here — they select the OpenTelemetry-service slot, and
      *   the traces explorer already resolves that split client-side