@oneuptime/common 12.0.1 → 12.0.3

package/Server/Services/ProjectService.ts

@@ -606,6 +606,7 @@ export class ProjectService extends DatabaseService<Model> {
       subscriptionId: string;
       meteredSubscriptionId: string;
       trialEndsAt?: Date | undefined;
+      subscriptionIdsPendingCancellation: Array<string>;
     } = await BillingService.changePlan({
       projectId: project.id!,
       subscriptionId: project.paymentProviderSubscriptionId,
@@ -617,36 +618,51 @@ export class ProjectService extends DatabaseService<Model> {
       endTrialAt: endTrialAt,
     });
 
-    const subscriptionState: SubscriptionStatus =
-      await BillingService.getSubscriptionStatus(subscription.subscriptionId);
+    const subscriptionState: SubscriptionStatus | null =
+      await this.findSubscriptionStatus(subscription.subscriptionId);
 
-    const meteredSubscriptionState: SubscriptionStatus =
-      await BillingService.getSubscriptionStatus(
-        subscription.meteredSubscriptionId,
-      );
+    const meteredSubscriptionState: SubscriptionStatus | null =
+      await this.findSubscriptionStatus(subscription.meteredSubscriptionId);
+
+    const updateData: QueryDeepPartialEntity<Model> = {
+      paymentProviderPlanId: params.paymentProviderPlanId,
+      paymentProviderSubscriptionId: subscription.subscriptionId,
+      paymentProviderMeteredSubscriptionId: subscription.meteredSubscriptionId,
+      paymentProviderSubscriptionSeats: seats,
+      trialEndsAt: subscription.trialEndsAt || endTrialAt || new Date(),
+      planName: SubscriptionPlan.getPlanType(
+        params.paymentProviderPlanId,
+        getAllEnvVars(),
+      ),
+    };
+
+    if (meteredSubscriptionState) {
+      updateData.paymentProviderMeteredSubscriptionStatus =
+        meteredSubscriptionState;
+    }
+
+    if (subscriptionState) {
+      updateData.paymentProviderSubscriptionStatus = subscriptionState;
+    }
 
     await this.updateOneById({
       id: project.id!,
-      data: {
-        paymentProviderPlanId: params.paymentProviderPlanId,
-        paymentProviderSubscriptionId: subscription.subscriptionId,
-        paymentProviderMeteredSubscriptionId:
-          subscription.meteredSubscriptionId,
-        paymentProviderSubscriptionSeats: seats,
-        trialEndsAt: subscription.trialEndsAt || endTrialAt || new Date(),
-        planName: SubscriptionPlan.getPlanType(
-          params.paymentProviderPlanId,
-          getAllEnvVars(),
-        ),
-        paymentProviderMeteredSubscriptionStatus: meteredSubscriptionState,
-        paymentProviderSubscriptionStatus: subscriptionState,
-      },
+      data: updateData,
       props: {
         isRoot: true,
         ignoreHooks: true,
       },
     });
 
+    /*
+     * Raised after the row is written, because the row is what makes these ids
+     * unreachable: it now carries the replacements.
+     */
+    this.alertOnSubscriptionsPendingCancellation({
+      projectId: project.id!,
+      subscriptionIds: subscription.subscriptionIdsPendingCancellation,
+    });
+
     this.capturePlanChangeAnalytics({
       project: project,
       newPlan: plan,
@@ -657,6 +673,103 @@ export class ProjectService extends DatabaseService<Model> {
     await this.sendSubscriptionChangeWebhookSlackNotification(project.id!);
   }
 
+  /*
+   * Reads a subscription's status back, and gives up rather than throwing.
+   *
+   * This sits between a plan change returning and the project row being
+   * written, and that write is what records the subscription ids the plan
+   * change produced. A replacement already exists at the payment provider and
+   * is already billing by this point, and the row is the only thing in
+   * OneUptime that will ever point at it - so a rate limit or a dropped
+   * connection reading a status back must not be what stops it being written
+   * down. Worse than the missing status: reactivation is retried from
+   * BillingInvoiceService whenever the customer's invoices look settled, so a
+   * row left pointing at the old id mints another live subscription every
+   * time it runs.
+   *
+   * The status columns are refreshed daily by
+   * PaymentProvider:CheckSubscriptionStatus, so leaving one behind costs a
+   * day of staleness. Losing the id costs a subscription nobody can find.
+   */
+  private async findSubscriptionStatus(
+    subscriptionId: string,
+  ): Promise<SubscriptionStatus | null> {
+    try {
+      return await BillingService.getSubscriptionStatus(subscriptionId);
+    } catch (err) {
+      logger.error(err, { subscriptionId } as LogAttributes);
+      return null;
+    }
+  }
+
+  /*
+   * Raises the subscriptions a plan change replaced but could not cancel.
+   *
+   * The project row now carries the replacement ids, so these ones exist
+   * nowhere in OneUptime any more. They were not billing when the plan change
+   * read them and the cancel has already been retried, so this is a loose end
+   * rather than a double charge - but it is one nobody will find on their own,
+   * which is why the ids travel with the alert instead of only reaching a log
+   * line. The replacements also carry them in their payment-provider metadata,
+   * as replacedSubscriptionId.
+   *
+   * Called once the plan change has already been applied and written down, so
+   * nothing in here is allowed to throw, and nothing in here is allowed to
+   * hold the response open either: failing - or being slow - to raise a loose
+   * end must not turn a plan change that worked into one the caller is told
+   * failed, or one that never comes back.
+   */
+  private alertOnSubscriptionsPendingCancellation(data: {
+    projectId: ObjectID;
+    subscriptionIds: Array<string>;
+  }): void {
+    if (!data.subscriptionIds || data.subscriptionIds.length === 0) {
+      return;
+    }
+
+    logger.error(
+      `Could not cancel subscriptions replaced by a plan change for project ${data.projectId.toString()}. They are still open at the payment provider and have to be cancelled by hand: ${data.subscriptionIds.join(
+        ", ",
+      )}`,
+      {
+        projectId: data.projectId.toString(),
+      } as LogAttributes,
+    );
+
+    if (!NotificationSlackWebhookOnSubscriptionUpdate) {
+      return;
+    }
+
+    const slackMessage: string = `*Subscriptions could not be cancelled after a plan change:*
+*Project ID:* ${data.projectId.toString()}
+*Subscriptions still open at the payment provider:* ${data.subscriptionIds.join(", ")}
+These are no longer recorded against the project and have to be cancelled by hand.
+`;
+
+    /*
+     * Delivery is deliberately not awaited, and neither is reading the
+     * configured webhook. SlackUtil retries three times with exponential
+     * backoff and sets no HTTP timeout, so awaiting it would let a webhook
+     * that hangs hold the plan-change response open with it - for a message
+     * whose contents are already on the error log above. Same shape as
+     * sendSubscriptionChangeWebhookSlackNotification below.
+     */
+    try {
+      SlackUtil.sendMessageToChannelViaIncomingWebhook({
+        url: URL.fromString(NotificationSlackWebhookOnSubscriptionUpdate),
+        text: slackMessage,
+      }).catch((error: Error) => {
+        logger.error("Error sending slack message: " + error, {
+          projectId: data.projectId.toString(),
+        } as LogAttributes);
+      });
+    } catch (error) {
+      logger.error("Error sending slack message: " + error, {
+        projectId: data.projectId.toString(),
+      } as LogAttributes);
+    }
+  }
+
   /*
    * Pushes a project's trial end date out to a new date, in the payment
    * provider first and then on the project row.
@@ -2224,11 +2337,13 @@ export class ProjectService extends DatabaseService<Model> {
     }
 
     /*
-     * Reactivating goes through changePlan, which cancels both subscriptions
-     * and creates new ones - so a trial the project still has to run has to be
-     * carried onto the replacements explicitly. Otherwise a customer whose
-     * trial a master admin extended as a goodwill gesture loses the extension
-     * the moment their subscription is reactivated, and is billed right away.
+     * Reactivating goes through changePlan, and it is the one caller that
+     * genuinely lands on its recreate path: the subscriptions being reactivated
+     * are not live, so there is nothing to update and new ones are created. A
+     * trial the project still has to run therefore has to be carried onto the
+     * replacements explicitly. Otherwise a customer whose trial a master admin
+     * extended as a goodwill gesture loses the extension the moment their
+     * subscription is reactivated, and is billed right away.
      *
      * A trial that has already lapsed is not revived: reactivation is not the
      * place to hand out free service.
@@ -2242,6 +2357,7 @@ export class ProjectService extends DatabaseService<Model> {
       subscriptionId: string;
       meteredSubscriptionId: string;
       trialEndsAt?: Date | undefined;
+      subscriptionIdsPendingCancellation: Array<string>;
     } = await BillingService.changePlan({
       projectId: project.id as ObjectID,
       subscriptionId: project.paymentProviderSubscriptionId,
@@ -2254,25 +2370,34 @@ export class ProjectService extends DatabaseService<Model> {
     });
 
     // refresh subscription status.
-    const subscriptionState: SubscriptionStatus =
-      await BillingService.getSubscriptionStatus(
-        result.subscriptionId as string,
-      );
+    const subscriptionState: SubscriptionStatus | null =
+      await this.findSubscriptionStatus(result.subscriptionId as string);
 
-    const meteredSubscriptionState: SubscriptionStatus =
-      await BillingService.getSubscriptionStatus(
-        project.paymentProviderMeteredSubscriptionId as string,
-      );
+    /*
+     * The metered subscription changePlan returns, not the one the project row
+     * still points at. When changePlan replaced the subscriptions the old id
+     * names a cancelled subscription, and its status would be persisted against
+     * the new one.
+     */
+    const meteredSubscriptionState: SubscriptionStatus | null =
+      await this.findSubscriptionStatus(result.meteredSubscriptionId as string);
 
     // now update project with new subscription id.
 
     const updateData: QueryDeepPartialEntity<Model> = {
       paymentProviderSubscriptionId: result.subscriptionId,
       paymentProviderMeteredSubscriptionId: result.meteredSubscriptionId,
-      paymentProviderSubscriptionStatus: subscriptionState,
-      paymentProviderMeteredSubscriptionStatus: meteredSubscriptionState,
     };
 
+    if (subscriptionState) {
+      updateData.paymentProviderSubscriptionStatus = subscriptionState;
+    }
+
+    if (meteredSubscriptionState) {
+      updateData.paymentProviderMeteredSubscriptionStatus =
+        meteredSubscriptionState;
+    }
+
     /*
      * The payment provider is the source of truth for the trial the new
      * subscriptions were created with, so its date wins over the one sent.
@@ -2293,6 +2418,11 @@ export class ProjectService extends DatabaseService<Model> {
       },
     });
 
+    this.alertOnSubscriptionsPendingCancellation({
+      projectId: project.id!,
+      subscriptionIds: result.subscriptionIdsPendingCancellation,
+    });
+
     // send slack message on plan change.
     await this.sendSubscriptionChangeWebhookSlackNotification(projectId);
   }

package/Server/Services/ProxmoxClusterService.ts

@@ -5,6 +5,7 @@ import Model from "../../Models/DatabaseModels/ProxmoxCluster";
 import Label from "../../Models/DatabaseModels/Label";
 import { OnCreate } from "../Types/Database/Hooks";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
 import ObjectID from "../../Types/ObjectID";
 import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
@@ -177,7 +178,6 @@ export class Service extends DatabaseService<Model> {
       guestsWithoutBackupCount?: number | undefined;
     },
   ): Promise<void> {
-    const cacheKey: string = clusterId.toString();
     const extrasFingerprint: string = crypto
       .createHash("sha1")
       .update(
@@ -193,71 +193,53 @@ export class Service extends DatabaseService<Model> {
       )
       .digest("hex");
 
-    let cached: string | null = null;
-    try {
-      cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-    } catch {
-      /*
-       * Cache unavailable — fail open and refresh lastSeenAt anyway. A
-       * cache error must never skip the DB write below, otherwise the
-       * resource is wrongly marked "disconnected" while telemetry is
-       * still flowing. Mirrors shouldRunMaintenance's fail-open stance.
-       */
-      cached = null;
-    }
-
-    if (cached === extrasFingerprint) {
-      return; // same data was written recently
-    }
-
-    try {
-      await GlobalCache.setString(
-        LAST_SEEN_CACHE_NAMESPACE,
-        cacheKey,
-        extrasFingerprint,
-        { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
-      );
-    } catch {
-      // Best-effort throttle write; proceed with the DB update regardless.
-    }
-
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = {
+    const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
       otelCollectorStatus: "connected",
     };
 
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const metadata: any = {};
+
     if (extra?.pveVersion) {
-      data.pveVersion = extra.pveVersion;
+      metadata.pveVersion = extra.pveVersion;
     }
     if (extra?.agentVersion) {
-      data.agentVersion = extra.agentVersion;
+      metadata.agentVersion = extra.agentVersion;
     }
     // Counts: 0 is a legitimate value — gate on undefined, not falsiness.
     if (extra?.nodeCount !== undefined) {
-      data.nodeCount = extra.nodeCount;
+      metadata.nodeCount = extra.nodeCount;
     }
     if (extra?.onlineNodeCount !== undefined) {
-      data.onlineNodeCount = extra.onlineNodeCount;
+      metadata.onlineNodeCount = extra.onlineNodeCount;
     }
     if (extra?.guestCount !== undefined) {
-      data.guestCount = extra.guestCount;
+      metadata.guestCount = extra.guestCount;
     }
     if (extra?.storageCount !== undefined) {
-      data.storageCount = extra.storageCount;
+      metadata.storageCount = extra.storageCount;
     }
     if (extra?.guestsWithoutBackupCount !== undefined) {
-      data.guestsWithoutBackupCount = extra.guestsWithoutBackupCount;
+      metadata.guestsWithoutBackupCount = extra.guestsWithoutBackupCount;
     }
 
     /*
-     * Heartbeat write: a single-statement UPDATE with no hooks and no
-     * `version` bump, avoiding the hot-row Postgres lock convoy that the
-     * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
+     * One gated, non-blocking heartbeat write. The gates, the fail-open /
+     * fail-closed split and the liveness-only fallback all live in
+     * ResourceHeartbeat — see there for why this row's throttle used to
+     * provide no throttling at all.
      */
-    await this.updateColumnsByIdWithoutHooks({
+    await ResourceHeartbeat.write({
+      service: this,
       id: clusterId,
-      data: data,
+      cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
+      throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
+      liveness: liveness,
+      metadata: metadata,
+      fingerprint: extrasFingerprint,
+      describe: `proxmox cluster ${clusterId.toString()}`,
     });
   }
 

package/Server/Services/ProxmoxResourceService.ts

@@ -1,9 +1,11 @@
 import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/ProxmoxResource";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ColumnLength from "../../Types/Database/ColumnLength";
 import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import QueryHelper from "../Types/Database/QueryHelper";
+import { truncateShortText } from "../Utils/Database/TruncateColumnValue";
 import logger from "../Utils/Logger";
 
 /*
@@ -89,6 +91,26 @@ const UPSERT_COLUMNS: Array<string> = [
   "version",
 ];
 
+/*
+ * ProxmoxResource's text columns are all ShortText (100 chars). The
+ * bulk paths below go through manager.query(), which skips the length
+ * validation DatabaseService applies to ordinary writes, so a single
+ * oversized value aborts the whole 500-row INSERT chunk it rides in.
+ * Clamp per value instead, so one pathological guest can never drop the
+ * rest of the scrape.
+ */
+function sanitizeResource(r: ParsedProxmoxResource): ParsedProxmoxResource {
+  return {
+    ...r,
+    kind: truncateShortText(r.kind),
+    externalId: truncateShortText(r.externalId),
+    name: truncateShortText(r.name),
+    guestType: truncateShortText(r.guestType),
+    parentNodeName: truncateShortText(r.parentNodeName),
+    haState: truncateShortText(r.haState),
+  };
+}
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
@@ -116,9 +138,21 @@ export class Service extends DatabaseService<Model> {
       return;
     }
 
+    const resources: Array<ParsedProxmoxResource> = data.resources.map(
+      (r: ParsedProxmoxResource) => {
+        const sanitized: ParsedProxmoxResource = sanitizeResource(r);
+        if (sanitized.externalId !== r.externalId) {
+          logger.warn(
+            `ProxmoxResource externalId exceeds ${ColumnLength.ShortText} chars; truncated to "${sanitized.externalId}" (cluster ${data.proxmoxClusterId.toString()}).`,
+          );
+        }
+        return sanitized;
+      },
+    );
+
     // Chunk to keep individual statement parameter counts reasonable.
-    for (let i: number = 0; i < data.resources.length; i += UPSERT_BATCH_SIZE) {
-      const chunk: Array<ParsedProxmoxResource> = data.resources.slice(
+    for (let i: number = 0; i < resources.length; i += UPSERT_BATCH_SIZE) {
+      const chunk: Array<ParsedProxmoxResource> = resources.slice(
         i,
         i + UPSERT_BATCH_SIZE,
       );
@@ -219,9 +253,14 @@ export class Service extends DatabaseService<Model> {
         valueFragments.push(
           `($${paramIndex++}, $${paramIndex++}, $${paramIndex++}::numeric, $${paramIndex++}::bigint, $${paramIndex++}::bigint, $${paramIndex++}::numeric, $${paramIndex++}::bigint, $${paramIndex++}::bigint, $${paramIndex++}::timestamptz)`,
         );
+        /*
+         * The identity clamps MUST match bulkUpsert's, or this mirror
+         * UPDATE would silently miss the row the upsert just wrote
+         * under the truncated externalId.
+         */
         params.push(
-          m.kind,
-          m.externalId,
+          truncateShortText(m.kind),
+          truncateShortText(m.externalId),
           m.cpuPercent !== null && m.cpuPercent !== undefined
             ? m.cpuPercent
             : null,

package/Server/Services/RumApplicationService.ts

@@ -2,6 +2,7 @@ import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/RumApplication";
 import Label from "../../Models/DatabaseModels/Label";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
 import ObjectID from "../../Types/ObjectID";
 import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
@@ -276,7 +277,6 @@ export class Service extends DatabaseService<Model> {
       sdkLanguage?: string | undefined;
     },
   ): Promise<void> {
-    const cacheKey: string = rumApplicationId.toString();
     const extrasFingerprint: string = crypto
       .createHash("sha1")
       .update(
@@ -288,58 +288,40 @@ export class Service extends DatabaseService<Model> {
       )
       .digest("hex");
 
-    let cached: string | null = null;
-    try {
-      cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-    } catch {
-      /*
-       * Cache unavailable — fail open and refresh lastSeenAt anyway. A
-       * cache error must never skip the DB write below, otherwise the
-       * resource is wrongly marked "disconnected" while telemetry is
-       * still flowing. Mirrors shouldRunMaintenance's fail-open stance.
-       */
-      cached = null;
-    }
-
-    if (cached === extrasFingerprint) {
-      return; // same data was written recently
-    }
-
-    try {
-      await GlobalCache.setString(
-        LAST_SEEN_CACHE_NAMESPACE,
-        cacheKey,
-        extrasFingerprint,
-        { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
-      );
-    } catch {
-      // Best-effort throttle write; proceed with the DB update regardless.
-    }
-
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = {
+    const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
       otelCollectorStatus: "connected",
     };
 
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const metadata: any = {};
+
     if (extra?.agentVersion) {
-      data.agentVersion = extra.agentVersion;
+      metadata.agentVersion = extra.agentVersion;
     }
     if (extra?.clientType) {
-      data.clientType = extra.clientType;
+      metadata.clientType = extra.clientType;
     }
     if (extra?.sdkLanguage) {
-      data.sdkLanguage = extra.sdkLanguage;
+      metadata.sdkLanguage = extra.sdkLanguage;
     }
 
     /*
-     * Heartbeat write: a single-statement UPDATE with no hooks and no
-     * `version` bump, avoiding the hot-row Postgres lock convoy that the
-     * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
+     * One gated, non-blocking heartbeat write. The gates, the fail-open /
+     * fail-closed split and the liveness-only fallback all live in
+     * ResourceHeartbeat — see there for why this row's throttle used to
+     * provide no throttling at all.
      */
-    await this.updateColumnsByIdWithoutHooks({
+    await ResourceHeartbeat.write({
+      service: this,
       id: rumApplicationId,
-      data: data,
+      cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
+      throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
+      liveness: liveness,
+      metadata: metadata,
+      fingerprint: extrasFingerprint,
+      describe: `rum application ${rumApplicationId.toString()}`,
     });
   }
 

package/Server/Services/ScheduledMaintenanceStateTimelineService.ts

@@ -26,6 +26,7 @@ import ScheduledMaintenanceStateTimeline from "../../Models/DatabaseModels/Sched
 import { IsBillingEnabled } from "../EnvironmentConfig";
 import ScheduledMaintenanceFeedService from "./ScheduledMaintenanceFeedService";
 import { ScheduledMaintenanceFeedEventType } from "../../Models/DatabaseModels/ScheduledMaintenanceFeed";
+import ProjectScopedReferenceValidator from "../Utils/Database/ProjectScopedReferenceValidator";
 import logger, { LogAttributes } from "../Utils/Logger";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { LIMIT_PER_PROJECT } from "../../Types/Database/LimitMax";
@@ -122,6 +123,26 @@ export class Service extends DatabaseService<ScheduledMaintenanceStateTimeline>
         throw new BadDataException("scheduledMaintenanceStateId is null");
       }
 
+      /*
+       * Same guard, same reason, as MonitorStatusTimelineService: an id that
+       * exists nowhere (or belongs to another project) used to reach Postgres
+       * and come back as a raw foreign key violation. The ordering check below
+       * does look the state up, but only when there IS a preceding row with an
+       * order, so the first timeline row for a scheduled maintenance event went
+       * through unchecked.
+       */
+      await ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: createBy.props.tenantId || createBy.data.projectId,
+        subject: "scheduled maintenance state timeline",
+        references: [
+          {
+            modelName: "Scheduled Maintenance State",
+            id: scheduledMaintenanceStateId,
+            service: ScheduledMaintenanceStateService,
+          },
+        ],
+      });
+
       const stateBeforeThis: ScheduledMaintenanceStateTimeline | null =
         await this.findOneBy({
           query: {