@oneuptime/common 11.0.1 → 11.0.2

package/Server/Infrastructure/Postgres/SchemaMigrations/1781700000000-AddProxmoxCephV3Columns.ts

@@ -0,0 +1,64 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Proxmox + Ceph V3 (Internal/Roadmap/ProxmoxCephProductsV3.md):
+ *
+ * 1. WI-24 backup coverage — ProxmoxCluster.guestsWithoutBackupCount
+ *    (written from pve_not_backed_up_total on the WI-3 extras path) and
+ *    ProxmoxResource.isBackedUp (per-guest flag derived from
+ *    pve_not_backed_up_info presence: an info series carrying the
+ *    guest's id means NOT covered by any backup job). Both nullable —
+ *    NULL means the exporter's cluster-level backup-info collector has
+ *    not reported, which is distinct from "0 uncovered guests".
+ *
+ * 2. WI-28 hyperconverged cross-link — nullable
+ *    ProxmoxCluster.cephClusterId FK (SET NULL on delete), manually
+ *    linked via the cluster's Settings page. No auto-link heuristic:
+ *    pve-exporter exposes no fsid, so there is no honest join signal
+ *    (contrast WI-17's Host link, which had name equality). Mirrors
+ *    Host.proxmoxClusterId: SET NULL, no index.
+ *
+ * Columns/FKs are derived from the model decorators in
+ * Common/Models/DatabaseModels/{ProxmoxCluster,ProxmoxResource}.ts.
+ */
+export class AddProxmoxCephV3Columns1781700000000
+  implements MigrationInterface
+{
+  public name = "AddProxmoxCephV3Columns1781700000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    // WI-24: backup coverage snapshot columns.
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxCluster" ADD "guestsWithoutBackupCount" integer`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxResource" ADD "isBackedUp" boolean`,
+    );
+
+    // WI-28: ProxmoxCluster -> CephCluster cross-link (manual only).
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxCluster" ADD "cephClusterId" uuid`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxCluster" ADD CONSTRAINT "FK_proxmox_cluster_cephClusterId" FOREIGN KEY ("cephClusterId") REFERENCES "CephCluster"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    // WI-28: Ceph cross-link.
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxCluster" DROP CONSTRAINT "FK_proxmox_cluster_cephClusterId"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxCluster" DROP COLUMN "cephClusterId"`,
+    );
+
+    // WI-24: backup coverage snapshot columns.
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxResource" DROP COLUMN "isBackedUp"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxCluster" DROP COLUMN "guestsWithoutBackupCount"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -385,6 +385,10 @@ import { AddTelemetryEntityRelationshipTable1781200000001 } from "./178120000000
 import { MigrationName1781250074195 } from "./1781250074195-MigrationName";
 import { AddTelemetryEntityLabels1781300000000 } from "./1781300000000-AddTelemetryEntityLabels";
 import { AddServiceTelemetrySdkLanguage1781400000000 } from "./1781400000000-AddServiceTelemetrySdkLanguage";
+import { AddProxmoxAndCephClusterTables1781500000000 } from "./1781500000000-AddProxmoxAndCephClusterTables";
+import { AddProxmoxCephV2Columns1781600000000 } from "./1781600000000-AddProxmoxCephV2Columns";
+import { AddProxmoxCephActivityAndRules1781600000001 } from "./1781600000001-AddProxmoxCephActivityAndRules";
+import { AddProxmoxCephV3Columns1781700000000 } from "./1781700000000-AddProxmoxCephV3Columns";
 
 export default [
   InitialMigration,
@@ -774,4 +778,8 @@ export default [
   AddTelemetryEntityLabels1781300000000,
   AddServiceTelemetrySdkLanguage1781400000000,
   MigrationName1781250074195,
+  AddProxmoxAndCephClusterTables1781500000000,
+  AddProxmoxCephV2Columns1781600000000,
+  AddProxmoxCephActivityAndRules1781600000001,
+  AddProxmoxCephV3Columns1781700000000,
 ];

package/Server/Infrastructure/Redis.ts

@@ -100,22 +100,50 @@ export default abstract class Redis {
       const connectToDatabase: ConnectToDatabaseFunction = async (
         client: RedisClient,
       ): Promise<void> => {
-        try {
-          await client.connect();
-        } catch (err) {
-          if (retry < 3) {
-            logger.debug(
-              "Cannot connect to Redis. Retrying again in 5 seconds",
-            );
-            // sleep for 5 seconds.
+        // A previous attempt (or ioredis' background reconnect) already succeeded.
+        if (client.status === "ready") {
+          return;
+        }
 
-            await Sleep.sleep(5000);
+        /*
+         * The client is created with lazyConnect, so it starts in the "wait" state
+         * and only opens when we call connect(). If the first attempt fails, ioredis
+         * begins auto-reconnecting in the background (status "connecting" /
+         * "reconnecting"), and calling connect() again in that state throws
+         * "Redis is already connecting/connected" — which previously propagated out
+         * of init() and crashlooped the process. So only issue connect() when the
+         * client is idle; when a (re)connect is already in flight, skip the call and
+         * wait for it to settle on the next iteration.
+         */
+        const isIdle: boolean =
+          client.status === "wait" ||
+          client.status === "end" ||
+          client.status === "close";
 
-            retry++;
-            return await connectToDatabase(client);
+        try {
+          if (isIdle) {
+            await client.connect();
+            return;
+          }
+        } catch (err) {
+          if (retry >= 3) {
+            throw err;
           }
-          throw err;
         }
+
+        if (retry >= 3) {
+          throw new Error(
+            `Unable to connect to Redis at ${RedisHostname}:${RedisPort.toNumber()} (status: ${client.status})`,
+          );
+        }
+
+        logger.debug("Cannot connect to Redis. Retrying again in 5 seconds");
+        // sleep for 5 seconds.
+
+        await Sleep.sleep(5000);
+
+        retry++;
+        return await connectToDatabase(client);
       };
 
       await connectToDatabase(this.client);

package/Server/Services/AnalyticsDatabaseService.ts

@@ -285,7 +285,7 @@ export default class AnalyticsDatabaseService<
 
     const columnName: string = column.key;
 
-    if (!this.doesColumnExistInDatabase(columnName)) {
+    if (!(await this.doesColumnExistInDatabase(columnName))) {
       return null;
     }
 

package/Server/Services/BillingService.ts

@@ -793,14 +793,35 @@ export class BillingService extends BaseService {
     const customer: Stripe.Response<Stripe.Customer | Stripe.DeletedCustomer> =
       await this.stripe.customers.retrieve(customerId);
 
-    if (
+    const defaultPaymentMethod:
+      | string
+      | Stripe.PaymentMethod
+      | null
+      | undefined = (customer as Stripe.Customer).invoice_settings
+      ?.default_payment_method;
+
+    const defaultPaymentMethodId: string | undefined =
+      typeof defaultPaymentMethod === "string"
+        ? defaultPaymentMethod
+        : defaultPaymentMethod?.id;
+
+    if (defaultPaymentMethodId) {
+      for (const paymentMethod of paymentMethods) {
+        paymentMethod.isDefault = paymentMethod.id === defaultPaymentMethodId;
+      }
+
+      // default payment method first — it's charged first when paying invoices.
+      paymentMethods.sort((a: PaymentMethod, b: PaymentMethod) => {
+        return Number(b.isDefault) - Number(a.isDefault);
+      });
+    } else if (
       (customer as Stripe.Customer).invoice_settings &&
-      !(customer as Stripe.Customer).invoice_settings?.default_payment_method
+      paymentMethods.length > 0 &&
+      paymentMethods[0]?.id
     ) {
       // set the first payment method as default.
-      if (paymentMethods.length > 0 && paymentMethods[0]?.id) {
-        await this.setDefaultPaymentMethod(customerId, paymentMethods[0]?.id);
-      }
+      await this.setDefaultPaymentMethod(customerId, paymentMethods[0].id);
+      paymentMethods[0].isDefault = true;
     }
 
     return paymentMethods;
@@ -1237,6 +1258,41 @@ export class BillingService extends BaseService {
     return invoice;
   }
 
+  /*
+   * Returns true if the error is attributable to the payment method itself
+   * (declined, expired, unusable), meaning paying with a different payment
+   * method may succeed.
+   *
+   * Not retryable:
+   * - invoice_payment_intent_requires_action (3DS/SCA): the payment method
+   *   works but needs customer authentication — throw so the interactive
+   *   flow (BillingInvoiceAPI) surfaces the authentication prompt for the
+   *   default method instead of silently charging a backup method.
+   * - Invoice-state or connectivity errors: another payment method would
+   *   not help, and if the outcome of the attempt is unknown a retry could
+   *   double-charge the customer.
+   *
+   * Note: most bank debit (ACH/SEPA/BACS) failures are asynchronous — the
+   * pay call succeeds and the failure arrives days later via webhook — so
+   * failover only catches the synchronous ones (unverified/unusable
+   * accounts).
+   */
+  private canRetryWithDifferentPaymentMethod(err: unknown): boolean {
+    const stripeError: { type?: string; code?: string } = err as {
+      type?: string;
+      code?: string;
+    };
+
+    if (stripeError?.type === "StripeCardError") {
+      return true;
+    }
+
+    return Boolean(
+      stripeError?.code?.startsWith("payment_method_") ||
+        stripeError?.code?.startsWith("bank_account_"),
+    );
+  }
+
   @CaptureSpan()
   public async payInvoice(
     customerId: string,
@@ -1250,23 +1306,55 @@ export class BillingService extends BaseService {
       throw new BadDataException(Errors.BillingService.NO_PAYMENTS_METHODS);
     }
 
-    const invoice: Stripe.Invoice = await this.stripe.invoices.pay(invoiceId, {
-      payment_method: paymentMethods[0]?.id || "",
-    });
+    /*
+     * getPaymentMethods returns the default payment method first. If it is
+     * declined, fall back to the customer's other payment methods before
+     * giving up. Attempts are capped to bound decline traffic on unattended
+     * retry paths (e.g. recharge-on-low-balance), which create a fresh
+     * invoice and retry on every cycle.
+     */
+    const maxPaymentMethodsToTry: number = 3;
+    let lastError: unknown = null;
 
-    return {
-      id: invoice.id!,
-      amount: invoice.amount_due,
-      currencyCode: invoice.currency,
-      subscriptionId: invoice.subscription?.toString() || undefined,
-      status: invoice.status?.toString() || "Unknown",
-      downloadableLink: invoice.invoice_pdf?.toString() || "",
-      customerId: invoice.customer?.toString() || "",
-      invoiceDate: invoice.created
-        ? new Date(invoice.created * 1000)
-        : OneUptimeDate.getCurrentDate(),
-      invoiceNumber: invoice.number || undefined,
-    };
+    for (const paymentMethod of paymentMethods.slice(
+      0,
+      maxPaymentMethodsToTry,
+    )) {
+      try {
+        const invoice: Stripe.Invoice = await this.stripe.invoices.pay(
+          invoiceId,
+          {
+            payment_method: paymentMethod.id,
+          },
+        );
+
+        return {
+          id: invoice.id!,
+          amount: invoice.amount_due,
+          currencyCode: invoice.currency,
+          subscriptionId: invoice.subscription?.toString() || undefined,
+          status: invoice.status?.toString() || "Unknown",
+          downloadableLink: invoice.invoice_pdf?.toString() || "",
+          customerId: invoice.customer?.toString() || "",
+          invoiceDate: invoice.created
+            ? new Date(invoice.created * 1000)
+            : OneUptimeDate.getCurrentDate(),
+          invoiceNumber: invoice.number || undefined,
+        };
+      } catch (err) {
+        logger.error(
+          `Failed to pay invoice ${invoiceId} with payment method ${paymentMethod.type} ending in ${paymentMethod.last4Digits}: ${err}`,
+        );
+
+        if (!this.canRetryWithDifferentPaymentMethod(err)) {
+          throw err;
+        }
+
+        lastError = err;
+      }
+    }
+
+    throw lastError;
   }
 
   public getMeteredPlanPriceId(productType: ProductType): string {

package/Server/Services/CephClusterLabelRuleEngineService.ts

@@ -0,0 +1,200 @@
+import Label from "../../Models/DatabaseModels/Label";
+import CephCluster from "../../Models/DatabaseModels/CephCluster";
+import CephClusterLabelRule from "../../Models/DatabaseModels/CephClusterLabelRule";
+import CephClusterLabelRuleService from "./CephClusterLabelRuleService";
+import CephClusterService from "./CephClusterService";
+import ObjectID from "../../Types/ObjectID";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import logger, { LogAttributes } from "../Utils/Logger";
+
+class CephClusterLabelRuleEngineServiceClass {
+  /**
+   * Evaluates CephClusterLabelRule rows for the given Ceph cluster and attaches matched
+   * labels to it. The union is deduped against labels already on the Ceph cluster
+   * before insert to avoid PK conflicts on the join table.
+   */
+  @CaptureSpan()
+  public async applyRulesToCephCluster(
+    cephCluster: CephCluster,
+  ): Promise<void> {
+    if (!cephCluster.id || !cephCluster.projectId) {
+      return;
+    }
+
+    try {
+      const rules: Array<CephClusterLabelRule> =
+        await CephClusterLabelRuleService.findBy({
+          query: {
+            projectId: cephCluster.projectId,
+            isEnabled: true,
+          },
+          props: { isRoot: true },
+          select: {
+            _id: true,
+            name: true,
+            cephClusterLabels: { _id: true },
+            cephClusterNamePattern: true,
+            cephClusterDescriptionPattern: true,
+            labelsToAdd: { _id: true },
+          },
+          limit: 100,
+          skip: 0,
+        });
+
+      if (rules.length === 0) {
+        return;
+      }
+
+      const cephClusterWithDetails: CephCluster | null =
+        await CephClusterService.findOneById({
+          id: cephCluster.id,
+          select: {
+            name: true,
+            description: true,
+            labels: { _id: true },
+          },
+          props: { isRoot: true },
+        });
+
+      if (!cephClusterWithDetails) {
+        return;
+      }
+
+      const labelIdsToAdd: Set<string> = new Set();
+
+      for (const rule of rules) {
+        const matches: boolean = this.doesCephClusterMatchRule(
+          cephClusterWithDetails,
+          rule,
+        );
+        if (!matches) {
+          continue;
+        }
+        for (const label of rule.labelsToAdd || []) {
+          if (label.id) {
+            labelIdsToAdd.add(label.id.toString());
+          }
+        }
+      }
+
+      if (labelIdsToAdd.size === 0) {
+        return;
+      }
+
+      const existingLabelIds: Set<string> = new Set(
+        (cephClusterWithDetails.labels || [])
+          .map((l: Label) => {
+            return l.id?.toString() || "";
+          })
+          .filter((id: string) => {
+            return id !== "";
+          }),
+      );
+
+      const newLabelIds: Array<string> = Array.from(labelIdsToAdd).filter(
+        (id: string) => {
+          return !existingLabelIds.has(id);
+        },
+      );
+      if (newLabelIds.length === 0) {
+        return;
+      }
+
+      await CephClusterService.getRepository()
+        .createQueryBuilder()
+        .relation(CephCluster, "labels")
+        .of(cephCluster.id.toString())
+        .add(newLabelIds);
+
+      /*
+       * Sync in-memory cephCluster.labels so a downstream owner-rule engine in
+       * the same onCreateSuccess chain can match on rule-added labels.
+       */
+      const mergedLabelIds: Set<string> = new Set([
+        ...existingLabelIds,
+        ...newLabelIds,
+      ]);
+      cephCluster.labels = Array.from(mergedLabelIds).map((id: string) => {
+        const label: Label = new Label();
+        label.id = new ObjectID(id);
+        return label;
+      });
+
+      logger.debug(
+        `CephClusterLabelRuleEngine attached ${newLabelIds.length} labels to Ceph cluster ${cephCluster.id}`,
+        { projectId: cephCluster.projectId.toString() } as LogAttributes,
+      );
+    } catch (error) {
+      logger.error(`Error applying Ceph cluster label rules: ${error}`, {
+        projectId: cephCluster.projectId?.toString(),
+        cephClusterId: cephCluster.id?.toString(),
+      } as LogAttributes);
+    }
+  }
+
+  private doesCephClusterMatchRule(
+    cephCluster: CephCluster,
+    rule: CephClusterLabelRule,
+  ): boolean {
+    if (rule.cephClusterLabels && rule.cephClusterLabels.length > 0) {
+      if (!cephCluster.labels || cephCluster.labels.length === 0) {
+        return false;
+      }
+      const ruleLabelIds: Array<string> = rule.cephClusterLabels.map(
+        (l: Label) => {
+          return l.id?.toString() || "";
+        },
+      );
+      const labelIds: Array<string> = cephCluster.labels.map((l: Label) => {
+        return l.id?.toString() || "";
+      });
+      if (
+        !ruleLabelIds.some((id: string) => {
+          return labelIds.includes(id);
+        })
+      ) {
+        return false;
+      }
+    }
+
+    if (
+      rule.cephClusterNamePattern &&
+      (!cephCluster.name ||
+        !this.testRegex(rule.cephClusterNamePattern, cephCluster.name, rule))
+    ) {
+      return false;
+    }
+
+    if (
+      rule.cephClusterDescriptionPattern &&
+      (!cephCluster.description ||
+        !this.testRegex(
+          rule.cephClusterDescriptionPattern,
+          cephCluster.description,
+          rule,
+        ))
+    ) {
+      return false;
+    }
+
+    return true;
+  }
+
+  private testRegex(
+    pattern: string,
+    value: string,
+    rule: CephClusterLabelRule,
+  ): boolean {
+    try {
+      const regex: RegExp = new RegExp(pattern, "i");
+      return regex.test(value);
+    } catch {
+      logger.warn(
+        `Invalid regex in Ceph cluster label rule ${rule.id}: ${pattern}`,
+      );
+      return false;
+    }
+  }
+}
+
+export default new CephClusterLabelRuleEngineServiceClass();

package/Server/Services/CephClusterLabelRuleService.ts

@@ -0,0 +1,14 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/CephClusterLabelRule";
+import { IsBillingEnabled } from "../EnvironmentConfig";
+
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+    if (IsBillingEnabled) {
+      this.hardDeleteItemsOlderThanInDays("createdAt", 3 * 365);
+    }
+  }
+}
+
+export default new Service();