@oneuptime/common 11.0.1 → 11.0.3

package/Server/Infrastructure/Postgres/SchemaMigrations/1781700000000-AddProxmoxCephV3Columns.ts

@@ -0,0 +1,64 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+/*
+ * Proxmox + Ceph V3 (Internal/Roadmap/ProxmoxCephProductsV3.md):
+ *
+ * 1. WI-24 backup coverage — ProxmoxCluster.guestsWithoutBackupCount
+ *    (written from pve_not_backed_up_total on the WI-3 extras path) and
+ *    ProxmoxResource.isBackedUp (per-guest flag derived from
+ *    pve_not_backed_up_info presence: an info series carrying the
+ *    guest's id means NOT covered by any backup job). Both nullable —
+ *    NULL means the exporter's cluster-level backup-info collector has
+ *    not reported, which is distinct from "0 uncovered guests".
+ *
+ * 2. WI-28 hyperconverged cross-link — nullable
+ *    ProxmoxCluster.cephClusterId FK (SET NULL on delete), manually
+ *    linked via the cluster's Settings page. No auto-link heuristic:
+ *    pve-exporter exposes no fsid, so there is no honest join signal
+ *    (contrast WI-17's Host link, which had name equality). Mirrors
+ *    Host.proxmoxClusterId: SET NULL, no index.
+ *
+ * Columns/FKs are derived from the model decorators in
+ * Common/Models/DatabaseModels/{ProxmoxCluster,ProxmoxResource}.ts.
+ */
+export class AddProxmoxCephV3Columns1781700000000
+  implements MigrationInterface
+{
+  public name = "AddProxmoxCephV3Columns1781700000000";
+
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    // WI-24: backup coverage snapshot columns.
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxCluster" ADD "guestsWithoutBackupCount" integer`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxResource" ADD "isBackedUp" boolean`,
+    );
+
+    // WI-28: ProxmoxCluster -> CephCluster cross-link (manual only).
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxCluster" ADD "cephClusterId" uuid`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxCluster" ADD CONSTRAINT "FK_proxmox_cluster_cephClusterId" FOREIGN KEY ("cephClusterId") REFERENCES "CephCluster"("_id") ON DELETE SET NULL ON UPDATE NO ACTION`,
+    );
+  }
+
+  public async down(queryRunner: QueryRunner): Promise<void> {
+    // WI-28: Ceph cross-link.
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxCluster" DROP CONSTRAINT "FK_proxmox_cluster_cephClusterId"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxCluster" DROP COLUMN "cephClusterId"`,
+    );
+
+    // WI-24: backup coverage snapshot columns.
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxResource" DROP COLUMN "isBackedUp"`,
+    );
+    await queryRunner.query(
+      `ALTER TABLE "ProxmoxCluster" DROP COLUMN "guestsWithoutBackupCount"`,
+    );
+  }
+}

package/Server/Infrastructure/Postgres/SchemaMigrations/Index.ts

@@ -385,6 +385,11 @@ import { AddTelemetryEntityRelationshipTable1781200000001 } from "./178120000000
 import { MigrationName1781250074195 } from "./1781250074195-MigrationName";
 import { AddTelemetryEntityLabels1781300000000 } from "./1781300000000-AddTelemetryEntityLabels";
 import { AddServiceTelemetrySdkLanguage1781400000000 } from "./1781400000000-AddServiceTelemetrySdkLanguage";
+import { AddProxmoxAndCephClusterTables1781500000000 } from "./1781500000000-AddProxmoxAndCephClusterTables";
+import { MigrationName1781587937032 } from "./1781587937032-MigrationName";
+import { AddProxmoxCephV2Columns1781600000000 } from "./1781600000000-AddProxmoxCephV2Columns";
+import { AddProxmoxCephActivityAndRules1781600000001 } from "./1781600000001-AddProxmoxCephActivityAndRules";
+import { AddProxmoxCephV3Columns1781700000000 } from "./1781700000000-AddProxmoxCephV3Columns";
 
 export default [
   InitialMigration,
@@ -774,4 +779,9 @@ export default [
   AddTelemetryEntityLabels1781300000000,
   AddServiceTelemetrySdkLanguage1781400000000,
   MigrationName1781250074195,
+  AddProxmoxAndCephClusterTables1781500000000,
+  AddProxmoxCephV2Columns1781600000000,
+  AddProxmoxCephActivityAndRules1781600000001,
+  AddProxmoxCephV3Columns1781700000000,
+  MigrationName1781587937032,
 ];

package/Server/Infrastructure/Redis.ts

@@ -100,22 +100,50 @@ export default abstract class Redis {
       const connectToDatabase: ConnectToDatabaseFunction = async (
         client: RedisClient,
       ): Promise<void> => {
-        try {
-          await client.connect();
-        } catch (err) {
-          if (retry < 3) {
-            logger.debug(
-              "Cannot connect to Redis. Retrying again in 5 seconds",
-            );
-            // sleep for 5 seconds.
+        // A previous attempt (or ioredis' background reconnect) already succeeded.
+        if (client.status === "ready") {
+          return;
+        }
 
-            await Sleep.sleep(5000);
+        /*
+         * The client is created with lazyConnect, so it starts in the "wait" state
+         * and only opens when we call connect(). If the first attempt fails, ioredis
+         * begins auto-reconnecting in the background (status "connecting" /
+         * "reconnecting"), and calling connect() again in that state throws
+         * "Redis is already connecting/connected" — which previously propagated out
+         * of init() and crashlooped the process. So only issue connect() when the
+         * client is idle; when a (re)connect is already in flight, skip the call and
+         * wait for it to settle on the next iteration.
+         */
+        const isIdle: boolean =
+          client.status === "wait" ||
+          client.status === "end" ||
+          client.status === "close";
 
-            retry++;
-            return await connectToDatabase(client);
+        try {
+          if (isIdle) {
+            await client.connect();
+            return;
+          }
+        } catch (err) {
+          if (retry >= 3) {
+            throw err;
           }
-          throw err;
         }
+
+        if (retry >= 3) {
+          throw new Error(
+            `Unable to connect to Redis at ${RedisHostname}:${RedisPort.toNumber()} (status: ${client.status})`,
+          );
+        }
+
+        logger.debug("Cannot connect to Redis. Retrying again in 5 seconds");
+        // sleep for 5 seconds.
+
+        await Sleep.sleep(5000);
+
+        retry++;
+        return await connectToDatabase(client);
       };
 
       await connectToDatabase(this.client);

package/Server/Services/AlertLabelRuleEngineService.ts

@@ -6,11 +6,13 @@ import Host from "../../Models/DatabaseModels/Host";
 import KubernetesCluster from "../../Models/DatabaseModels/KubernetesCluster";
 import Label from "../../Models/DatabaseModels/Label";
 import Monitor from "../../Models/DatabaseModels/Monitor";
+import PodmanHost from "../../Models/DatabaseModels/PodmanHost";
 import Service from "../../Models/DatabaseModels/Service";
 import AlertFeedService from "./AlertFeedService";
 import AlertLabelRuleService from "./AlertLabelRuleService";
 import AlertService from "./AlertService";
 import DockerHostService from "./DockerHostService";
+import PodmanHostService from "./PodmanHostService";
 import HostService from "./HostService";
 import KubernetesClusterService from "./KubernetesClusterService";
 import LabelService from "./LabelService";
@@ -33,6 +35,7 @@ class AlertLabelRuleEngineServiceClass {
    *   - all labels of the alert's hosts when `inheritLabelsFromHosts`
    *   - all labels of the alert's Kubernetes clusters when `inheritLabelsFromKubernetesClusters`
    *   - all labels of the alert's Docker hosts when `inheritLabelsFromDockerHosts`
+   *   - all labels of the alert's Podman hosts when `inheritLabelsFromPodmanHosts`
    *   - all labels of the alert's services when `inheritLabelsFromServices`
    * The union is deduped against labels already on the alert before insert
    * to avoid PK conflicts on the AlertLabel join table.
@@ -66,6 +69,7 @@ class AlertLabelRuleEngineServiceClass {
           inheritLabelsFromHosts: true,
           inheritLabelsFromKubernetesClusters: true,
           inheritLabelsFromDockerHosts: true,
+          inheritLabelsFromPodmanHosts: true,
           inheritLabelsFromServices: true,
         },
         limit: 100,
@@ -81,6 +85,7 @@ class AlertLabelRuleEngineServiceClass {
       let inheritFromHosts: boolean = false;
       let inheritFromKubernetesClusters: boolean = false;
       let inheritFromDockerHosts: boolean = false;
+      let inheritFromPodmanHosts: boolean = false;
       let inheritFromServices: boolean = false;
       const matchedRules: Array<AlertLabelRule> = [];
 
@@ -107,6 +112,9 @@ class AlertLabelRuleEngineServiceClass {
         if (rule.inheritLabelsFromDockerHosts) {
           inheritFromDockerHosts = true;
         }
+        if (rule.inheritLabelsFromPodmanHosts) {
+          inheritFromPodmanHosts = true;
+        }
         if (rule.inheritLabelsFromServices) {
           inheritFromServices = true;
         }
@@ -116,6 +124,7 @@ class AlertLabelRuleEngineServiceClass {
         inheritFromHosts ||
         inheritFromKubernetesClusters ||
         inheritFromDockerHosts ||
+        inheritFromPodmanHosts ||
         inheritFromServices;
 
       let alertWithResources: Alert | null = null;
@@ -126,6 +135,7 @@ class AlertLabelRuleEngineServiceClass {
             hosts: { _id: true },
             kubernetesClusters: { _id: true },
             dockerHosts: { _id: true },
+            podmanHosts: { _id: true },
             services: { _id: true },
           },
           props: { isRoot: true },
@@ -204,6 +214,25 @@ class AlertLabelRuleEngineServiceClass {
         }
       }
 
+      if (inheritFromPodmanHosts && alertWithResources?.podmanHosts?.length) {
+        for (const alertPodmanHost of alertWithResources.podmanHosts) {
+          if (!alertPodmanHost.id) {
+            continue;
+          }
+          const podmanHost: PodmanHost | null =
+            await PodmanHostService.findOneById({
+              id: alertPodmanHost.id,
+              select: { labels: { _id: true } },
+              props: { isRoot: true },
+            });
+          for (const label of podmanHost?.labels || []) {
+            if (label.id) {
+              labelIdsToAdd.add(label.id.toString());
+            }
+          }
+        }
+      }
+
       if (inheritFromServices && alertWithResources?.services?.length) {
         for (const alertService of alertWithResources.services) {
           if (!alertService.id) {

package/Server/Services/AlertOwnerRuleEngineService.ts

@@ -14,6 +14,9 @@ import Label from "../../Models/DatabaseModels/Label";
 import Monitor from "../../Models/DatabaseModels/Monitor";
 import MonitorOwnerTeam from "../../Models/DatabaseModels/MonitorOwnerTeam";
 import MonitorOwnerUser from "../../Models/DatabaseModels/MonitorOwnerUser";
+import PodmanHost from "../../Models/DatabaseModels/PodmanHost";
+import PodmanHostOwnerTeam from "../../Models/DatabaseModels/PodmanHostOwnerTeam";
+import PodmanHostOwnerUser from "../../Models/DatabaseModels/PodmanHostOwnerUser";
 import Service from "../../Models/DatabaseModels/Service";
 import ServiceOwnerTeam from "../../Models/DatabaseModels/ServiceOwnerTeam";
 import ServiceOwnerUser from "../../Models/DatabaseModels/ServiceOwnerUser";
@@ -31,6 +34,8 @@ import KubernetesClusterOwnerUserService from "./KubernetesClusterOwnerUserServi
 import MonitorOwnerTeamService from "./MonitorOwnerTeamService";
 import MonitorOwnerUserService from "./MonitorOwnerUserService";
 import MonitorService from "./MonitorService";
+import PodmanHostOwnerTeamService from "./PodmanHostOwnerTeamService";
+import PodmanHostOwnerUserService from "./PodmanHostOwnerUserService";
 import ServiceOwnerTeamService from "./ServiceOwnerTeamService";
 import ServiceOwnerUserService from "./ServiceOwnerUserService";
 import TeamService from "./TeamService";
@@ -79,6 +84,7 @@ class AlertOwnerRuleEngineServiceClass {
           inheritOwnersFromHosts: true,
           inheritOwnersFromKubernetesClusters: true,
           inheritOwnersFromDockerHosts: true,
+          inheritOwnersFromPodmanHosts: true,
           inheritOwnersFromServices: true,
         },
         limit: 100,
@@ -105,6 +111,7 @@ class AlertOwnerRuleEngineServiceClass {
       let inheritFromHosts: boolean = false;
       let inheritFromKubernetesClusters: boolean = false;
       let inheritFromDockerHosts: boolean = false;
+      let inheritFromPodmanHosts: boolean = false;
       let inheritFromServices: boolean = false;
       const inheritNotifyMode: { value: boolean | null } = { value: null };
 
@@ -153,6 +160,12 @@ class AlertOwnerRuleEngineServiceClass {
           inheritNotifyMode.value =
             inheritNotifyMode.value === true ? true : notify;
         }
+        if (rule.inheritOwnersFromPodmanHosts) {
+          inheritFromPodmanHosts = true;
+          ruleAddedAny = true;
+          inheritNotifyMode.value =
+            inheritNotifyMode.value === true ? true : notify;
+        }
         if (rule.inheritOwnersFromServices) {
           inheritFromServices = true;
           ruleAddedAny = true;
@@ -172,6 +185,8 @@ class AlertOwnerRuleEngineServiceClass {
       const inheritedFromKubernetesClusterTeamIds: Set<string> = new Set();
       const inheritedFromDockerHostUserIds: Set<string> = new Set();
       const inheritedFromDockerHostTeamIds: Set<string> = new Set();
+      const inheritedFromPodmanHostUserIds: Set<string> = new Set();
+      const inheritedFromPodmanHostTeamIds: Set<string> = new Set();
       const inheritedFromServiceUserIds: Set<string> = new Set();
       const inheritedFromServiceTeamIds: Set<string> = new Set();
 
@@ -334,6 +349,47 @@ class AlertOwnerRuleEngineServiceClass {
         }
       }
 
+      if (inheritFromPodmanHosts && alert.podmanHosts?.length) {
+        const podmanHostIds: Array<ObjectID> = alert.podmanHosts
+          .map((p: PodmanHost) => {
+            return p.id;
+          })
+          .filter((id: ObjectID | null | undefined): id is ObjectID => {
+            return Boolean(id);
+          });
+        if (podmanHostIds.length > 0) {
+          const [podmanHostOwnerUsers, podmanHostOwnerTeams]: [
+            Array<PodmanHostOwnerUser>,
+            Array<PodmanHostOwnerTeam>,
+          ] = await Promise.all([
+            PodmanHostOwnerUserService.findBy({
+              query: { podmanHostId: QueryHelper.any(podmanHostIds) },
+              select: { userId: true },
+              props: { isRoot: true },
+              limit: LIMIT_MAX,
+              skip: 0,
+            }),
+            PodmanHostOwnerTeamService.findBy({
+              query: { podmanHostId: QueryHelper.any(podmanHostIds) },
+              select: { teamId: true },
+              props: { isRoot: true },
+              limit: LIMIT_MAX,
+              skip: 0,
+            }),
+          ]);
+          for (const ownerUser of podmanHostOwnerUsers) {
+            if (ownerUser.userId) {
+              inheritedFromPodmanHostUserIds.add(ownerUser.userId.toString());
+            }
+          }
+          for (const ownerTeam of podmanHostOwnerTeams) {
+            if (ownerTeam.teamId) {
+              inheritedFromPodmanHostTeamIds.add(ownerTeam.teamId.toString());
+            }
+          }
+        }
+      }
+
       if (inheritFromServices && alert.services?.length) {
         const serviceIds: Array<ObjectID> = alert.services
           .map((s: Service) => {
@@ -380,6 +436,7 @@ class AlertOwnerRuleEngineServiceClass {
         ...inheritedFromHostUserIds,
         ...inheritedFromKubernetesClusterUserIds,
         ...inheritedFromDockerHostUserIds,
+        ...inheritedFromPodmanHostUserIds,
         ...inheritedFromServiceUserIds,
       ]);
       const inheritedTeamIds: Set<string> = new Set([
@@ -387,6 +444,7 @@ class AlertOwnerRuleEngineServiceClass {
         ...inheritedFromHostTeamIds,
         ...inheritedFromKubernetesClusterTeamIds,
         ...inheritedFromDockerHostTeamIds,
+        ...inheritedFromPodmanHostTeamIds,
         ...inheritedFromServiceTeamIds,
       ]);
 
@@ -458,6 +516,10 @@ class AlertOwnerRuleEngineServiceClass {
           inheritedFromDockerHostUserIds.size +
             inheritedFromDockerHostTeamIds.size >
           0,
+        inheritedFromPodmanHosts:
+          inheritedFromPodmanHostUserIds.size +
+            inheritedFromPodmanHostTeamIds.size >
+          0,
         inheritedFromServices:
           inheritedFromServiceUserIds.size + inheritedFromServiceTeamIds.size >
           0,
@@ -480,6 +542,7 @@ class AlertOwnerRuleEngineServiceClass {
     inheritedFromHosts: boolean;
     inheritedFromKubernetesClusters: boolean;
     inheritedFromDockerHosts: boolean;
+    inheritedFromPodmanHosts: boolean;
     inheritedFromServices: boolean;
   }): Promise<void> {
     const {
@@ -491,6 +554,7 @@ class AlertOwnerRuleEngineServiceClass {
       inheritedFromHosts,
       inheritedFromKubernetesClusters,
       inheritedFromDockerHosts,
+      inheritedFromPodmanHosts,
       inheritedFromServices,
     } = data;
     if (
@@ -575,6 +639,9 @@ class AlertOwnerRuleEngineServiceClass {
       if (inheritedFromDockerHosts) {
         inheritedSources.push("Docker hosts");
       }
+      if (inheritedFromPodmanHosts) {
+        inheritedSources.push("Podman hosts");
+      }
       if (inheritedFromServices) {
         inheritedSources.push("services");
       }

package/Server/Services/AnalyticsDatabaseService.ts

@@ -285,7 +285,7 @@ export default class AnalyticsDatabaseService<
 
     const columnName: string = column.key;
 
-    if (!this.doesColumnExistInDatabase(columnName)) {
+    if (!(await this.doesColumnExistInDatabase(columnName))) {
       return null;
     }
 

package/Server/Services/BillingService.ts

@@ -793,14 +793,35 @@ export class BillingService extends BaseService {
     const customer: Stripe.Response<Stripe.Customer | Stripe.DeletedCustomer> =
       await this.stripe.customers.retrieve(customerId);
 
-    if (
+    const defaultPaymentMethod:
+      | string
+      | Stripe.PaymentMethod
+      | null
+      | undefined = (customer as Stripe.Customer).invoice_settings
+      ?.default_payment_method;
+
+    const defaultPaymentMethodId: string | undefined =
+      typeof defaultPaymentMethod === "string"
+        ? defaultPaymentMethod
+        : defaultPaymentMethod?.id;
+
+    if (defaultPaymentMethodId) {
+      for (const paymentMethod of paymentMethods) {
+        paymentMethod.isDefault = paymentMethod.id === defaultPaymentMethodId;
+      }
+
+      // default payment method first — it's charged first when paying invoices.
+      paymentMethods.sort((a: PaymentMethod, b: PaymentMethod) => {
+        return Number(b.isDefault) - Number(a.isDefault);
+      });
+    } else if (
       (customer as Stripe.Customer).invoice_settings &&
-      !(customer as Stripe.Customer).invoice_settings?.default_payment_method
+      paymentMethods.length > 0 &&
+      paymentMethods[0]?.id
     ) {
       // set the first payment method as default.
-      if (paymentMethods.length > 0 && paymentMethods[0]?.id) {
-        await this.setDefaultPaymentMethod(customerId, paymentMethods[0]?.id);
-      }
+      await this.setDefaultPaymentMethod(customerId, paymentMethods[0].id);
+      paymentMethods[0].isDefault = true;
     }
 
     return paymentMethods;
@@ -1237,6 +1258,41 @@ export class BillingService extends BaseService {
     return invoice;
   }
 
+  /*
+   * Returns true if the error is attributable to the payment method itself
+   * (declined, expired, unusable), meaning paying with a different payment
+   * method may succeed.
+   *
+   * Not retryable:
+   * - invoice_payment_intent_requires_action (3DS/SCA): the payment method
+   *   works but needs customer authentication — throw so the interactive
+   *   flow (BillingInvoiceAPI) surfaces the authentication prompt for the
+   *   default method instead of silently charging a backup method.
+   * - Invoice-state or connectivity errors: another payment method would
+   *   not help, and if the outcome of the attempt is unknown a retry could
+   *   double-charge the customer.
+   *
+   * Note: most bank debit (ACH/SEPA/BACS) failures are asynchronous — the
+   * pay call succeeds and the failure arrives days later via webhook — so
+   * failover only catches the synchronous ones (unverified/unusable
+   * accounts).
+   */
+  private canRetryWithDifferentPaymentMethod(err: unknown): boolean {
+    const stripeError: { type?: string; code?: string } = err as {
+      type?: string;
+      code?: string;
+    };
+
+    if (stripeError?.type === "StripeCardError") {
+      return true;
+    }
+
+    return Boolean(
+      stripeError?.code?.startsWith("payment_method_") ||
+        stripeError?.code?.startsWith("bank_account_"),
+    );
+  }
+
   @CaptureSpan()
   public async payInvoice(
     customerId: string,
@@ -1250,23 +1306,55 @@ export class BillingService extends BaseService {
       throw new BadDataException(Errors.BillingService.NO_PAYMENTS_METHODS);
     }
 
-    const invoice: Stripe.Invoice = await this.stripe.invoices.pay(invoiceId, {
-      payment_method: paymentMethods[0]?.id || "",
-    });
+    /*
+     * getPaymentMethods returns the default payment method first. If it is
+     * declined, fall back to the customer's other payment methods before
+     * giving up. Attempts are capped to bound decline traffic on unattended
+     * retry paths (e.g. recharge-on-low-balance), which create a fresh
+     * invoice and retry on every cycle.
+     */
+    const maxPaymentMethodsToTry: number = 3;
+    let lastError: unknown = null;
 
-    return {
-      id: invoice.id!,
-      amount: invoice.amount_due,
-      currencyCode: invoice.currency,
-      subscriptionId: invoice.subscription?.toString() || undefined,
-      status: invoice.status?.toString() || "Unknown",
-      downloadableLink: invoice.invoice_pdf?.toString() || "",
-      customerId: invoice.customer?.toString() || "",
-      invoiceDate: invoice.created
-        ? new Date(invoice.created * 1000)
-        : OneUptimeDate.getCurrentDate(),
-      invoiceNumber: invoice.number || undefined,
-    };
+    for (const paymentMethod of paymentMethods.slice(
+      0,
+      maxPaymentMethodsToTry,
+    )) {
+      try {
+        const invoice: Stripe.Invoice = await this.stripe.invoices.pay(
+          invoiceId,
+          {
+            payment_method: paymentMethod.id,
+          },
+        );
+
+        return {
+          id: invoice.id!,
+          amount: invoice.amount_due,
+          currencyCode: invoice.currency,
+          subscriptionId: invoice.subscription?.toString() || undefined,
+          status: invoice.status?.toString() || "Unknown",
+          downloadableLink: invoice.invoice_pdf?.toString() || "",
+          customerId: invoice.customer?.toString() || "",
+          invoiceDate: invoice.created
+            ? new Date(invoice.created * 1000)
+            : OneUptimeDate.getCurrentDate(),
+          invoiceNumber: invoice.number || undefined,
+        };
+      } catch (err) {
+        logger.error(
+          `Failed to pay invoice ${invoiceId} with payment method ${paymentMethod.type} ending in ${paymentMethod.last4Digits}: ${err}`,
+        );
+
+        if (!this.canRetryWithDifferentPaymentMethod(err)) {
+          throw err;
+        }
+
+        lastError = err;
+      }
+    }
+
+    throw lastError;
   }
 
   public getMeteredPlanPriceId(productType: ProductType): string {