@oneuptime/common 11.7.1 → 11.7.3

package/Server/Services/ProbeService.ts

@@ -1,6 +1,7 @@
+import crypto from "crypto";
 import User from "../../Models/DatabaseModels/User";
 import CreateBy from "../Types/Database/CreateBy";
-import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
+import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
 import DatabaseService from "./DatabaseService";
 import ObjectID from "../../Types/ObjectID";
 import Version from "../../Types/Version";
@@ -36,11 +37,253 @@ import GlobalCache from "../Infrastructure/GlobalCache";
 import { createWhatsAppMessageFromTemplate } from "../Utils/WhatsAppTemplateUtil";
 import { WhatsAppMessagePayload } from "../../Types/WhatsApp/WhatsAppMessage";
 
+/*
+ * Cache namespace for the probe heartbeat throttle: holds the wall-clock of
+ * the last attempted lastAlive write per probe, so at most one Postgres
+ * write happens per probe per 30 seconds no matter how many requests the
+ * probe sends.
+ */
+const PROBE_LAST_ALIVE_CACHE_NAMESPACE: string = "probe-last-alive";
+
+/*
+ * Cache namespace for probe authentication: probeId → SHA-256 of the
+ * probe's CURRENT key, written only after the key was verified against the
+ * database. Lets the probe-auth middleware skip its per-request Probe
+ * SELECT — the exact query that queues behind a starved connection pool and
+ * stalls heartbeats. Entries are deleted whenever a probe's key changes or
+ * the probe is deleted (see onUpdateSuccess / onDeleteSuccess below), and
+ * the TTL bounds the exposure if that invalidation is ever missed.
+ */
+const PROBE_AUTH_KEY_HASH_CACHE_NAMESPACE: string = "probe-auth-key-hash";
+const PROBE_AUTH_KEY_HASH_CACHE_TTL_IN_SECONDS: number = 120;
+
+/*
+ * Revocation sentinel. Invalidation WRITES this value (with the normal TTL)
+ * instead of deleting the key: a bare DELETE can be silently undone by an
+ * in-flight verification that DB-checked the OLD key just before a rotation
+ * committed and then re-cached it just after the delete ran. The sentinel —
+ * combined with verifyProbeKey's write-only-if-unchanged re-read below —
+ * makes the straggler skip its write, so invalidation wins the race. It can
+ * never equal a real SHA-256 hex digest, so on the auth path it always
+ * behaves as a cache miss.
+ */
+const PROBE_AUTH_KEY_REVOKED_SENTINEL: string = "revoked";
+
+/*
+ * Upper bound for a single Redis cache operation on the probe request path.
+ * The ioredis client has no command timeout, so a stalled-but-open Redis
+ * socket would otherwise hang the probe-auth middleware forever with no
+ * error at all. Cache is an optimization here — on timeout we fall back to
+ * the database, exactly as we do when Redis rejects.
+ */
+const PROBE_CACHE_OPERATION_TIMEOUT_IN_MS: number = 2000;
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
   }
 
+  /*
+   * Bounds a cache operation in time. Promise.race subscribes to both
+   * promises, so a late settlement of the losing cache operation is
+   * observed and can never surface as an unhandled rejection.
+   */
+  private async boundedCacheOperation<T>(
+    operationName: string,
+    operation: Promise<T>,
+  ): Promise<T> {
+    let timer: ReturnType<typeof setTimeout> | undefined = undefined;
+
+    const timeout: Promise<never> = new Promise<never>(
+      (_resolve: (value: never) => void, reject: (err: Error) => void) => {
+        timer = setTimeout(() => {
+          reject(
+            new Error(
+              `${operationName} timed out after ${PROBE_CACHE_OPERATION_TIMEOUT_IN_MS}ms`,
+            ),
+          );
+        }, PROBE_CACHE_OPERATION_TIMEOUT_IN_MS);
+      },
+    );
+
+    try {
+      return await Promise.race([operation, timeout]);
+    } finally {
+      if (timer) {
+        clearTimeout(timer);
+      }
+    }
+  }
+
+  public hashProbeKey(probeKey: string): string {
+    return crypto.createHash("sha256").update(probeKey).digest("hex");
+  }
+
+  /*
+   * Verifies a probeId/probeKey pair, serving repeat verifications from
+   * cache. This runs on EVERY authenticated probe request (heartbeats,
+   * monitor list fetches, result ingest), so it must stay responsive even
+   * when the Postgres pool is saturated — which is exactly when the
+   * heartbeat matters most: if this check queues behind a starved pool, the
+   * probe's lastAlive goes stale and a healthy probe gets flagged
+   * Disconnected.
+   *
+   * Security invariants:
+   * - The cache stores only a SHA-256 hash of the key, never the key.
+   * - A cache entry is only ever written from a value that was JUST
+   *   verified against the database.
+   * - A cache mismatch falls through to the database, so a rotated key
+   *   works immediately.
+   * - Rotating or deleting a probe overwrites its cache entry cluster-wide
+   *   with the revocation sentinel (GlobalCache is Redis-backed), and the
+   *   write below is write-only-if-unchanged so an in-flight verification
+   *   straddling the rotation cannot resurrect the old hash. The TTL caps
+   *   the stale window if the invalidation write itself fails: a REVOKED
+   *   key can then still authenticate for at most
+   *   PROBE_AUTH_KEY_HASH_CACHE_TTL_IN_SECONDS.
+   * - Any cache failure (Redis down, slow, timeout) falls back to the
+   *   database check — never open.
+   */
+  public async verifyProbeKey(data: {
+    probeId: ObjectID;
+    probeKey: string;
+  }): Promise<boolean> {
+    if (!data.probeId) {
+      throw new BadDataException("probeId is required");
+    }
+
+    if (!data.probeKey) {
+      throw new BadDataException("probeKey is required");
+    }
+
+    const presentedKeyHash: string = this.hashProbeKey(data.probeKey);
+
+    try {
+      const cachedKeyHash: string | null = await this.boundedCacheOperation(
+        "Probe auth cache read",
+        GlobalCache.getString(
+          PROBE_AUTH_KEY_HASH_CACHE_NAMESPACE,
+          data.probeId.toString(),
+        ),
+      );
+
+      if (cachedKeyHash && cachedKeyHash === presentedKeyHash) {
+        return true;
+      }
+    } catch (err) {
+      logger.error("Error in reading probe auth cache", {
+        probeId: data.probeId?.toString(),
+      } as LogAttributes);
+      logger.error(err, {
+        probeId: data.probeId?.toString(),
+      } as LogAttributes);
+    }
+
+    // Cache miss, mismatch (e.g. key rotated), or cache failure: ask the DB.
+    const probe: Model | null = await this.findOneBy({
+      query: {
+        _id: data.probeId.toString(),
+        key: data.probeKey,
+      },
+      select: {
+        _id: true,
+      },
+      props: {
+        isRoot: true,
+      },
+    });
+
+    if (!probe) {
+      return false;
+    }
+
+    try {
+      /*
+       * Write-only-if-unchanged: re-read the entry right before writing and
+       * skip the write when someone else changed it since our first read.
+       * The DB verification above can straddle a key rotation — its SELECT
+       * resolves against the pre-rotation row, the rotation commits and
+       * invalidates, and an unconditional write here would then re-cache
+       * the just-revoked key for a fresh TTL. Seeing the revocation
+       * sentinel (or any other concurrent write) means skip: the DB check
+       * stays the source of truth and caching resumes once the entry
+       * expires. The remaining race window is the microseconds between
+       * this re-read and the write — down from the full DB round-trip.
+       */
+      const currentCacheValue: string | null = await this.boundedCacheOperation(
+        "Probe auth cache pre-write read",
+        GlobalCache.getString(
+          PROBE_AUTH_KEY_HASH_CACHE_NAMESPACE,
+          data.probeId.toString(),
+        ),
+      );
+
+      if (
+        currentCacheValue === null ||
+        currentCacheValue === presentedKeyHash
+      ) {
+        await this.boundedCacheOperation(
+          "Probe auth cache write",
+          GlobalCache.setString(
+            PROBE_AUTH_KEY_HASH_CACHE_NAMESPACE,
+            data.probeId.toString(),
+            presentedKeyHash,
+            {
+              expiresInSeconds: PROBE_AUTH_KEY_HASH_CACHE_TTL_IN_SECONDS,
+            },
+          ),
+        );
+      }
+    } catch (err) {
+      // Next request re-verifies against the DB. Nothing is broken.
+      logger.error("Error in writing probe auth cache", {
+        probeId: data.probeId?.toString(),
+      } as LogAttributes);
+      logger.error(err, {
+        probeId: data.probeId?.toString(),
+      } as LogAttributes);
+    }
+
+    return true;
+  }
+
+  /*
+   * Replaces cached auth entries with the revocation sentinel so a rotated
+   * key stops working everywhere immediately (the cache is Redis-backed and
+   * shared by all pods). A sentinel WRITE rather than a delete: a deleted
+   * key can be silently re-created by an in-flight verification that
+   * DB-checked the old key just before the rotation committed, whereas the
+   * sentinel makes that straggler's write-only-if-unchanged check fail.
+   * Errors are logged loudly but not thrown: the entries also expire via
+   * TTL, and failing the surrounding update/delete over a cache hiccup
+   * would be worse than a bounded stale window.
+   */
+  public async invalidateProbeAuthCache(
+    probeIds: Array<ObjectID>,
+  ): Promise<void> {
+    for (const probeId of probeIds) {
+      try {
+        await this.boundedCacheOperation(
+          "Probe auth cache invalidation",
+          GlobalCache.setString(
+            PROBE_AUTH_KEY_HASH_CACHE_NAMESPACE,
+            probeId.toString(),
+            PROBE_AUTH_KEY_REVOKED_SENTINEL,
+            {
+              expiresInSeconds: PROBE_AUTH_KEY_HASH_CACHE_TTL_IN_SECONDS,
+            },
+          ),
+        );
+      } catch (err) {
+        logger.error(
+          `CRITICAL: failed to invalidate probe auth cache for probe ${probeId.toString()} — a rotated/revoked probe key may keep authenticating for up to ${PROBE_AUTH_KEY_HASH_CACHE_TTL_IN_SECONDS}s`,
+          { probeId: probeId?.toString() } as LogAttributes,
+        );
+        logger.error(err, { probeId: probeId?.toString() } as LogAttributes);
+      }
+    }
+  }
+
   public async saveLastAliveInCache(
     probeId: ObjectID,
     lastAlive: Date,
@@ -50,10 +293,13 @@ export class Service extends DatabaseService<Model> {
     }
 
     try {
-      await GlobalCache.setString(
-        "probe-last-alive",
-        probeId.toString(),
-        OneUptimeDate.toString(lastAlive),
+      await this.boundedCacheOperation(
+        "Probe last-alive cache write",
+        GlobalCache.setString(
+          PROBE_LAST_ALIVE_CACHE_NAMESPACE,
+          probeId.toString(),
+          OneUptimeDate.toString(lastAlive),
+        ),
       );
     } catch (err) {
       logger.error("Error in saving last alive in cache", {
@@ -68,10 +314,14 @@ export class Service extends DatabaseService<Model> {
 
     try {
       // before we hit the database, we need to check if the lastAlive was updated in Global Cache.
-      const previousLastAliveCheck: string | null = await GlobalCache.getString(
-        "probe-last-alive",
-        probeId.toString(),
-      );
+      const previousLastAliveCheck: string | null =
+        await this.boundedCacheOperation(
+          "Probe last-alive cache read",
+          GlobalCache.getString(
+            PROBE_LAST_ALIVE_CACHE_NAMESPACE,
+            probeId.toString(),
+          ),
+        );
 
       if (!previousLastAliveCheck) {
         await this.saveLastAliveInCache(probeId, now);
@@ -129,12 +379,44 @@ export class Service extends DatabaseService<Model> {
      * hooks (onBeforeUpdate/onUpdateSuccess) are inert here. See
      * ServiceService.updateLastSeen.
      */
-    await this.updateColumnsByIdWithoutHooks({
-      id: probeId,
-      data: {
-        lastAlive: now,
-      },
-    });
+    try {
+      await this.updateColumnsByIdWithoutHooks({
+        id: probeId,
+        data: {
+          lastAlive: now,
+        },
+      });
+    } catch (err) {
+      /*
+       * The 30s throttle stamp was written BEFORE this write was attempted
+       * (shouldSaveLastAlive sets it — deliberately, as dogpile
+       * protection). If we leave it in place after a FAILED write, every
+       * request for the next 30 seconds silently skips the retry, lastAlive
+       * quietly crosses the connection-status worker's staleness threshold,
+       * and a healthy probe gets flagged Disconnected. Clear the stamp so
+       * the very next request retries the write.
+       */
+      await this.clearLastAliveThrottle(probeId);
+      throw err;
+    }
+  }
+
+  // Best-effort: on failure the throttle simply expires on its own.
+  private async clearLastAliveThrottle(probeId: ObjectID): Promise<void> {
+    try {
+      await this.boundedCacheOperation(
+        "Probe last-alive throttle clear",
+        GlobalCache.deleteKey(
+          PROBE_LAST_ALIVE_CACHE_NAMESPACE,
+          probeId.toString(),
+        ),
+      );
+    } catch (err) {
+      logger.error("Error in clearing last alive throttle", {
+        probeId: probeId?.toString(),
+      } as LogAttributes);
+      logger.error(err, { probeId: probeId?.toString() } as LogAttributes);
+    }
   }
 
   @CaptureSpan()
@@ -266,8 +548,18 @@ export class Service extends DatabaseService<Model> {
   @CaptureSpan()
   protected override async onUpdateSuccess(
     onUpdate: OnUpdate<Model>,
-    _updatedItemIds: Array<ObjectID>,
+    updatedItemIds: Array<ObjectID>,
   ): Promise<OnUpdate<Model>> {
+    /*
+     * Key rotation: the auth middleware trusts a cached hash of the key, so
+     * the moment the key column changes the cached entries for the updated
+     * probes must go — otherwise the OLD key keeps authenticating until the
+     * cache TTL runs out.
+     */
+    if (onUpdate.updateBy.data.key !== undefined) {
+      await this.invalidateProbeAuthCache(updatedItemIds);
+    }
+
     if (
       onUpdate.carryForward &&
       onUpdate.carryForward.probesToNotifyOwners.length > 0
@@ -284,6 +576,17 @@ export class Service extends DatabaseService<Model> {
     return Promise.resolve(onUpdate);
   }
 
+  @CaptureSpan()
+  protected override async onDeleteSuccess(
+    onDelete: OnDelete<Model>,
+    itemIdsBeforeDelete: Array<ObjectID>,
+  ): Promise<OnDelete<Model>> {
+    // A deleted probe's key must stop authenticating.
+    await this.invalidateProbeAuthCache(itemIdsBeforeDelete);
+
+    return Promise.resolve(onDelete);
+  }
+
   @CaptureSpan()
   public async notifyOwnersOnStatusChange(data: {
     probeId: ObjectID;

package/Server/Services/StatusPageService.ts

@@ -58,7 +58,7 @@ import StatusPageResourceService from "./StatusPageResourceService";
 import Dictionary from "../../Types/Dictionary";
 import { JSONObject } from "../../Types/JSON";
 import MonitorGroupResource from "../../Models/DatabaseModels/MonitorGroupResource";
-import MonitorGroupResourceService from "./MonitorGroupResourceService";
+import MonitorGroupService from "./MonitorGroupService";
 import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
 import IncidentService from "./IncidentService";
@@ -107,6 +107,22 @@ export class Service extends DatabaseService<StatusPage> {
     10_000,
   );
 
+  /*
+   * Caches verified custom-domain -> statusPageId resolution. Every public
+   * status-page API call served on a custom domain (overview, incidents,
+   * announcements, polling — several per page view) starts with this lookup,
+   * and the mapping only changes when a customer provisions or removes a
+   * domain (which takes minutes anyway), so a 60s staleness window is the
+   * same tradeoff `statusPageUrlCache` above already accepts. Only
+   * SUCCESSFUL resolutions are cached: misses re-query, so a freshly
+   * verified domain works immediately and attacker-controlled Host headers
+   * cannot fill the cache with negative entries.
+   */
+  private statusPageDomainToIdCache: InMemoryTTLCache<string> =
+    new InMemoryTTLCache(10_000);
+
+  private static readonly DOMAIN_TO_ID_CACHE_TTL_MS: number = 60 * 1000;
+
   public constructor() {
     super(StatusPage);
   }
@@ -119,6 +135,14 @@ export class Service extends DatabaseService<StatusPage> {
     this.statusPageUrlCache.clear();
   }
 
+  public clearStatusPageDomainToIdCache(fullDomain?: string): void {
+    if (fullDomain) {
+      this.statusPageDomainToIdCache.delete(fullDomain);
+      return;
+    }
+    this.statusPageDomainToIdCache.clear();
+  }
+
   /*
    * Mirrors `resolveStatusPageIdOrThrow` in `Common/Server/API/StatusPageAPI.ts`
    * (module-private there), but returns null instead of throwing so callers can
@@ -135,6 +159,13 @@ export class Service extends DatabaseService<StatusPage> {
     }
 
     if (statusPageIdOrDomain.includes(".")) {
+      const cachedStatusPageId: string | undefined =
+        this.statusPageDomainToIdCache.get(statusPageIdOrDomain);
+
+      if (cachedStatusPageId) {
+        return new ObjectID(cachedStatusPageId);
+      }
+
       const statusPageDomain: StatusPageDomain | null =
         await StatusPageDomainService.findOneBy({
           query: {
@@ -155,6 +186,12 @@ export class Service extends DatabaseService<StatusPage> {
         return null;
       }
 
+      this.statusPageDomainToIdCache.set(
+        statusPageIdOrDomain,
+        statusPageDomain.statusPageId.toString(),
+        Service.DOMAIN_TO_ID_CACHE_TTL_MS,
+      );
+
       return statusPageDomain.statusPageId;
     }
 
@@ -1378,12 +1415,22 @@ export class Service extends DatabaseService<StatusPage> {
   @CaptureSpan()
   public async getMonitorIdsOnStatusPage(data: {
     statusPageId: ObjectID;
+    /*
+     * Pass when the caller has already loaded the page's resources —
+     * several public endpoints fetch them right before calling this, and
+     * without this parameter the identical StatusPageResource query used to
+     * run twice per request.
+     */
+    statusPageResources?: Array<StatusPageResource> | undefined;
   }): Promise<{
     monitorsOnStatusPage: Array<ObjectID>;
     monitorsInGroup: Dictionary<Array<ObjectID>>;
   }> {
     const statusPageResources: Array<StatusPageResource> =
-      await this.getStatusPageResources(data);
+      data.statusPageResources ||
+      (await this.getStatusPageResources({
+        statusPageId: data.statusPageId,
+      }));
 
     const monitorGroupIds: Array<ObjectID> = statusPageResources
       .map((resource: StatusPageResource) => {
@@ -1404,33 +1451,15 @@ export class Service extends DatabaseService<StatusPage> {
         return Boolean(id); // remove nulls
       });
 
-    for (const monitorGroupId of monitorGroupIds) {
-      // get current status of monitors in the group.
+    // Batched: one query for all monitor groups instead of one per group.
+    const monitorIdsByGroupId: Dictionary<Array<ObjectID>> =
+      await MonitorGroupService.getMonitorIdsInMonitorGroups(monitorGroupIds);
 
+    for (const monitorGroupId of monitorGroupIds) {
       // get monitors in the group.
 
-      const groupResources: Array<MonitorGroupResource> =
-        await MonitorGroupResourceService.findBy({
-          query: {
-            monitorGroupId: monitorGroupId,
-          },
-          select: {
-            monitorId: true,
-          },
-          props: {
-            isRoot: true,
-          },
-          limit: LIMIT_PER_PROJECT,
-          skip: 0,
-        });
-
-      const monitorsInGroupIds: Array<ObjectID> = groupResources
-        .map((resource: MonitorGroupResource) => {
-          return resource.monitorId!;
-        })
-        .filter((id: ObjectID) => {
-          return Boolean(id); // remove nulls
-        });
+      const monitorsInGroupIds: Array<ObjectID> =
+        monitorIdsByGroupId[monitorGroupId.toString()] || [];
 
       for (const monitorId of monitorsInGroupIds) {
         if (
@@ -1511,25 +1540,26 @@ export class Service extends DatabaseService<StatusPage> {
   }): Promise<Dictionary<ObjectID>> {
     const monitorGroupCurrentStatuses: Dictionary<ObjectID> = {};
 
+    /*
+     * Batched: this used to issue one MonitorGroupResource query per status
+     * page resource (not even per distinct group) on every badge render and
+     * subscriber report. One query now serves all groups.
+     */
+    const resourcesByGroupId: Dictionary<Array<MonitorGroupResource>> =
+      await MonitorGroupService.getMonitorGroupResourcesByGroupIds(
+        data.statusPageResources
+          .map((resource: StatusPageResource) => {
+            return resource.monitorGroupId!;
+          })
+          .filter((id: ObjectID) => {
+            return Boolean(id); // remove nulls
+          }),
+      );
+
     for (const resource of data.statusPageResources) {
       if (resource.monitorGroupId) {
         const monitorGroupResources: Array<MonitorGroupResource> =
-          await MonitorGroupResourceService.findBy({
-            query: {
-              monitorGroupId: resource.monitorGroupId,
-            },
-            select: {
-              monitorId: true,
-              monitor: {
-                currentMonitorStatusId: true,
-              },
-            },
-            skip: 0,
-            limit: LIMIT_PER_PROJECT,
-            props: {
-              isRoot: true,
-            },
-          });
+          resourcesByGroupId[resource.monitorGroupId.toString()] || [];
 
         const statuses: Array<ObjectID> = monitorGroupResources
           .filter((item: MonitorGroupResource) => {