@oneuptime/common 12.0.2 → 12.0.3

package/Server/Services/CephClusterService.ts

@@ -5,6 +5,7 @@ import Model from "../../Models/DatabaseModels/CephCluster";
 import Label from "../../Models/DatabaseModels/Label";
 import { OnCreate } from "../Types/Database/Hooks";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
 import ObjectID from "../../Types/ObjectID";
 import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
@@ -181,7 +182,6 @@ export class Service extends DatabaseService<Model> {
       capacityUsedPercent?: number | undefined;
     },
   ): Promise<void> {
-    const cacheKey: string = clusterId.toString();
     const extrasFingerprint: string = crypto
       .createHash("sha1")
       .update(
@@ -200,34 +200,6 @@ export class Service extends DatabaseService<Model> {
       )
       .digest("hex");
 
-    let cached: string | null = null;
-    try {
-      cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-    } catch {
-      /*
-       * Cache unavailable — fail open and refresh lastSeenAt anyway. A
-       * cache error must never skip the DB write below, otherwise the
-       * resource is wrongly marked "disconnected" while telemetry is
-       * still flowing. Mirrors shouldRunMaintenance's fail-open stance.
-       */
-      cached = null;
-    }
-
-    if (cached === extrasFingerprint) {
-      return; // same data was written recently
-    }
-
-    try {
-      await GlobalCache.setString(
-        LAST_SEEN_CACHE_NAMESPACE,
-        cacheKey,
-        extrasFingerprint,
-        { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
-      );
-    } catch {
-      // Best-effort throttle write; proceed with the DB update regardless.
-    }
-
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
@@ -235,91 +207,59 @@ export class Service extends DatabaseService<Model> {
     };
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = { ...liveness };
+    const metadata: any = {};
 
     if (extra?.cephVersion) {
-      data.cephVersion = extra.cephVersion;
+      metadata.cephVersion = extra.cephVersion;
     }
     if (extra?.fsid) {
-      data.fsid = extra.fsid;
+      metadata.fsid = extra.fsid;
     }
     if (extra?.agentVersion) {
-      data.agentVersion = extra.agentVersion;
+      metadata.agentVersion = extra.agentVersion;
     }
     /*
      * Counts and health: 0 is a legitimate value (healthStatus 0 = OK,
      * osdUpCount 0 = every OSD down) — gate on undefined, not falsiness.
      */
     if (extra?.monCount !== undefined) {
-      data.monCount = extra.monCount;
+      metadata.monCount = extra.monCount;
     }
     if (extra?.osdCount !== undefined) {
-      data.osdCount = extra.osdCount;
+      metadata.osdCount = extra.osdCount;
     }
     if (extra?.osdUpCount !== undefined) {
-      data.osdUpCount = extra.osdUpCount;
+      metadata.osdUpCount = extra.osdUpCount;
     }
     if (extra?.osdInCount !== undefined) {
-      data.osdInCount = extra.osdInCount;
+      metadata.osdInCount = extra.osdInCount;
     }
     if (extra?.poolCount !== undefined) {
-      data.poolCount = extra.poolCount;
+      metadata.poolCount = extra.poolCount;
     }
     if (extra?.healthStatus !== undefined) {
-      data.healthStatus = extra.healthStatus;
+      metadata.healthStatus = extra.healthStatus;
     }
     if (extra?.capacityUsedPercent !== undefined) {
-      data.capacityUsedPercent = extra.capacityUsedPercent;
+      metadata.capacityUsedPercent = extra.capacityUsedPercent;
     }
 
     /*
-     * Heartbeat write: a single-statement UPDATE with no hooks and no
-     * `version` bump, avoiding the hot-row Postgres lock convoy that the
-     * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
-     *
-     * Collector-supplied strings are clamped to their declared column
-     * widths inside updateColumnsByIdWithoutHooks — see
-     * DatabaseService.truncateStringColumnsToMaxLength.
+     * One gated, non-blocking heartbeat write. The gates, the fail-open /
+     * fail-closed split and the liveness-only fallback all live in
+     * ResourceHeartbeat — see there for why this row's throttle used to
+     * provide no throttling at all.
      */
-    try {
-      await this.updateColumnsByIdWithoutHooks({
-        id: clusterId,
-        data: data,
-      });
-    } catch (err) {
-      /*
-       * Liveness must survive bad metadata. The enriched write carries
-       * `lastSeenAt` and `otelCollectorStatus` alongside optional,
-       * collector-supplied columns; if any one of them makes Postgres
-       * reject the statement, lastSeenAt silently stops advancing and
-       * markDisconnectedClusters flips a perfectly healthy Ceph cluster to
-       * "disconnected" 15 minutes later — with telemetry still arriving the
-       * whole time. That was issue #3006 on the Host path (a host.ip list
-       * longer than the then-varchar(500) column). Retry with liveness only
-       * so the enrichment is what gets dropped, never the heartbeat.
-       */
-      logger.warn(
-        `CephClusterService.updateLastSeen: metadata write failed for Ceph cluster ${clusterId.toString()}, falling back to a liveness-only update: ${
-          err instanceof Error ? err.message : String(err)
-        }`,
-      );
-
-      /*
-       * The throttle fingerprint was already stored, so without this the
-       * next batch would short-circuit on a cache hit and skip the retry
-       * for the rest of the window.
-       */
-      try {
-        await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-      } catch {
-        // Best-effort; the fallback write below is what actually matters.
-      }
-
-      await this.updateColumnsByIdWithoutHooks({
-        id: clusterId,
-        data: liveness,
-      });
-    }
+    await ResourceHeartbeat.write({
+      service: this,
+      id: clusterId,
+      cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
+      throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
+      liveness: liveness,
+      metadata: metadata,
+      fingerprint: extrasFingerprint,
+      describe: `ceph cluster ${clusterId.toString()}`,
+    });
   }
 
   /**

package/Server/Services/CloudResourceService.ts

@@ -2,6 +2,7 @@ import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/CloudResource";
 import Label from "../../Models/DatabaseModels/Label";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
 import ObjectID from "../../Types/ObjectID";
 import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
@@ -154,7 +155,6 @@ export class Service extends DatabaseService<Model> {
       runtimeVersion?: string | undefined;
     },
   ): Promise<void> {
-    const cacheKey: string = cloudResourceId.toString();
     const extrasFingerprint: string = crypto
       .createHash("sha1")
       .update(
@@ -170,34 +170,6 @@ export class Service extends DatabaseService<Model> {
       )
       .digest("hex");
 
-    let cached: string | null = null;
-    try {
-      cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-    } catch {
-      /*
-       * Cache unavailable — fail open and refresh lastSeenAt anyway. A
-       * cache error must never skip the DB write below, otherwise the
-       * resource is wrongly marked "disconnected" while telemetry is
-       * still flowing. Mirrors shouldRunMaintenance's fail-open stance.
-       */
-      cached = null;
-    }
-
-    if (cached === extrasFingerprint) {
-      return; // same data was written recently
-    }
-
-    try {
-      await GlobalCache.setString(
-        LAST_SEEN_CACHE_NAMESPACE,
-        cacheKey,
-        extrasFingerprint,
-        { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
-      );
-    } catch {
-      // Best-effort throttle write; proceed with the DB update regardless.
-    }
-
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
@@ -205,78 +177,46 @@ export class Service extends DatabaseService<Model> {
     };
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = { ...liveness };
+    const metadata: any = {};
 
     if (extra?.agentVersion) {
-      data.agentVersion = extra.agentVersion;
+      metadata.agentVersion = extra.agentVersion;
     }
     if (extra?.cloudPlatform) {
-      data.cloudPlatform = extra.cloudPlatform;
+      metadata.cloudPlatform = extra.cloudPlatform;
     }
     if (extra?.cloudProvider) {
-      data.cloudProvider = extra.cloudProvider;
+      metadata.cloudProvider = extra.cloudProvider;
     }
     if (extra?.cloudRegion) {
-      data.cloudRegion = extra.cloudRegion;
+      metadata.cloudRegion = extra.cloudRegion;
     }
     if (extra?.cloudAccountId) {
-      data.cloudAccountId = extra.cloudAccountId;
+      metadata.cloudAccountId = extra.cloudAccountId;
     }
     if (extra?.runtimeName) {
-      data.runtimeName = extra.runtimeName;
+      metadata.runtimeName = extra.runtimeName;
     }
     if (extra?.runtimeVersion) {
-      data.runtimeVersion = extra.runtimeVersion;
+      metadata.runtimeVersion = extra.runtimeVersion;
     }
 
     /*
-     * Heartbeat write: a single-statement UPDATE with no hooks and no
-     * `version` bump, avoiding the hot-row Postgres lock convoy that the
-     * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
-     *
-     * Collector-supplied strings are clamped to their declared column
-     * widths inside updateColumnsByIdWithoutHooks — see
-     * DatabaseService.truncateStringColumnsToMaxLength.
+     * One gated, non-blocking heartbeat write. The gates, the fail-open /
+     * fail-closed split and the liveness-only fallback all live in
+     * ResourceHeartbeat — see there for why this row's throttle used to
+     * provide no throttling at all.
      */
-    try {
-      await this.updateColumnsByIdWithoutHooks({
-        id: cloudResourceId,
-        data: data,
-      });
-    } catch (err) {
-      /*
-       * Liveness must survive bad metadata. The enriched write carries
-       * `lastSeenAt` and `otelCollectorStatus` alongside optional,
-       * collector-supplied columns; if any one of them makes Postgres
-       * reject the statement, lastSeenAt silently stops advancing and
-       * markDisconnectedResources flips a perfectly healthy cloud resource
-       * to "disconnected" 15 minutes later — with telemetry still arriving
-       * the whole time. That was issue #3006 on the Host path (a host.ip
-       * list longer than the then-varchar(500) column). Retry with liveness
-       * only so the enrichment is what gets dropped, never the heartbeat.
-       */
-      logger.warn(
-        `CloudResourceService.updateLastSeen: metadata write failed for cloud resource ${cloudResourceId.toString()}, falling back to a liveness-only update: ${
-          err instanceof Error ? err.message : String(err)
-        }`,
-      );
-
-      /*
-       * The throttle fingerprint was already stored, so without this the
-       * next batch would short-circuit on a cache hit and skip the retry
-       * for the rest of the window.
-       */
-      try {
-        await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-      } catch {
-        // Best-effort; the fallback write below is what actually matters.
-      }
-
-      await this.updateColumnsByIdWithoutHooks({
-        id: cloudResourceId,
-        data: liveness,
-      });
-    }
+    await ResourceHeartbeat.write({
+      service: this,
+      id: cloudResourceId,
+      cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
+      throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
+      liveness: liveness,
+      metadata: metadata,
+      fingerprint: extrasFingerprint,
+      describe: `cloud resource ${cloudResourceId.toString()}`,
+    });
   }
 
   @CaptureSpan()

package/Server/Services/DatabaseService.ts

@@ -2400,6 +2400,117 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
     await repository.manager.query(sql, params);
   }
 
+  /*
+   * Same single-statement, hook-free, no-version-bump write as
+   * `updateColumnsByIdWithoutHooks`, except it YIELDS instead of queueing:
+   * if another transaction already holds the row's lock, this write is
+   * abandoned and the method returns false. It never waits.
+   *
+   * Why this exists. A single-statement UPDATE holds its row lock for a very
+   * short time, but "short" does not mean "safe" once the arrival rate on ONE
+   * row exceeds the service rate. Postgres queues lock waiters strictly, so
+   * every waiter pins a backend for the sum of everyone ahead of it — the
+   * classic convoy. In the outage this was written for, ~25K concurrent
+   * ingest jobs across 100 worker pods funnelled onto ~2,300 Service rows and
+   * produced 1,017 active connections with 892 of them parked on row locks;
+   * the tail had been waiting 3.7 hours. Nothing about that is fixable by
+   * making the statement faster, and no Postgres tier changes it: the queue
+   * is the bug.
+   *
+   * `FOR UPDATE SKIP LOCKED` inverts it. A contended writer does zero work
+   * and returns immediately, so the number of backends blocked on any single
+   * row is capped at ONE no matter how many workers arrive together. The
+   * convoy cannot form — and cannot form even if every upstream throttle
+   * fails open at once (Redis down, fence bug, cache flush). That is the
+   * point: this is the structural backstop underneath the throttles, not a
+   * second copy of them.
+   *
+   * Use ONLY where losing the write is harmless because a concurrent writer
+   * is writing the same thing — liveness timestamps and other idempotent
+   * bookkeeping, whose contended value is by construction the value the
+   * winner is already storing. It is the WRONG primitive for a write carrying
+   * information the winner does not have (a changed attribute); for those the
+   * caller must re-attempt, and `false` is the signal to do it. Returning a
+   * boolean rather than swallowing the skip is deliberate: a silent drop here
+   * would be indistinguishable from a successful write at the call site,
+   * which is exactly how a "reliable" heartbeat quietly stops beating.
+   *
+   * Returns true when the row was updated, false when it was locked (or no
+   * longer exists).
+   */
+  @CaptureSpan()
+  public async updateColumnsByIdIfUnlockedWithoutHooks(input: {
+    id: ObjectID;
+    data: PartialEntity<TBaseModel>;
+    skipUpdateDateColumn?: boolean;
+  }): Promise<boolean> {
+    if (!input.id) {
+      throw new BadDataException("id is required");
+    }
+
+    const repository: Repository<TBaseModel> = this.getRepository();
+    const metadata: EntityMetadata = repository.metadata;
+    const driver: Driver = repository.manager.connection.driver;
+
+    const setClauses: Array<string> = [];
+    const params: Array<unknown> = [];
+
+    // Clamp on a shallow COPY — see updateColumnsByIdWithoutHooks.
+    const data: ObjectLiteral = this.truncateStringColumnsToMaxLength({
+      ...(input.data as ObjectLiteral),
+    });
+
+    for (const [propertyName, value] of Object.entries(data)) {
+      const column: ColumnMetadata | undefined =
+        metadata.findColumnWithPropertyName(propertyName);
+      if (!column) {
+        throw new BadDataException(
+          `updateColumnsByIdIfUnlockedWithoutHooks: unknown column "${propertyName}" on "${metadata.tableName}"`,
+        );
+      }
+      if (typeof value === "function") {
+        throw new BadDataException(
+          `updateColumnsByIdIfUnlockedWithoutHooks: SQL-expression values are not supported (column "${propertyName}"); pass a literal value.`,
+        );
+      }
+      params.push(driver.preparePersistentValue(value, column));
+      setClauses.push(`"${column.databaseName}" = $${params.length}`);
+    }
+
+    if (setClauses.length === 0) {
+      return false;
+    }
+
+    if (metadata.updateDateColumn && !input.skipUpdateDateColumn) {
+      setClauses.push(
+        `"${metadata.updateDateColumn.databaseName}" = CURRENT_TIMESTAMP`,
+      );
+    }
+
+    const primaryColumnName: string =
+      metadata.primaryColumns[0]?.databaseName || "_id";
+    params.push(input.id.toString());
+
+    /*
+     * The lock is taken by the sub-SELECT and inherited by the UPDATE within
+     * the same implicit transaction, so the outer write never blocks either.
+     * RETURNING is how the affected-row count comes back: TypeORM's `query()`
+     * surfaces the driver's rows and a bare UPDATE returns none, so without it
+     * a skipped write is indistinguishable from an applied one.
+     */
+    const sql: string = `UPDATE "${metadata.tableName}" SET ${setClauses.join(
+      ", ",
+    )} WHERE "${primaryColumnName}" IN (SELECT "${primaryColumnName}" FROM "${
+      metadata.tableName
+    }" WHERE "${primaryColumnName}" = $${
+      params.length
+    } FOR UPDATE SKIP LOCKED) RETURNING "${primaryColumnName}"`;
+
+    const result: unknown = await repository.manager.query(sql, params);
+
+    return Array.isArray(result) && result.length > 0;
+  }
+
   /*
    * Atomically add to numeric columns and set literal columns for one row,
    * in a SINGLE auto-committed UPDATE, with no hooks and no `version` bump.

package/Server/Services/DockerHostService.ts

@@ -5,6 +5,7 @@ import Model from "../../Models/DatabaseModels/DockerHost";
 import Label from "../../Models/DatabaseModels/Label";
 import { OnCreate } from "../Types/Database/Hooks";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
 import ObjectID from "../../Types/ObjectID";
 import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
@@ -180,7 +181,6 @@ export class Service extends DatabaseService<Model> {
       agentVersion?: string | undefined;
     },
   ): Promise<void> {
-    const cacheKey: string = hostId.toString();
     const extrasFingerprint: string = crypto
       .createHash("sha1")
       .update(
@@ -192,34 +192,6 @@ export class Service extends DatabaseService<Model> {
       )
       .digest("hex");
 
-    let cached: string | null = null;
-    try {
-      cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-    } catch {
-      /*
-       * Cache unavailable — fail open and refresh lastSeenAt anyway. A
-       * cache error must never skip the DB write below, otherwise the
-       * resource is wrongly marked "disconnected" while telemetry is
-       * still flowing. Mirrors shouldRunMaintenance's fail-open stance.
-       */
-      cached = null;
-    }
-
-    if (cached === extrasFingerprint) {
-      return; // same data was written recently
-    }
-
-    try {
-      await GlobalCache.setString(
-        LAST_SEEN_CACHE_NAMESPACE,
-        cacheKey,
-        extrasFingerprint,
-        { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
-      );
-    } catch {
-      // Best-effort throttle write; proceed with the DB update regardless.
-    }
-
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
@@ -227,66 +199,34 @@ export class Service extends DatabaseService<Model> {
     };
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = { ...liveness };
+    const metadata: any = {};
 
     if (extra?.osType) {
-      data.osType = extra.osType;
+      metadata.osType = extra.osType;
     }
     if (extra?.osVersion) {
-      data.osVersion = extra.osVersion;
+      metadata.osVersion = extra.osVersion;
     }
     if (extra?.agentVersion) {
-      data.agentVersion = extra.agentVersion;
+      metadata.agentVersion = extra.agentVersion;
     }
 
     /*
-     * Heartbeat write: a single-statement UPDATE with no hooks and no
-     * `version` bump, avoiding the hot-row Postgres lock convoy that the
-     * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
-     *
-     * Collector-supplied strings are clamped to their declared column
-     * widths inside updateColumnsByIdWithoutHooks — see
-     * DatabaseService.truncateStringColumnsToMaxLength.
+     * One gated, non-blocking heartbeat write. The gates, the fail-open /
+     * fail-closed split and the liveness-only fallback all live in
+     * ResourceHeartbeat — see there for why this row's throttle used to
+     * provide no throttling at all.
      */
-    try {
-      await this.updateColumnsByIdWithoutHooks({
-        id: hostId,
-        data: data,
-      });
-    } catch (err) {
-      /*
-       * Liveness must survive bad metadata. The enriched write carries
-       * `lastSeenAt` and `otelCollectorStatus` alongside optional,
-       * collector-supplied columns; if any one of them makes Postgres
-       * reject the statement, lastSeenAt silently stops advancing and
-       * markDisconnectedHosts flips a perfectly healthy Docker host to
-       * "disconnected" 15 minutes later — with telemetry still arriving the
-       * whole time. That was issue #3006 on the Host path (a host.ip list
-       * longer than the then-varchar(500) column). Retry with liveness only
-       * so the enrichment is what gets dropped, never the heartbeat.
-       */
-      logger.warn(
-        `DockerHostService.updateLastSeen: metadata write failed for Docker host ${hostId.toString()}, falling back to a liveness-only update: ${
-          err instanceof Error ? err.message : String(err)
-        }`,
-      );
-
-      /*
-       * The throttle fingerprint was already stored, so without this the
-       * next batch would short-circuit on a cache hit and skip the retry
-       * for the rest of the window.
-       */
-      try {
-        await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-      } catch {
-        // Best-effort; the fallback write below is what actually matters.
-      }
-
-      await this.updateColumnsByIdWithoutHooks({
-        id: hostId,
-        data: liveness,
-      });
-    }
+    await ResourceHeartbeat.write({
+      service: this,
+      id: hostId,
+      cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
+      throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
+      liveness: liveness,
+      metadata: metadata,
+      fingerprint: extrasFingerprint,
+      describe: `docker host ${hostId.toString()}`,
+    });
   }
 
   /**