@oneuptime/common 12.0.2 → 12.0.3

package/Server/Services/DockerSwarmClusterService.ts

@@ -5,6 +5,7 @@ import Model from "../../Models/DatabaseModels/DockerSwarmCluster";
 import Label from "../../Models/DatabaseModels/Label";
 import { OnCreate } from "../Types/Database/Hooks";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
 import ObjectID from "../../Types/ObjectID";
 import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
@@ -182,7 +183,6 @@ export class Service extends DatabaseService<Model> {
       networkCount?: number | undefined;
     },
   ): Promise<void> {
-    const cacheKey: string = clusterId.toString();
     const extrasFingerprint: string = crypto
       .createHash("sha1")
       .update(
@@ -202,34 +202,6 @@ export class Service extends DatabaseService<Model> {
       )
       .digest("hex");
 
-    let cached: string | null = null;
-    try {
-      cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-    } catch {
-      /*
-       * Cache unavailable — fail open and refresh lastSeenAt anyway. A
-       * cache error must never skip the DB write below, otherwise the
-       * resource is wrongly marked "disconnected" while telemetry is
-       * still flowing. Mirrors shouldRunMaintenance's fail-open stance.
-       */
-      cached = null;
-    }
-
-    if (cached === extrasFingerprint) {
-      return; // same data was written recently
-    }
-
-    try {
-      await GlobalCache.setString(
-        LAST_SEEN_CACHE_NAMESPACE,
-        cacheKey,
-        extrasFingerprint,
-        { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
-      );
-    } catch {
-      // Best-effort throttle write; proceed with the DB update regardless.
-    }
-
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
@@ -237,92 +209,59 @@ export class Service extends DatabaseService<Model> {
     };
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = { ...liveness };
+    const metadata: any = {};
 
     if (extra?.dockerVersion) {
-      data.dockerVersion = extra.dockerVersion;
+      metadata.dockerVersion = extra.dockerVersion;
     }
     if (extra?.swarmId) {
-      data.swarmId = extra.swarmId;
+      metadata.swarmId = extra.swarmId;
     }
     if (extra?.agentVersion) {
-      data.agentVersion = extra.agentVersion;
+      metadata.agentVersion = extra.agentVersion;
     }
     // Counts: 0 is a legitimate value — gate on undefined, not falsiness.
     if (extra?.nodeCount !== undefined) {
-      data.nodeCount = extra.nodeCount;
+      metadata.nodeCount = extra.nodeCount;
     }
     if (extra?.readyNodeCount !== undefined) {
-      data.readyNodeCount = extra.readyNodeCount;
+      metadata.readyNodeCount = extra.readyNodeCount;
     }
     if (extra?.managerNodeCount !== undefined) {
-      data.managerNodeCount = extra.managerNodeCount;
+      metadata.managerNodeCount = extra.managerNodeCount;
     }
     if (extra?.serviceCount !== undefined) {
-      data.serviceCount = extra.serviceCount;
+      metadata.serviceCount = extra.serviceCount;
     }
     if (extra?.taskCount !== undefined) {
-      data.taskCount = extra.taskCount;
+      metadata.taskCount = extra.taskCount;
     }
     if (extra?.runningTaskCount !== undefined) {
-      data.runningTaskCount = extra.runningTaskCount;
+      metadata.runningTaskCount = extra.runningTaskCount;
     }
     if (extra?.stackCount !== undefined) {
-      data.stackCount = extra.stackCount;
+      metadata.stackCount = extra.stackCount;
     }
     if (extra?.networkCount !== undefined) {
-      data.networkCount = extra.networkCount;
+      metadata.networkCount = extra.networkCount;
     }
 
     /*
-     * Heartbeat write: a single-statement UPDATE with no hooks and no
-     * `version` bump, avoiding the hot-row Postgres lock convoy that the
-     * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
-     *
-     * Collector-supplied strings are clamped to their declared column
-     * widths inside updateColumnsByIdWithoutHooks — see
-     * DatabaseService.truncateStringColumnsToMaxLength.
+     * One gated, non-blocking heartbeat write. The gates, the fail-open /
+     * fail-closed split and the liveness-only fallback all live in
+     * ResourceHeartbeat — see there for why this row's throttle used to
+     * provide no throttling at all.
      */
-    try {
-      await this.updateColumnsByIdWithoutHooks({
-        id: clusterId,
-        data: data,
-      });
-    } catch (err) {
-      /*
-       * Liveness must survive bad metadata. The enriched write carries
-       * `lastSeenAt` and `otelCollectorStatus` alongside optional,
-       * collector-supplied columns; if any one of them makes Postgres
-       * reject the statement, lastSeenAt silently stops advancing and
-       * markDisconnectedClusters flips a perfectly healthy Docker Swarm
-       * cluster to "disconnected" 15 minutes later — with telemetry still
-       * arriving the whole time. That was issue #3006 on the Host path (a
-       * host.ip list longer than the then-varchar(500) column). Retry with
-       * liveness only so the enrichment is what gets dropped, never the
-       * heartbeat.
-       */
-      logger.warn(
-        `DockerSwarmClusterService.updateLastSeen: metadata write failed for Docker Swarm cluster ${clusterId.toString()}, falling back to a liveness-only update: ${
-          err instanceof Error ? err.message : String(err)
-        }`,
-      );
-
-      /*
-       * The throttle fingerprint was already stored, so without this the
-       * next batch would short-circuit on a cache hit and skip the retry
-       * for the rest of the window.
-       */
-      try {
-        await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-      } catch {
-        // Best-effort; the fallback write below is what actually matters.
-      }
-
-      await this.updateColumnsByIdWithoutHooks({
-        id: clusterId,
-        data: liveness,
-      });
-    }
+    await ResourceHeartbeat.write({
+      service: this,
+      id: clusterId,
+      cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
+      throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
+      liveness: liveness,
+      metadata: metadata,
+      fingerprint: extrasFingerprint,
+      describe: `docker swarm cluster ${clusterId.toString()}`,
+    });
   }
 
   /**

package/Server/Services/HostService.ts

@@ -5,6 +5,7 @@ import Model from "../../Models/DatabaseModels/Host";
 import Label from "../../Models/DatabaseModels/Label";
 import { OnCreate } from "../Types/Database/Hooks";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
 import ObjectID from "../../Types/ObjectID";
 import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
@@ -212,35 +213,7 @@ export class Service extends DatabaseService<Model> {
      * once per throttle window. If any extra value changed (e.g. cpuCores
      * updated, IP address changed), bust the cache and write immediately.
      */
-    const cacheKey: string = hostId.toString();
     const extrasFingerprint: string = this.fingerprintExtras(extra);
-    let cached: string | null = null;
-    try {
-      cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-    } catch {
-      /*
-       * Cache unavailable — fail open and refresh lastSeenAt anyway. A
-       * cache error must never skip the DB write below, otherwise the
-       * resource is wrongly marked "disconnected" while telemetry is
-       * still flowing. Mirrors shouldRunMaintenance's fail-open stance.
-       */
-      cached = null;
-    }
-
-    if (cached === extrasFingerprint) {
-      return; // same data was written recently
-    }
-
-    try {
-      await GlobalCache.setString(
-        LAST_SEEN_CACHE_NAMESPACE,
-        cacheKey,
-        extrasFingerprint,
-        { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
-      );
-    } catch {
-      // Best-effort throttle write; proceed with the DB update regardless.
-    }
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const liveness: any = {
@@ -249,118 +222,85 @@ export class Service extends DatabaseService<Model> {
     };
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = { ...liveness };
+    const metadata: any = {};
 
     if (extra?.osType) {
-      data.osType = extra.osType;
+      metadata.osType = extra.osType;
     }
     if (extra?.osVersion) {
-      data.osVersion = extra.osVersion;
+      metadata.osVersion = extra.osVersion;
     }
     if (extra?.hostId) {
-      data.hostId = extra.hostId;
+      metadata.hostId = extra.hostId;
     }
     if (extra?.hostArch) {
-      data.hostArch = extra.hostArch;
+      metadata.hostArch = extra.hostArch;
     }
     if (extra?.hostType) {
-      data.hostType = extra.hostType;
+      metadata.hostType = extra.hostType;
     }
     if (extra?.hostIpAddresses) {
-      data.hostIpAddresses = extra.hostIpAddresses;
+      metadata.hostIpAddresses = extra.hostIpAddresses;
     }
     if (extra?.cpuCores !== undefined) {
-      data.cpuCores = extra.cpuCores;
+      metadata.cpuCores = extra.cpuCores;
     }
     if (extra?.totalMemoryBytes !== undefined) {
-      data.totalMemoryBytes = extra.totalMemoryBytes;
+      metadata.totalMemoryBytes = extra.totalMemoryBytes;
     }
     if (extra?.processCount !== undefined) {
-      data.processCount = extra.processCount;
+      metadata.processCount = extra.processCount;
     }
     if (extra?.containerRuntime) {
-      data.containerRuntime = extra.containerRuntime;
+      metadata.containerRuntime = extra.containerRuntime;
     }
     if (extra?.dockerHostId) {
-      data.dockerHostId = extra.dockerHostId;
+      metadata.dockerHostId = extra.dockerHostId;
     }
     if (extra?.kubernetesClusterId) {
-      data.kubernetesClusterId = extra.kubernetesClusterId;
+      metadata.kubernetesClusterId = extra.kubernetesClusterId;
     }
     if (extra?.agentVersion) {
-      data.agentVersion = extra.agentVersion;
+      metadata.agentVersion = extra.agentVersion;
     }
     if (extra?.deploymentEnvironment) {
-      data.deploymentEnvironment = extra.deploymentEnvironment;
+      metadata.deploymentEnvironment = extra.deploymentEnvironment;
     }
     if (extra?.runtimeName) {
-      data.runtimeName = extra.runtimeName;
+      metadata.runtimeName = extra.runtimeName;
     }
     if (extra?.runtimeVersion) {
-      data.runtimeVersion = extra.runtimeVersion;
+      metadata.runtimeVersion = extra.runtimeVersion;
     }
     if (extra?.cloudProvider) {
-      data.cloudProvider = extra.cloudProvider;
+      metadata.cloudProvider = extra.cloudProvider;
     }
     if (extra?.cloudPlatform) {
-      data.cloudPlatform = extra.cloudPlatform;
+      metadata.cloudPlatform = extra.cloudPlatform;
     }
     if (extra?.cloudRegion) {
-      data.cloudRegion = extra.cloudRegion;
+      metadata.cloudRegion = extra.cloudRegion;
     }
     if (extra?.cloudAccountId) {
-      data.cloudAccountId = extra.cloudAccountId;
+      metadata.cloudAccountId = extra.cloudAccountId;
     }
 
     /*
-     * Heartbeat write: a single-statement UPDATE with no hooks and no
-     * `version` bump, avoiding the hot-row Postgres lock convoy that the
-     * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
-     *
-     * Every optional value above comes from an OpenTelemetry resource
-     * attribute, so its length is whatever the collector felt like sending.
-     * updateColumnsByIdWithoutHooks clamps them to their declared column
-     * widths for us — see truncateStringColumnsToMaxLength.
+     * One gated, non-blocking heartbeat write. The gates, the fail-open /
+     * fail-closed split and the liveness-only fallback all live in
+     * ResourceHeartbeat — see there for why this row's throttle used to
+     * provide no throttling at all.
      */
-    try {
-      await this.updateColumnsByIdWithoutHooks({
-        id: hostId,
-        data: data,
-      });
-    } catch (err) {
-      /*
-       * Liveness must survive bad metadata. The enriched write carries
-       * `lastSeenAt` and `otelCollectorStatus` alongside a dozen optional,
-       * collector-supplied columns; if any one of them makes Postgres
-       * reject the statement, the host silently stops advancing
-       * lastSeenAt and MarkDisconnectedHosts flips a perfectly healthy
-       * host to "disconnected" 15 minutes later — with telemetry still
-       * arriving the whole time. That was issue #3006 (a host.ip list
-       * longer than the then-varchar(500) column). Retry with liveness
-       * only so the enrichment is what gets dropped, never the heartbeat.
-       */
-      logger.warn(
-        `HostService.updateLastSeen: metadata write failed for host ${hostId.toString()}, falling back to a liveness-only update: ${
-          err instanceof Error ? err.message : String(err)
-        }`,
-      );
-
-      /*
-       * The throttle fingerprint was already stored, so without this the
-       * next batch would short-circuit on a cache hit and skip the retry
-       * for the rest of the window.
-       */
-      try {
-        await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-      } catch {
-        // Best-effort; the fallback write below is what actually matters.
-      }
-
-      await this.updateColumnsByIdWithoutHooks({
-        id: hostId,
-        data: liveness,
-      });
-    }
+    await ResourceHeartbeat.write({
+      service: this,
+      id: hostId,
+      cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
+      throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
+      liveness: liveness,
+      metadata: metadata,
+      fingerprint: extrasFingerprint,
+      describe: `host ${hostId.toString()}`,
+    });
   }
 
   private fingerprintExtras(extra?: {
@@ -392,9 +332,16 @@ export class Service extends DatabaseService<Model> {
       hostArch: extra?.hostArch ?? null,
       hostType: extra?.hostType ?? null,
       hostIpAddresses: extra?.hostIpAddresses ?? null,
-      cpuCores: extra?.cpuCores ?? null,
-      totalMemoryBytes: extra?.totalMemoryBytes ?? null,
-      processCount: extra?.processCount ?? null,
+      /*
+       * cpuCores / totalMemoryBytes / processCount are DELIBERATELY not
+       * hashed. They are per-scrape gauges, and `system.processes.count` in
+       * particular is partitioned by process.status and summed, so it changes
+       * on essentially every scrape. Hashing them made the throttle key differ
+       * every time and the 60s window never engaged — for exactly the hosts
+       * producing the most batches, which is the worst possible place to lose
+       * a throttle. The columns are still written; they simply ride along on
+       * whatever write the throttle admits rather than forcing one.
+       */
       containerRuntime: extra?.containerRuntime ?? null,
       dockerHostId: extra?.dockerHostId?.toString() ?? null,
       kubernetesClusterId: extra?.kubernetesClusterId?.toString() ?? null,

package/Server/Services/IncidentStateTimelineService.ts

@@ -25,6 +25,7 @@ import IncidentStateTimeline from "../../Models/DatabaseModels/IncidentStateTime
 import IncidentMember from "../../Models/DatabaseModels/IncidentMember";
 import IncidentRole from "../../Models/DatabaseModels/IncidentRole";
 import { IsBillingEnabled } from "../EnvironmentConfig";
+import ProjectScopedReferenceValidator from "../Utils/Database/ProjectScopedReferenceValidator";
 import logger, { LogAttributes } from "../Utils/Logger";
 import IncidentFeedService from "./IncidentFeedService";
 import AIIncidentPostmortemRunner from "../Utils/AI/SRE/IncidentPostmortemRunner";
@@ -128,6 +129,25 @@ export class Service extends DatabaseService<IncidentStateTimeline> {
         throw new BadDataException("incidentStateId is null");
       }
 
+      /*
+       * Same guard, same reason, as MonitorStatusTimelineService: an id that
+       * exists nowhere (or belongs to another project) used to reach Postgres
+       * and come back as a raw foreign key violation. The ordering check below
+       * does look the state up, but only when there IS a preceding row with an
+       * order, so the first timeline row for an incident went through unchecked.
+       */
+      await ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+        projectId: createBy.props.tenantId || createBy.data.projectId,
+        subject: "incident state timeline",
+        references: [
+          {
+            modelName: "Incident State",
+            id: incidentStateId,
+            service: IncidentStateService,
+          },
+        ],
+      });
+
       // Execute queries for before and after states in parallel for better performance
       const [stateBeforeThis, stateAfterThis] = await Promise.all([
         this.findOneBy({

package/Server/Services/Index.ts

@@ -177,6 +177,7 @@ import StatusPageOwnerUserService from "./StatusPageOwnerUserService";
 import StatusPagePrivateUserService from "./StatusPagePrivateUserService";
 import StatusPagePrivateUserSessionService from "./StatusPagePrivateUserSessionService";
 import StatusPageResourceService from "./StatusPageResourceService";
+import StatusPageMonitorRuleService from "./StatusPageMonitorRuleService";
 // Status Page
 import StatusPageService from "./StatusPageService";
 import StatusPageSsoService from "./StatusPageSsoService";
@@ -438,6 +439,7 @@ const services: Array<BaseService> = [
   StatusPagePrivateUserService,
   StatusPagePrivateUserSessionService,
   StatusPageResourceService,
+  StatusPageMonitorRuleService,
   StatusPageService,
   StatusPageSsoService,
   StatusPageOidcService,

package/Server/Services/IoTFleetService.ts

@@ -5,6 +5,7 @@ import Model from "../../Models/DatabaseModels/IoTFleet";
 import Label from "../../Models/DatabaseModels/Label";
 import { OnCreate } from "../Types/Database/Hooks";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
 import ObjectID from "../../Types/ObjectID";
 import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
@@ -173,7 +174,6 @@ export class Service extends DatabaseService<Model> {
       onlineDeviceCount?: number | undefined;
     },
   ): Promise<void> {
-    const cacheKey: string = fleetId.toString();
     const extrasFingerprint: string = crypto
       .createHash("sha1")
       .update(
@@ -185,34 +185,6 @@ export class Service extends DatabaseService<Model> {
       )
       .digest("hex");
 
-    let cached: string | null = null;
-    try {
-      cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-    } catch {
-      /*
-       * Cache unavailable — fail open and refresh lastSeenAt anyway. A
-       * cache error must never skip the DB write below, otherwise the
-       * resource is wrongly marked "disconnected" while telemetry is
-       * still flowing. Mirrors shouldRunMaintenance's fail-open stance.
-       */
-      cached = null;
-    }
-
-    if (cached === extrasFingerprint) {
-      return; // same data was written recently
-    }
-
-    try {
-      await GlobalCache.setString(
-        LAST_SEEN_CACHE_NAMESPACE,
-        cacheKey,
-        extrasFingerprint,
-        { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
-      );
-    } catch {
-      // Best-effort throttle write; proceed with the DB update regardless.
-    }
-
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
@@ -220,67 +192,35 @@ export class Service extends DatabaseService<Model> {
     };
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = { ...liveness };
+    const metadata: any = {};
 
     if (extra?.agentVersion) {
-      data.agentVersion = extra.agentVersion;
+      metadata.agentVersion = extra.agentVersion;
     }
     // Counts: 0 is a legitimate value — gate on undefined, not falsiness.
     if (extra?.deviceCount !== undefined) {
-      data.deviceCount = extra.deviceCount;
+      metadata.deviceCount = extra.deviceCount;
     }
     if (extra?.onlineDeviceCount !== undefined) {
-      data.onlineDeviceCount = extra.onlineDeviceCount;
+      metadata.onlineDeviceCount = extra.onlineDeviceCount;
     }
 
     /*
-     * Heartbeat write: a single-statement UPDATE with no hooks and no
-     * `version` bump, avoiding the hot-row Postgres lock convoy that the
-     * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
-     *
-     * Collector-supplied strings are clamped to their declared column
-     * widths inside updateColumnsByIdWithoutHooks — see
-     * DatabaseService.truncateStringColumnsToMaxLength.
+     * One gated, non-blocking heartbeat write. The gates, the fail-open /
+     * fail-closed split and the liveness-only fallback all live in
+     * ResourceHeartbeat — see there for why this row's throttle used to
+     * provide no throttling at all.
      */
-    try {
-      await this.updateColumnsByIdWithoutHooks({
-        id: fleetId,
-        data: data,
-      });
-    } catch (err) {
-      /*
-       * Liveness must survive bad metadata. The enriched write carries
-       * `lastSeenAt` and `otelCollectorStatus` alongside optional,
-       * collector-supplied columns; if any one of them makes Postgres
-       * reject the statement, lastSeenAt silently stops advancing and
-       * markDisconnectedFleets flips a perfectly healthy IoT fleet to
-       * "disconnected" 15 minutes later — with telemetry still arriving the
-       * whole time. That was issue #3006 on the Host path (a host.ip list
-       * longer than the then-varchar(500) column). Retry with liveness only
-       * so the enrichment is what gets dropped, never the heartbeat.
-       */
-      logger.warn(
-        `IoTFleetService.updateLastSeen: metadata write failed for IoT fleet ${fleetId.toString()}, falling back to a liveness-only update: ${
-          err instanceof Error ? err.message : String(err)
-        }`,
-      );
-
-      /*
-       * The throttle fingerprint was already stored, so without this the
-       * next batch would short-circuit on a cache hit and skip the retry
-       * for the rest of the window.
-       */
-      try {
-        await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-      } catch {
-        // Best-effort; the fallback write below is what actually matters.
-      }
-
-      await this.updateColumnsByIdWithoutHooks({
-        id: fleetId,
-        data: liveness,
-      });
-    }
+    await ResourceHeartbeat.write({
+      service: this,
+      id: fleetId,
+      cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
+      throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
+      liveness: liveness,
+      metadata: metadata,
+      fingerprint: extrasFingerprint,
+      describe: `iot fleet ${fleetId.toString()}`,
+    });
   }
 
   /**