@oneuptime/common 12.0.2 → 12.0.3

package/Server/Services/KubernetesClusterService.ts

@@ -5,6 +5,7 @@ import Model from "../../Models/DatabaseModels/KubernetesCluster";
 import Label from "../../Models/DatabaseModels/Label";
 import { OnCreate } from "../Types/Database/Hooks";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
 import ObjectID from "../../Types/ObjectID";
 import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
@@ -145,7 +146,6 @@ export class Service extends DatabaseService<Model> {
       agentVersion?: string | undefined;
     },
   ): Promise<void> {
-    const cacheKey: string = clusterId.toString();
     const extrasFingerprint: string = crypto
       .createHash("sha1")
       .update(
@@ -155,34 +155,6 @@ export class Service extends DatabaseService<Model> {
       )
       .digest("hex");
 
-    let cached: string | null = null;
-    try {
-      cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-    } catch {
-      /*
-       * Cache unavailable — fail open and refresh lastSeenAt anyway. A
-       * cache error must never skip the DB write below, otherwise the
-       * resource is wrongly marked "disconnected" while telemetry is
-       * still flowing. Mirrors shouldRunMaintenance's fail-open stance.
-       */
-      cached = null;
-    }
-
-    if (cached === extrasFingerprint) {
-      return; // same data was written recently
-    }
-
-    try {
-      await GlobalCache.setString(
-        LAST_SEEN_CACHE_NAMESPACE,
-        cacheKey,
-        extrasFingerprint,
-        { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
-      );
-    } catch {
-      // Best-effort throttle write; proceed with the DB update regardless.
-    }
-
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
@@ -190,61 +162,28 @@ export class Service extends DatabaseService<Model> {
     };
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = { ...liveness };
+    const metadata: any = {};
 
     if (extra?.agentVersion) {
-      data.agentVersion = extra.agentVersion;
+      metadata.agentVersion = extra.agentVersion;
     }
 
     /*
-     * Heartbeat write: a single-statement UPDATE with no hooks and no
-     * `version` bump, avoiding the hot-row Postgres lock convoy that the
-     * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
-     *
-     * Collector-supplied strings are clamped to their declared column
-     * widths inside updateColumnsByIdWithoutHooks — see
-     * DatabaseService.truncateStringColumnsToMaxLength.
+     * One gated, non-blocking heartbeat write. The gates, the fail-open /
+     * fail-closed split and the liveness-only fallback all live in
+     * ResourceHeartbeat — see there for why this row's throttle used to
+     * provide no throttling at all.
      */
-    try {
-      await this.updateColumnsByIdWithoutHooks({
-        id: clusterId,
-        data: data,
-      });
-    } catch (err) {
-      /*
-       * Liveness must survive bad metadata. The enriched write carries
-       * `lastSeenAt` and `otelCollectorStatus` alongside optional,
-       * collector-supplied columns; if any one of them makes Postgres
-       * reject the statement, lastSeenAt silently stops advancing and
-       * markDisconnectedClusters flips a perfectly healthy Kubernetes
-       * cluster to "disconnected" 15 minutes later — with telemetry still
-       * arriving the whole time. That was issue #3006 on the Host path (a
-       * host.ip list longer than the then-varchar(500) column). Retry with
-       * liveness only so the enrichment is what gets dropped, never the
-       * heartbeat.
-       */
-      logger.warn(
-        `KubernetesClusterService.updateLastSeen: metadata write failed for Kubernetes cluster ${clusterId.toString()}, falling back to a liveness-only update: ${
-          err instanceof Error ? err.message : String(err)
-        }`,
-      );
-
-      /*
-       * The throttle fingerprint was already stored, so without this the
-       * next batch would short-circuit on a cache hit and skip the retry
-       * for the rest of the window.
-       */
-      try {
-        await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-      } catch {
-        // Best-effort; the fallback write below is what actually matters.
-      }
-
-      await this.updateColumnsByIdWithoutHooks({
-        id: clusterId,
-        data: liveness,
-      });
-    }
+    await ResourceHeartbeat.write({
+      service: this,
+      id: clusterId,
+      cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
+      throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
+      liveness: liveness,
+      metadata: metadata,
+      fingerprint: extrasFingerprint,
+      describe: `kubernetes cluster ${clusterId.toString()}`,
+    });
   }
 
   /**

package/Server/Services/LabelService.ts

@@ -5,7 +5,10 @@ import QueryHelper from "../Types/Database/QueryHelper";
 import DatabaseService from "./DatabaseService";
 import ServiceLevelObjectiveMonitorRuleEngineService from "./ServiceLevelObjectiveMonitorRuleEngineService";
 import ServiceLevelObjectiveService from "./ServiceLevelObjectiveService";
+import StatusPageMonitorRuleEngineService from "./StatusPageMonitorRuleEngineService";
+import StatusPageMonitorRuleService from "./StatusPageMonitorRuleService";
 import ServiceLevelObjective from "../../Models/DatabaseModels/ServiceLevelObjective";
+import StatusPageMonitorRule from "../../Models/DatabaseModels/StatusPageMonitorRule";
 import BadDataException from "../../Types/Exception/BadDataException";
 import LIMIT_MAX from "../../Types/Database/LimitMax";
 import ObjectID from "../../Types/ObjectID";
@@ -65,16 +68,17 @@ export class Service extends DatabaseService<Model> {
 
   /*
    * Deleting a label cascades its join rows away at the database level, so no
-   * service hook ever fires for the SLO label rules that referenced it. Those
-   * SLOs would keep the monitors the rule attached, with nothing left to
-   * explain why. Note the ids down while the label still exists, and re-run
-   * the (now smaller) rules once it is gone.
+   * service hook ever fires for the SLO label rules or status page monitor
+   * rules that referenced it. Those would keep the monitors the rule attached,
+   * with nothing left to explain why. Note the ids down while the label still
+   * exists, and re-run the (now smaller) rules once it is gone.
    */
   @CaptureSpan()
   protected override async onBeforeDelete(
     deleteBy: DeleteBy<Model>,
   ): Promise<OnDelete<Model>> {
     let serviceLevelObjectiveIds: Array<ObjectID> = [];
+    let statusPageMonitorRuleIds: Array<ObjectID> = [];
 
     try {
       const labels: Array<Model> = await this.findBy({
@@ -120,10 +124,33 @@ export class Service extends DatabaseService<Model> {
           .filter((id: ObjectID | null): id is ObjectID => {
             return Boolean(id);
           });
+
+        const statusPageMonitorRules: Array<StatusPageMonitorRule> =
+          await StatusPageMonitorRuleService.findBy({
+            query: {
+              monitorLabels: labelIds,
+            },
+            select: {
+              _id: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: {
+              isRoot: true,
+            },
+          });
+
+        statusPageMonitorRuleIds = statusPageMonitorRules
+          .map((rule: StatusPageMonitorRule) => {
+            return rule.id;
+          })
+          .filter((id: ObjectID | null): id is ObjectID => {
+            return Boolean(id);
+          });
       }
     } catch (err) {
       logger.error(
-        `Error collecting SLO label rules affected by a label delete: ${err}`,
+        `Error collecting label rules affected by a label delete: ${err}`,
       );
     }
 
@@ -131,6 +158,7 @@ export class Service extends DatabaseService<Model> {
       deleteBy,
       carryForward: {
         serviceLevelObjectiveIds: serviceLevelObjectiveIds,
+        statusPageMonitorRuleIds: statusPageMonitorRuleIds,
       },
     };
   }
@@ -140,12 +168,16 @@ export class Service extends DatabaseService<Model> {
     onDelete: OnDelete<Model>,
     _itemIdsBeforeDelete: Array<ObjectID>,
   ): Promise<OnDelete<Model>> {
+    const carryForward: {
+      serviceLevelObjectiveIds?: Array<ObjectID>;
+      statusPageMonitorRuleIds?: Array<ObjectID>;
+    } | null = onDelete.carryForward as {
+      serviceLevelObjectiveIds?: Array<ObjectID>;
+      statusPageMonitorRuleIds?: Array<ObjectID>;
+    } | null;
+
     const serviceLevelObjectiveIds: Array<ObjectID> =
-      (
-        onDelete.carryForward as {
-          serviceLevelObjectiveIds?: Array<ObjectID>;
-        } | null
-      )?.serviceLevelObjectiveIds || [];
+      carryForward?.serviceLevelObjectiveIds || [];
 
     for (const serviceLevelObjectiveId of serviceLevelObjectiveIds) {
       try {
@@ -159,6 +191,21 @@ export class Service extends DatabaseService<Model> {
       }
     }
 
+    const statusPageMonitorRuleIds: Array<ObjectID> =
+      carryForward?.statusPageMonitorRuleIds || [];
+
+    for (const statusPageMonitorRuleId of statusPageMonitorRuleIds) {
+      try {
+        await StatusPageMonitorRuleEngineService.syncResourcesForRule({
+          statusPageMonitorRuleId: statusPageMonitorRuleId,
+        });
+      } catch (err) {
+        logger.error(
+          `Error re-applying status page monitor rule ${statusPageMonitorRuleId.toString()} after a label delete: ${err}`,
+        );
+      }
+    }
+
     return onDelete;
   }
 

package/Server/Services/MetricTypeService.ts

@@ -1,10 +1,94 @@
 import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/MetricType";
+import ObjectID from "../../Types/ObjectID";
+import BadDataException from "../../Types/Exception/BadDataException";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import { RelationMetadata } from "typeorm/metadata/RelationMetadata";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
   }
+
+  /*
+   * Additively associate services with a metric type, in ONE statement.
+   *
+   * This replaces routing the association through `updateOneById` with the
+   * whole `services` array in the payload. That looked innocuous and was not:
+   * `services` is a `TableColumnType.EntityArray`, and its mere PRESENCE as a
+   * key flips `hasRelationUpdates` in DatabaseService, which routes the write
+   * to `getRepository().save()` — a real BEGIN/COMMIT that reloads the entity,
+   * loads the relation ids, bumps `version`, and DELETEs then re-INSERTs
+   * junction rows, holding the MetricType row's write lock across every one of
+   * those round trips. On the ingest path, where this runs per metric name per
+   * batch with no backpressure, it was the only multi-round-trip lock hold in
+   * the pipeline.
+   *
+   * The whole-array write was also silently LOSING data. `save()` diffs the
+   * array it is given against what is currently in the database and deletes
+   * anything missing — but each ingest worker only knows the services in ITS
+   * batch. Two workers with different batches therefore deleted each other's
+   * associations and re-inserted them on the next batch: permanent junction
+   * churn, and real service-to-metric links disappearing from the UI in
+   * between. Additive insert makes that impossible to express.
+   *
+   * ON CONFLICT DO NOTHING is well-defined here: the junction's primary key is
+   * exactly (metricTypeId, serviceId), so a concurrent writer adding the same
+   * association is a no-op rather than a unique violation.
+   */
+  @CaptureSpan()
+  public async attachServices(data: {
+    metricTypeId: ObjectID;
+    serviceIds: Array<ObjectID>;
+  }): Promise<void> {
+    if (!data.metricTypeId) {
+      throw new BadDataException("metricTypeId is required");
+    }
+
+    if (!data.serviceIds || data.serviceIds.length === 0) {
+      return;
+    }
+
+    /*
+     * Identifiers come from entity metadata, never from the caller, and every
+     * value is bound as a parameter — the same rule the other raw-SQL paths in
+     * DatabaseService follow.
+     */
+    const relation: RelationMetadata | undefined =
+      this.getRepository().metadata.findRelationWithPropertyPath("services");
+
+    const junction: string | undefined =
+      relation?.junctionEntityMetadata?.tableName;
+    const metricTypeColumn: string | undefined =
+      relation?.junctionEntityMetadata?.columns[0]?.databaseName;
+    const serviceColumn: string | undefined =
+      relation?.junctionEntityMetadata?.columns[1]?.databaseName;
+
+    if (!junction || !metricTypeColumn || !serviceColumn) {
+      throw new BadDataException(
+        "MetricTypeService.attachServices: the services relation has no junction metadata",
+      );
+    }
+
+    /*
+     * Deduplicate and SORT. Concurrent statements that insert overlapping sets
+     * acquire their row locks in the same order this way, which is what keeps
+     * two workers associating the same services from deadlocking each other.
+     */
+    const serviceIds: Array<string> = Array.from(
+      new Set(
+        data.serviceIds.map((id: ObjectID) => {
+          return id.toString();
+        }),
+      ),
+    ).sort();
+
+    await this.getRepository().manager.query(
+      `INSERT INTO "${junction}" ("${metricTypeColumn}", "${serviceColumn}") ` +
+        `SELECT $1, unnest($2::uuid[]) ON CONFLICT DO NOTHING`,
+      [data.metricTypeId.toString(), serviceIds],
+    );
+  }
 }
 
 export default new Service();

package/Server/Services/MonitorService.ts

@@ -15,6 +15,7 @@ import MonitorOwnerUserService from "./MonitorOwnerUserService";
 import MonitorProbeService from "./MonitorProbeService";
 import MonitorStatusService from "./MonitorStatusService";
 import ServiceLevelObjectiveMonitorRuleEngineService from "./ServiceLevelObjectiveMonitorRuleEngineService";
+import StatusPageMonitorRuleEngineService from "./StatusPageMonitorRuleEngineService";
 import NetworkSiteService from "./NetworkSiteService";
 import MonitorStatusTimelineService, {
   MONITOR_STATUS_SAME_AS_PREVIOUS_ERROR_MESSAGE,
@@ -404,27 +405,47 @@ export class Service extends DatabaseService<Model> {
       resolveReferenceId(updateBy.data.currentMonitorStatusId) ||
       resolveReferenceId(updateBy.data.currentMonitorStatus);
 
-    if (updateBy.data.monitorSteps || currentMonitorStatusId) {
+    if (updateBy.data.monitorSteps) {
       /*
-       * Root/API updates do not always carry a tenantId, so fall back to the
-       * project of each monitor the query actually matches.
+       * Validated per matched monitor rather than per distinct project, because
+       * the check needs that monitor's CURRENT monitorSteps: a reference id it
+       * already holds is exempt from the existence check, so an update never
+       * refuses a monitor that was stored broken before the guard existed. See
+       * MonitorStepsProjectValidator.
        */
+      const monitors: Array<Model> = await this.findBy({
+        query: updateBy.query,
+        select: {
+          projectId: true,
+          monitorSteps: true,
+        },
+        limit: LIMIT_MAX,
+        skip: 0,
+        props: {
+          isRoot: true,
+          ignoreHooks: true,
+        },
+      });
+
+      for (const monitor of monitors) {
+        await MonitorStepsProjectValidator.validateMonitorStepsBelongToProject({
+          monitorSteps: updateBy.data.monitorSteps as MonitorSteps | JSONObject,
+          /*
+           * Root/API updates do not always carry a tenantId, so fall back to
+           * the project of the monitor being updated.
+           */
+          projectId: updateBy.props.tenantId || monitor.projectId,
+          alreadyStoredMonitorSteps: monitor.monitorSteps,
+        });
+      }
+    }
+
+    if (currentMonitorStatusId) {
       const projectIds: Array<ObjectID> = updateBy.props.tenantId
         ? [updateBy.props.tenantId]
         : await this.getProjectIdsForUpdateQuery(updateBy);
 
       for (const projectId of projectIds) {
-        if (updateBy.data.monitorSteps) {
-          await MonitorStepsProjectValidator.validateMonitorStepsBelongToProject(
-            {
-              monitorSteps: updateBy.data.monitorSteps as
-                | MonitorSteps
-                | JSONObject,
-              projectId: projectId,
-            },
-          );
-        }
-
         await ProjectScopedReferenceValidator.validateReferencesBelongToProject(
           {
             projectId: projectId,
@@ -644,6 +665,37 @@ export class Service extends DatabaseService<Model> {
       }
     }
 
+    /*
+     * Status page monitor rules match on labels, name and description, so an
+     * edit to any of the three can pull this monitor onto a status page or
+     * push it off one. Keyed on `!== undefined` for the same reason as the SLO
+     * block above: clearing every label arrives as `[]`, and that is exactly
+     * the edit that should detach the monitor from every label-driven rule.
+     */
+    if (
+      (onUpdate.updateBy.data.labels !== undefined ||
+        onUpdate.updateBy.data.name !== undefined ||
+        onUpdate.updateBy.data.description !== undefined) &&
+      updatedItemIds.length > 0
+    ) {
+      for (const monitorId of updatedItemIds) {
+        try {
+          await StatusPageMonitorRuleEngineService.syncRulesForMonitor({
+            monitorId: monitorId,
+            projectId: onUpdate.updateBy.props.tenantId as ObjectID,
+          });
+        } catch (error) {
+          logger.error(
+            "Syncing status page monitor rules failed in MonitorService.onUpdateSuccess",
+            {
+              monitorId: monitorId?.toString(),
+            } as LogAttributes,
+          );
+          logger.error(error as Error);
+        }
+      }
+    }
+
     return onUpdate;
   }
 
@@ -1040,6 +1092,29 @@ ${createdItem.description?.trim() || "No description provided."}
         }
         return Promise.resolve();
       })
+      .then(async () => {
+        /*
+         * Also runs after the label rules above, so a monitor that only earns
+         * its labels from a MonitorLabelRule still lands on the status pages
+         * those labels imply.
+         */
+        try {
+          await StatusPageMonitorRuleEngineService.syncRulesForMonitor({
+            monitorId: createdItem.id!,
+            projectId: createdItem.projectId!,
+          });
+        } catch (error) {
+          logger.error(
+            "Syncing status page monitor rules failed in MonitorService.onCreateSuccess",
+            {
+              projectId: createdItem.projectId?.toString(),
+              monitorId: createdItem.id?.toString(),
+            } as LogAttributes,
+          );
+          logger.error(error as Error);
+        }
+        return Promise.resolve();
+      })
       .then(async () => {
         try {
           return await this.refreshMonitorProbeStatus(createdItem.id!);

package/Server/Services/MonitorStatusTimelineService.ts

@@ -4,6 +4,7 @@ import DeleteBy from "../Types/Database/DeleteBy";
 import { OnCreate, OnDelete } from "../Types/Database/Hooks";
 import QueryHelper from "../Types/Database/QueryHelper";
 import logger, { LogAttributes } from "../Utils/Logger";
+import ProjectScopedReferenceValidator from "../Utils/Database/ProjectScopedReferenceValidator";
 import DatabaseService from "./DatabaseService";
 import MonitorService from "./MonitorService";
 import UserService from "./UserService";
@@ -191,6 +192,34 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
       throw new BadDataException("monitorStatusId is null");
     }
 
+    /*
+     * Every writer of this table funnels through here, so this is the one place
+     * that can turn issue #3039's failure mode into an answer. A monitorStatusId
+     * that does not exist (or belongs to another project) used to travel all the
+     * way to Postgres and come back as
+     *   insert or update on table "MonitorStatusTimeline" violates foreign key
+     *   constraint "FK_574feb4161c5216c2c7ee0faaf8"
+     * — an opaque 500 over the API, and a job that fails and retries forever in
+     * the probe worker. Reject it here instead, naming the id.
+     *
+     * The probe/telemetry ingest callers screen the id themselves before they
+     * get here (Utils/Monitor/MonitorStatusTimeline and Utils/Monitor/MonitorResource)
+     * so a monitor whose stored criteria already point at a deleted status skip
+     * the status change rather than failing their whole run. This check is the
+     * backstop for everything else, chiefly direct API writes.
+     */
+    await ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+      projectId: createBy.props.tenantId || createBy.data.projectId,
+      subject: "monitor status timeline",
+      references: [
+        {
+          modelName: "Monitor Status",
+          id: monitorStatusId,
+          service: MonitorStatusService,
+        },
+      ],
+    });
+
     const stateBeforeThis: MonitorStatusTimeline | null = await this.findOneBy({
       query: {
         monitorId: monitorId,

package/Server/Services/MonitorTemplateService.ts

@@ -1,8 +1,14 @@
 import DatabaseService from "./DatabaseService";
 import MonitorService from "./MonitorService";
+import CreateBy from "../Types/Database/CreateBy";
+import UpdateBy from "../Types/Database/UpdateBy";
+import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
+import MonitorStepsProjectValidator from "../Utils/Monitor/MonitorStepsProjectValidator";
 import DatabaseCommonInteractionProps from "../../Types/BaseDatabase/DatabaseCommonInteractionProps";
 import BadDataException from "../../Types/Exception/BadDataException";
 import LIMIT_MAX from "../../Types/Database/LimitMax";
+import { JSONObject } from "../../Types/JSON";
+import MonitorSteps from "../../Types/Monitor/MonitorSteps";
 import ObjectID from "../../Types/ObjectID";
 import PositiveNumber from "../../Types/PositiveNumber";
 import Model from "../../Models/DatabaseModels/MonitorTemplate";
@@ -37,6 +43,65 @@ export class Service extends DatabaseService<Model> {
     super(Model);
   }
 
+  /*
+   * A template's monitorSteps embeds the same reference ids a monitor's does,
+   * and every one of them reaches a real monitor eventually — through "create
+   * monitor from template" and through syncLinkedMonitors, which pushes the
+   * blob onto every linked monitor. Validating it here is what makes the error
+   * land on the template the bad id was typed into, rather than on a monitor
+   * sync days later. Without this the template is the one place a dangling id
+   * can still enter the system.
+   */
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    await MonitorStepsProjectValidator.validateMonitorStepsBelongToProject({
+      monitorSteps: createBy.data.monitorSteps,
+      projectId: createBy.props.tenantId || createBy.data.projectId,
+    });
+
+    return { createBy, carryForward: null };
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<Model>,
+  ): Promise<OnUpdate<Model>> {
+    if (!updateBy.data.monitorSteps) {
+      return { updateBy, carryForward: null };
+    }
+
+    /*
+     * Per matched template, so each is checked against its own project and its
+     * own currently-stored ids — see MonitorService.onBeforeUpdate for why the
+     * stored ids matter.
+     */
+    const templates: Array<Model> = await this.findBy({
+      query: updateBy.query,
+      select: {
+        projectId: true,
+        monitorSteps: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+        ignoreHooks: true,
+      },
+    });
+
+    for (const template of templates) {
+      await MonitorStepsProjectValidator.validateMonitorStepsBelongToProject({
+        monitorSteps: updateBy.data.monitorSteps as MonitorSteps | JSONObject,
+        projectId: updateBy.props.tenantId || template.projectId,
+        alreadyStoredMonitorSteps: template.monitorSteps,
+      });
+    }
+
+    return { updateBy, carryForward: null };
+  }
+
   /**
    * Count monitors created from this template.
    * Caller must already have read access on the template via the API layer.