@oneuptime/common 12.0.1 → 12.0.3

package/Server/Services/MonitorService.ts

@@ -14,6 +14,8 @@ import MonitorOwnerTeamService from "./MonitorOwnerTeamService";
 import MonitorOwnerUserService from "./MonitorOwnerUserService";
 import MonitorProbeService from "./MonitorProbeService";
 import MonitorStatusService from "./MonitorStatusService";
+import ServiceLevelObjectiveMonitorRuleEngineService from "./ServiceLevelObjectiveMonitorRuleEngineService";
+import StatusPageMonitorRuleEngineService from "./StatusPageMonitorRuleEngineService";
 import NetworkSiteService from "./NetworkSiteService";
 import MonitorStatusTimelineService, {
   MONITOR_STATUS_SAME_AS_PREVIOUS_ERROR_MESSAGE,
@@ -403,27 +405,47 @@ export class Service extends DatabaseService<Model> {
       resolveReferenceId(updateBy.data.currentMonitorStatusId) ||
       resolveReferenceId(updateBy.data.currentMonitorStatus);
 
-    if (updateBy.data.monitorSteps || currentMonitorStatusId) {
+    if (updateBy.data.monitorSteps) {
       /*
-       * Root/API updates do not always carry a tenantId, so fall back to the
-       * project of each monitor the query actually matches.
+       * Validated per matched monitor rather than per distinct project, because
+       * the check needs that monitor's CURRENT monitorSteps: a reference id it
+       * already holds is exempt from the existence check, so an update never
+       * refuses a monitor that was stored broken before the guard existed. See
+       * MonitorStepsProjectValidator.
        */
+      const monitors: Array<Model> = await this.findBy({
+        query: updateBy.query,
+        select: {
+          projectId: true,
+          monitorSteps: true,
+        },
+        limit: LIMIT_MAX,
+        skip: 0,
+        props: {
+          isRoot: true,
+          ignoreHooks: true,
+        },
+      });
+
+      for (const monitor of monitors) {
+        await MonitorStepsProjectValidator.validateMonitorStepsBelongToProject({
+          monitorSteps: updateBy.data.monitorSteps as MonitorSteps | JSONObject,
+          /*
+           * Root/API updates do not always carry a tenantId, so fall back to
+           * the project of the monitor being updated.
+           */
+          projectId: updateBy.props.tenantId || monitor.projectId,
+          alreadyStoredMonitorSteps: monitor.monitorSteps,
+        });
+      }
+    }
+
+    if (currentMonitorStatusId) {
       const projectIds: Array<ObjectID> = updateBy.props.tenantId
         ? [updateBy.props.tenantId]
         : await this.getProjectIdsForUpdateQuery(updateBy);
 
       for (const projectId of projectIds) {
-        if (updateBy.data.monitorSteps) {
-          await MonitorStepsProjectValidator.validateMonitorStepsBelongToProject(
-            {
-              monitorSteps: updateBy.data.monitorSteps as
-                | MonitorSteps
-                | JSONObject,
-              projectId: projectId,
-            },
-          );
-        }
-
         await ProjectScopedReferenceValidator.validateReferencesBelongToProject(
           {
             projectId: projectId,
@@ -612,6 +634,68 @@ export class Service extends DatabaseService<Model> {
       }
     }
 
+    /*
+     * Labels decide SLO membership, so a label added or removed here can pull
+     * this monitor into an SLO's error budget or push it out of one. Keyed on
+     * `!== undefined` rather than on a non-empty array: clearing every label
+     * arrives as `[]`, and that is precisely the edit that should detach the
+     * monitor from every rule-driven SLO.
+     */
+    if (
+      onUpdate.updateBy.data.labels !== undefined &&
+      updatedItemIds.length > 0
+    ) {
+      for (const monitorId of updatedItemIds) {
+        try {
+          await ServiceLevelObjectiveMonitorRuleEngineService.syncSlosForMonitor(
+            {
+              monitorId: monitorId,
+              projectId: onUpdate.updateBy.props.tenantId as ObjectID,
+            },
+          );
+        } catch (error) {
+          logger.error(
+            "Syncing SLO label rules failed in MonitorService.onUpdateSuccess",
+            {
+              monitorId: monitorId?.toString(),
+            } as LogAttributes,
+          );
+          logger.error(error as Error);
+        }
+      }
+    }
+
+    /*
+     * Status page monitor rules match on labels, name and description, so an
+     * edit to any of the three can pull this monitor onto a status page or
+     * push it off one. Keyed on `!== undefined` for the same reason as the SLO
+     * block above: clearing every label arrives as `[]`, and that is exactly
+     * the edit that should detach the monitor from every label-driven rule.
+     */
+    if (
+      (onUpdate.updateBy.data.labels !== undefined ||
+        onUpdate.updateBy.data.name !== undefined ||
+        onUpdate.updateBy.data.description !== undefined) &&
+      updatedItemIds.length > 0
+    ) {
+      for (const monitorId of updatedItemIds) {
+        try {
+          await StatusPageMonitorRuleEngineService.syncRulesForMonitor({
+            monitorId: monitorId,
+            projectId: onUpdate.updateBy.props.tenantId as ObjectID,
+          });
+        } catch (error) {
+          logger.error(
+            "Syncing status page monitor rules failed in MonitorService.onUpdateSuccess",
+            {
+              monitorId: monitorId?.toString(),
+            } as LogAttributes,
+          );
+          logger.error(error as Error);
+        }
+      }
+    }
+
     return onUpdate;
   }
 
@@ -983,6 +1067,54 @@ ${createdItem.description?.trim() || "No description provided."}
         }
         return Promise.resolve();
       })
+      .then(async () => {
+        /*
+         * Runs after the label rules above so a monitor created with no labels
+         * of its own, but given some by a MonitorLabelRule, still lands in the
+         * SLOs those labels imply.
+         */
+        try {
+          await ServiceLevelObjectiveMonitorRuleEngineService.syncSlosForMonitor(
+            {
+              monitorId: createdItem.id!,
+              projectId: createdItem.projectId!,
+            },
+          );
+        } catch (error) {
+          logger.error(
+            "Syncing SLO label rules failed in MonitorService.onCreateSuccess",
+            {
+              projectId: createdItem.projectId?.toString(),
+              monitorId: createdItem.id?.toString(),
+            } as LogAttributes,
+          );
+          logger.error(error as Error);
+        }
+        return Promise.resolve();
+      })
+      .then(async () => {
+        /*
+         * Also runs after the label rules above, so a monitor that only earns
+         * its labels from a MonitorLabelRule still lands on the status pages
+         * those labels imply.
+         */
+        try {
+          await StatusPageMonitorRuleEngineService.syncRulesForMonitor({
+            monitorId: createdItem.id!,
+            projectId: createdItem.projectId!,
+          });
+        } catch (error) {
+          logger.error(
+            "Syncing status page monitor rules failed in MonitorService.onCreateSuccess",
+            {
+              projectId: createdItem.projectId?.toString(),
+              monitorId: createdItem.id?.toString(),
+            } as LogAttributes,
+          );
+          logger.error(error as Error);
+        }
+        return Promise.resolve();
+      })
       .then(async () => {
         try {
           return await this.refreshMonitorProbeStatus(createdItem.id!);

package/Server/Services/MonitorStatusTimelineService.ts

@@ -4,6 +4,7 @@ import DeleteBy from "../Types/Database/DeleteBy";
 import { OnCreate, OnDelete } from "../Types/Database/Hooks";
 import QueryHelper from "../Types/Database/QueryHelper";
 import logger, { LogAttributes } from "../Utils/Logger";
+import ProjectScopedReferenceValidator from "../Utils/Database/ProjectScopedReferenceValidator";
 import DatabaseService from "./DatabaseService";
 import MonitorService from "./MonitorService";
 import UserService from "./UserService";
@@ -191,6 +192,34 @@ export class Service extends DatabaseService<MonitorStatusTimeline> {
       throw new BadDataException("monitorStatusId is null");
     }
 
+    /*
+     * Every writer of this table funnels through here, so this is the one place
+     * that can turn issue #3039's failure mode into an answer. A monitorStatusId
+     * that does not exist (or belongs to another project) used to travel all the
+     * way to Postgres and come back as
+     *   insert or update on table "MonitorStatusTimeline" violates foreign key
+     *   constraint "FK_574feb4161c5216c2c7ee0faaf8"
+     * — an opaque 500 over the API, and a job that fails and retries forever in
+     * the probe worker. Reject it here instead, naming the id.
+     *
+     * The probe/telemetry ingest callers screen the id themselves before they
+     * get here (Utils/Monitor/MonitorStatusTimeline and Utils/Monitor/MonitorResource)
+     * so a monitor whose stored criteria already point at a deleted status skip
+     * the status change rather than failing their whole run. This check is the
+     * backstop for everything else, chiefly direct API writes.
+     */
+    await ProjectScopedReferenceValidator.validateReferencesBelongToProject({
+      projectId: createBy.props.tenantId || createBy.data.projectId,
+      subject: "monitor status timeline",
+      references: [
+        {
+          modelName: "Monitor Status",
+          id: monitorStatusId,
+          service: MonitorStatusService,
+        },
+      ],
+    });
+
     const stateBeforeThis: MonitorStatusTimeline | null = await this.findOneBy({
       query: {
         monitorId: monitorId,

package/Server/Services/MonitorTemplateService.ts

@@ -1,8 +1,14 @@
 import DatabaseService from "./DatabaseService";
 import MonitorService from "./MonitorService";
+import CreateBy from "../Types/Database/CreateBy";
+import UpdateBy from "../Types/Database/UpdateBy";
+import { OnCreate, OnUpdate } from "../Types/Database/Hooks";
+import MonitorStepsProjectValidator from "../Utils/Monitor/MonitorStepsProjectValidator";
 import DatabaseCommonInteractionProps from "../../Types/BaseDatabase/DatabaseCommonInteractionProps";
 import BadDataException from "../../Types/Exception/BadDataException";
 import LIMIT_MAX from "../../Types/Database/LimitMax";
+import { JSONObject } from "../../Types/JSON";
+import MonitorSteps from "../../Types/Monitor/MonitorSteps";
 import ObjectID from "../../Types/ObjectID";
 import PositiveNumber from "../../Types/PositiveNumber";
 import Model from "../../Models/DatabaseModels/MonitorTemplate";
@@ -37,6 +43,65 @@ export class Service extends DatabaseService<Model> {
     super(Model);
   }
 
+  /*
+   * A template's monitorSteps embeds the same reference ids a monitor's does,
+   * and every one of them reaches a real monitor eventually — through "create
+   * monitor from template" and through syncLinkedMonitors, which pushes the
+   * blob onto every linked monitor. Validating it here is what makes the error
+   * land on the template the bad id was typed into, rather than on a monitor
+   * sync days later. Without this the template is the one place a dangling id
+   * can still enter the system.
+   */
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    await MonitorStepsProjectValidator.validateMonitorStepsBelongToProject({
+      monitorSteps: createBy.data.monitorSteps,
+      projectId: createBy.props.tenantId || createBy.data.projectId,
+    });
+
+    return { createBy, carryForward: null };
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeUpdate(
+    updateBy: UpdateBy<Model>,
+  ): Promise<OnUpdate<Model>> {
+    if (!updateBy.data.monitorSteps) {
+      return { updateBy, carryForward: null };
+    }
+
+    /*
+     * Per matched template, so each is checked against its own project and its
+     * own currently-stored ids — see MonitorService.onBeforeUpdate for why the
+     * stored ids matter.
+     */
+    const templates: Array<Model> = await this.findBy({
+      query: updateBy.query,
+      select: {
+        projectId: true,
+        monitorSteps: true,
+      },
+      limit: LIMIT_MAX,
+      skip: 0,
+      props: {
+        isRoot: true,
+        ignoreHooks: true,
+      },
+    });
+
+    for (const template of templates) {
+      await MonitorStepsProjectValidator.validateMonitorStepsBelongToProject({
+        monitorSteps: updateBy.data.monitorSteps as MonitorSteps | JSONObject,
+        projectId: updateBy.props.tenantId || template.projectId,
+        alreadyStoredMonitorSteps: template.monitorSteps,
+      });
+    }
+
+    return { updateBy, carryForward: null };
+  }
+
   /**
    * Count monitors created from this template.
    * Caller must already have read access on the template via the API layer.

package/Server/Services/PodmanHostService.ts

@@ -5,6 +5,7 @@ import Model from "../../Models/DatabaseModels/PodmanHost";
 import Label from "../../Models/DatabaseModels/Label";
 import { OnCreate } from "../Types/Database/Hooks";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
 import ObjectID from "../../Types/ObjectID";
 import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
@@ -180,7 +181,6 @@ export class Service extends DatabaseService<Model> {
       agentVersion?: string | undefined;
     },
   ): Promise<void> {
-    const cacheKey: string = hostId.toString();
     const extrasFingerprint: string = crypto
       .createHash("sha1")
       .update(
@@ -192,58 +192,40 @@ export class Service extends DatabaseService<Model> {
       )
       .digest("hex");
 
-    let cached: string | null = null;
-    try {
-      cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-    } catch {
-      /*
-       * Cache unavailable — fail open and refresh lastSeenAt anyway. A
-       * cache error must never skip the DB write below, otherwise the
-       * resource is wrongly marked "disconnected" while telemetry is
-       * still flowing. Mirrors shouldRunMaintenance's fail-open stance.
-       */
-      cached = null;
-    }
-
-    if (cached === extrasFingerprint) {
-      return; // same data was written recently
-    }
-
-    try {
-      await GlobalCache.setString(
-        LAST_SEEN_CACHE_NAMESPACE,
-        cacheKey,
-        extrasFingerprint,
-        { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
-      );
-    } catch {
-      // Best-effort throttle write; proceed with the DB update regardless.
-    }
-
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = {
+    const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
       otelCollectorStatus: "connected",
     };
 
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const metadata: any = {};
+
     if (extra?.osType) {
-      data.osType = extra.osType;
+      metadata.osType = extra.osType;
     }
     if (extra?.osVersion) {
-      data.osVersion = extra.osVersion;
+      metadata.osVersion = extra.osVersion;
     }
     if (extra?.agentVersion) {
-      data.agentVersion = extra.agentVersion;
+      metadata.agentVersion = extra.agentVersion;
     }
 
     /*
-     * Heartbeat write: a single-statement UPDATE with no hooks and no
-     * `version` bump, avoiding the hot-row Postgres lock convoy that the
-     * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
+     * One gated, non-blocking heartbeat write. The gates, the fail-open /
+     * fail-closed split and the liveness-only fallback all live in
+     * ResourceHeartbeat — see there for why this row's throttle used to
+     * provide no throttling at all.
      */
-    await this.updateColumnsByIdWithoutHooks({
+    await ResourceHeartbeat.write({
+      service: this,
       id: hostId,
-      data: data,
+      cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
+      throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
+      liveness: liveness,
+      metadata: metadata,
+      fingerprint: extrasFingerprint,
+      describe: `podman host ${hostId.toString()}`,
     });
   }
 

package/Server/Services/PodmanResourceService.ts

@@ -1,9 +1,14 @@
 import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/PodmanResource";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ColumnLength from "../../Types/Database/ColumnLength";
 import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import { JSONObject } from "../../Types/JSON";
+import {
+  truncateLongText,
+  truncateShortText,
+} from "../Utils/Database/TruncateColumnValue";
 import logger from "../Utils/Logger";
 
 /*
@@ -56,6 +61,35 @@ export interface PodmanHostInventoryCounts {
 const UPSERT_BATCH_SIZE: number = 500;
 const STALE_DELETE_WARN_THRESHOLD: number = 100;
 
+/*
+ * PodmanResource's text columns are bounded varchars — name / imageName
+ * at 500 chars, kind / containerId / state at 100. These bulk paths go
+ * through manager.query(), which skips the length validation
+ * DatabaseService applies to ordinary writes, so a single oversized
+ * value aborts the whole 500-row INSERT chunk it rides in. Clamp per
+ * value instead, so one pathological container can never drop the rest.
+ */
+function sanitizeContainer(c: ParsedPodmanContainer): ParsedPodmanContainer {
+  return {
+    ...c,
+    containerName: truncateLongText(c.containerName),
+    containerId: truncateShortText(c.containerId),
+    imageName: truncateLongText(c.imageName),
+    state: truncateShortText(c.state),
+  };
+}
+
+function sanitizeResource(r: ParsedPodmanResource): ParsedPodmanResource {
+  return {
+    ...r,
+    kind: truncateShortText(r.kind),
+    name: truncateLongText(r.name),
+    containerId: truncateShortText(r.containerId),
+    imageName: truncateLongText(r.imageName),
+    state: truncateShortText(r.state),
+  };
+}
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
@@ -83,12 +117,20 @@ export class Service extends DatabaseService<Model> {
       return;
     }
 
-    for (
-      let i: number = 0;
-      i < data.containers.length;
-      i += UPSERT_BATCH_SIZE
-    ) {
-      const chunk: Array<ParsedPodmanContainer> = data.containers.slice(
+    const containers: Array<ParsedPodmanContainer> = data.containers.map(
+      (c: ParsedPodmanContainer) => {
+        const sanitized: ParsedPodmanContainer = sanitizeContainer(c);
+        if (sanitized.containerName !== c.containerName) {
+          logger.warn(
+            `PodmanResource container name exceeds ${ColumnLength.LongText} chars; truncated to "${sanitized.containerName}" (host ${data.podmanHostId.toString()}).`,
+          );
+        }
+        return sanitized;
+      },
+    );
+
+    for (let i: number = 0; i < containers.length; i += UPSERT_BATCH_SIZE) {
+      const chunk: Array<ParsedPodmanContainer> = containers.slice(
         i,
         i + UPSERT_BATCH_SIZE,
       );
@@ -164,8 +206,20 @@ export class Service extends DatabaseService<Model> {
       return;
     }
 
-    for (let i: number = 0; i < data.resources.length; i += UPSERT_BATCH_SIZE) {
-      const chunk: Array<ParsedPodmanResource> = data.resources.slice(
+    const resources: Array<ParsedPodmanResource> = data.resources.map(
+      (r: ParsedPodmanResource) => {
+        const sanitized: ParsedPodmanResource = sanitizeResource(r);
+        if (sanitized.name !== r.name) {
+          logger.warn(
+            `PodmanResource name exceeds ${ColumnLength.LongText} chars; truncated to "${sanitized.name}" (host ${data.podmanHostId.toString()}).`,
+          );
+        }
+        return sanitized;
+      },
+    );
+
+    for (let i: number = 0; i < resources.length; i += UPSERT_BATCH_SIZE) {
+      const chunk: Array<ParsedPodmanResource> = resources.slice(
         i,
         i + UPSERT_BATCH_SIZE,
       );