@oneuptime/common 12.0.1 → 12.0.2

package/Server/Services/HostService.ts

@@ -243,11 +243,14 @@ export class Service extends DatabaseService<Model> {
     }
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = {
+    const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
       otelCollectorStatus: "connected",
     };
 
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const data: any = { ...liveness };
+
     if (extra?.osType) {
       data.osType = extra.osType;
     }
@@ -313,11 +316,51 @@ export class Service extends DatabaseService<Model> {
      * Heartbeat write: a single-statement UPDATE with no hooks and no
      * `version` bump, avoiding the hot-row Postgres lock convoy that the
      * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
+     *
+     * Every optional value above comes from an OpenTelemetry resource
+     * attribute, so its length is whatever the collector felt like sending.
+     * updateColumnsByIdWithoutHooks clamps them to their declared column
+     * widths for us — see truncateStringColumnsToMaxLength.
      */
-    await this.updateColumnsByIdWithoutHooks({
-      id: hostId,
-      data: data,
-    });
+    try {
+      await this.updateColumnsByIdWithoutHooks({
+        id: hostId,
+        data: data,
+      });
+    } catch (err) {
+      /*
+       * Liveness must survive bad metadata. The enriched write carries
+       * `lastSeenAt` and `otelCollectorStatus` alongside a dozen optional,
+       * collector-supplied columns; if any one of them makes Postgres
+       * reject the statement, the host silently stops advancing
+       * lastSeenAt and MarkDisconnectedHosts flips a perfectly healthy
+       * host to "disconnected" 15 minutes later — with telemetry still
+       * arriving the whole time. That was issue #3006 (a host.ip list
+       * longer than the then-varchar(500) column). Retry with liveness
+       * only so the enrichment is what gets dropped, never the heartbeat.
+       */
+      logger.warn(
+        `HostService.updateLastSeen: metadata write failed for host ${hostId.toString()}, falling back to a liveness-only update: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+
+      /*
+       * The throttle fingerprint was already stored, so without this the
+       * next batch would short-circuit on a cache hit and skip the retry
+       * for the rest of the window.
+       */
+      try {
+        await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
+      } catch {
+        // Best-effort; the fallback write below is what actually matters.
+      }
+
+      await this.updateColumnsByIdWithoutHooks({
+        id: hostId,
+        data: liveness,
+      });
+    }
   }
 
   private fingerprintExtras(extra?: {

package/Server/Services/IoTDeviceService.ts

@@ -5,6 +5,7 @@ import ColumnLength from "../../Types/Database/ColumnLength";
 import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import { clampIoTTimestamp } from "../Utils/Telemetry/IoTSnapshotScan";
+import { truncateShortText } from "../Utils/Database/TruncateColumnValue";
 import logger from "../Utils/Logger";
 
 /*
@@ -59,23 +60,6 @@ export interface IoTInventorySummary {
 const UPSERT_BATCH_SIZE: number = 500;
 const STALE_DELETE_WARN_THRESHOLD: number = 100;
 
-/*
- * The identity/text columns are ShortText (100 chars). A single value
- * over the column limit fails the whole multi-row INSERT chunk it rides
- * in — truncate per value instead so one malformed device can never
- * drop the other rows of its chunk.
- */
-function truncateShortText(value: string): string;
-function truncateShortText(value: string | null): string | null;
-function truncateShortText(value: string | null): string | null {
-  if (value === null) {
-    return value;
-  }
-  return value.length > ColumnLength.ShortText
-    ? value.substring(0, ColumnLength.ShortText)
-    : value;
-}
-
 /*
  * Column order used by bulkUpsert() and its generated parameter tuples.
  * Keep this and the INSERT column list in perfect sync.

package/Server/Services/IoTFleetService.ts

@@ -214,11 +214,14 @@ export class Service extends DatabaseService<Model> {
     }
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = {
+    const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
       otelCollectorStatus: "connected",
     };
 
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const data: any = { ...liveness };
+
     if (extra?.agentVersion) {
       data.agentVersion = extra.agentVersion;
     }
@@ -234,11 +237,50 @@ export class Service extends DatabaseService<Model> {
      * Heartbeat write: a single-statement UPDATE with no hooks and no
      * `version` bump, avoiding the hot-row Postgres lock convoy that the
      * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
+     *
+     * Collector-supplied strings are clamped to their declared column
+     * widths inside updateColumnsByIdWithoutHooks — see
+     * DatabaseService.truncateStringColumnsToMaxLength.
      */
-    await this.updateColumnsByIdWithoutHooks({
-      id: fleetId,
-      data: data,
-    });
+    try {
+      await this.updateColumnsByIdWithoutHooks({
+        id: fleetId,
+        data: data,
+      });
+    } catch (err) {
+      /*
+       * Liveness must survive bad metadata. The enriched write carries
+       * `lastSeenAt` and `otelCollectorStatus` alongside optional,
+       * collector-supplied columns; if any one of them makes Postgres
+       * reject the statement, lastSeenAt silently stops advancing and
+       * markDisconnectedFleets flips a perfectly healthy IoT fleet to
+       * "disconnected" 15 minutes later — with telemetry still arriving the
+       * whole time. That was issue #3006 on the Host path (a host.ip list
+       * longer than the then-varchar(500) column). Retry with liveness only
+       * so the enrichment is what gets dropped, never the heartbeat.
+       */
+      logger.warn(
+        `IoTFleetService.updateLastSeen: metadata write failed for IoT fleet ${fleetId.toString()}, falling back to a liveness-only update: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+
+      /*
+       * The throttle fingerprint was already stored, so without this the
+       * next batch would short-circuit on a cache hit and skip the retry
+       * for the rest of the window.
+       */
+      try {
+        await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
+      } catch {
+        // Best-effort; the fallback write below is what actually matters.
+      }
+
+      await this.updateColumnsByIdWithoutHooks({
+        id: fleetId,
+        data: liveness,
+      });
+    }
   }
 
   /**

package/Server/Services/KubernetesClusterService.ts

@@ -184,11 +184,14 @@ export class Service extends DatabaseService<Model> {
     }
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = {
+    const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
       otelCollectorStatus: "connected",
     };
 
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const data: any = { ...liveness };
+
     if (extra?.agentVersion) {
       data.agentVersion = extra.agentVersion;
     }
@@ -197,11 +200,51 @@ export class Service extends DatabaseService<Model> {
      * Heartbeat write: a single-statement UPDATE with no hooks and no
      * `version` bump, avoiding the hot-row Postgres lock convoy that the
      * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
+     *
+     * Collector-supplied strings are clamped to their declared column
+     * widths inside updateColumnsByIdWithoutHooks — see
+     * DatabaseService.truncateStringColumnsToMaxLength.
      */
-    await this.updateColumnsByIdWithoutHooks({
-      id: clusterId,
-      data: data,
-    });
+    try {
+      await this.updateColumnsByIdWithoutHooks({
+        id: clusterId,
+        data: data,
+      });
+    } catch (err) {
+      /*
+       * Liveness must survive bad metadata. The enriched write carries
+       * `lastSeenAt` and `otelCollectorStatus` alongside optional,
+       * collector-supplied columns; if any one of them makes Postgres
+       * reject the statement, lastSeenAt silently stops advancing and
+       * markDisconnectedClusters flips a perfectly healthy Kubernetes
+       * cluster to "disconnected" 15 minutes later — with telemetry still
+       * arriving the whole time. That was issue #3006 on the Host path (a
+       * host.ip list longer than the then-varchar(500) column). Retry with
+       * liveness only so the enrichment is what gets dropped, never the
+       * heartbeat.
+       */
+      logger.warn(
+        `KubernetesClusterService.updateLastSeen: metadata write failed for Kubernetes cluster ${clusterId.toString()}, falling back to a liveness-only update: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+
+      /*
+       * The throttle fingerprint was already stored, so without this the
+       * next batch would short-circuit on a cache hit and skip the retry
+       * for the rest of the window.
+       */
+      try {
+        await GlobalCache.deleteKey(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
+      } catch {
+        // Best-effort; the fallback write below is what actually matters.
+      }
+
+      await this.updateColumnsByIdWithoutHooks({
+        id: clusterId,
+        data: liveness,
+      });
+    }
   }
 
   /**

package/Server/Services/KubernetesContainerService.ts

@@ -1,7 +1,12 @@
 import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/KubernetesContainer";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ColumnLength from "../../Types/Database/ColumnLength";
 import ObjectID from "../../Types/ObjectID";
+import {
+  truncateLongText,
+  truncateShortText,
+} from "../Utils/Database/TruncateColumnValue";
 import logger from "../Utils/Logger";
 
 /*
@@ -61,6 +66,29 @@ const UPSERT_COLUMNS: Array<string> = [
   "version",
 ];
 
+/*
+ * KubernetesContainer's text columns are bounded varchars — podName and
+ * image at 500 chars, podNamespaceKey / name / state / reason at 100.
+ * The bulk paths below go through manager.query(), which skips the
+ * length validation DatabaseService applies to ordinary writes, so a
+ * single oversized value aborts the whole 500-row INSERT chunk it rides
+ * in. Clamp per value instead, so one pathological pod can never drop
+ * the rest of the snapshot.
+ */
+function sanitizeContainer(
+  c: ParsedKubernetesContainer,
+): ParsedKubernetesContainer {
+  return {
+    ...c,
+    podNamespaceKey: truncateShortText(c.podNamespaceKey),
+    podName: truncateLongText(c.podName),
+    name: truncateShortText(c.name),
+    image: truncateLongText(c.image),
+    state: truncateShortText(c.state),
+    reason: truncateShortText(c.reason),
+  };
+}
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
@@ -86,12 +114,20 @@ export class Service extends DatabaseService<Model> {
       return;
     }
 
-    for (
-      let i: number = 0;
-      i < data.containers.length;
-      i += UPSERT_BATCH_SIZE
-    ) {
-      const chunk: Array<ParsedKubernetesContainer> = data.containers.slice(
+    const containers: Array<ParsedKubernetesContainer> = data.containers.map(
+      (c: ParsedKubernetesContainer) => {
+        const sanitized: ParsedKubernetesContainer = sanitizeContainer(c);
+        if (sanitized.podName !== c.podName) {
+          logger.warn(
+            `KubernetesContainer podName exceeds ${ColumnLength.LongText} chars; truncated to "${sanitized.podName}" (cluster ${data.kubernetesClusterId.toString()}).`,
+          );
+        }
+        return sanitized;
+      },
+    );
+
+    for (let i: number = 0; i < containers.length; i += UPSERT_BATCH_SIZE) {
+      const chunk: Array<ParsedKubernetesContainer> = containers.slice(
         i,
         i + UPSERT_BATCH_SIZE,
       );
@@ -224,10 +260,15 @@ export class Service extends DatabaseService<Model> {
         valueFragments.push(
           `($${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}::numeric, $${paramIndex++}::bigint, $${paramIndex++}::timestamptz)`,
         );
+        /*
+         * The identity clamps MUST match bulkUpsert's, or this mirror
+         * UPDATE would silently miss the row the upsert just wrote
+         * under the truncated podName.
+         */
         params.push(
-          m.podNamespaceKey,
-          m.podName,
-          m.name,
+          truncateShortText(m.podNamespaceKey),
+          truncateLongText(m.podName),
+          truncateShortText(m.name),
           m.cpuPercent !== null && m.cpuPercent !== undefined
             ? m.cpuPercent
             : null,

package/Server/Services/KubernetesResourceService.ts

@@ -1,9 +1,14 @@
 import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/KubernetesResource";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ColumnLength from "../../Types/Database/ColumnLength";
 import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import { ParsedKubernetesResource } from "../../Types/Kubernetes/KubernetesInventoryExtractor";
+import {
+  truncateLongText,
+  truncateShortText,
+} from "../Utils/Database/TruncateColumnValue";
 import logger from "../Utils/Logger";
 
 /*
@@ -317,6 +322,28 @@ const UPSERT_COLUMNS: Array<keyof ParsedKubernetesResource | string> = [
   "version",
 ];
 
+/*
+ * KubernetesResource's text columns are bounded varchars — name and
+ * controllerDeploymentName at 500 chars, kind / namespaceKey / uid /
+ * phase at 100. The bulk paths below go through manager.query(), which
+ * skips the length validation DatabaseService applies to ordinary
+ * writes, so a single oversized value aborts the whole 500-row INSERT
+ * chunk it rides in. Clamp per value instead, so one pathological
+ * object can never drop the rest of the snapshot.
+ */
+function sanitizeResource(
+  r: ParsedKubernetesResource,
+): ParsedKubernetesResource {
+  return {
+    ...r,
+    kind: truncateShortText(r.kind),
+    namespaceKey: truncateShortText(r.namespaceKey),
+    name: truncateLongText(r.name),
+    uid: truncateShortText(r.uid),
+    phase: truncateShortText(r.phase),
+  };
+}
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
@@ -338,9 +365,21 @@ export class Service extends DatabaseService<Model> {
       return;
     }
 
+    const resources: Array<ParsedKubernetesResource> = data.resources.map(
+      (r: ParsedKubernetesResource) => {
+        const sanitized: ParsedKubernetesResource = sanitizeResource(r);
+        if (sanitized.name !== r.name) {
+          logger.warn(
+            `KubernetesResource name exceeds ${ColumnLength.LongText} chars; truncated to "${sanitized.name}" (cluster ${data.kubernetesClusterId.toString()}).`,
+          );
+        }
+        return sanitized;
+      },
+    );
+
     // Chunk to keep individual statement parameter counts reasonable.
-    for (let i: number = 0; i < data.resources.length; i += UPSERT_BATCH_SIZE) {
-      const chunk: Array<ParsedKubernetesResource> = data.resources.slice(
+    for (let i: number = 0; i < resources.length; i += UPSERT_BATCH_SIZE) {
+      const chunk: Array<ParsedKubernetesResource> = resources.slice(
         i,
         i + UPSERT_BATCH_SIZE,
       );
@@ -450,10 +489,17 @@ export class Service extends DatabaseService<Model> {
         valueFragments.push(
           `($${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}::numeric, $${paramIndex++}::bigint, $${paramIndex++}::numeric, $${paramIndex++}::timestamptz, $${paramIndex++}, $${paramIndex++})`,
         );
+        /*
+         * The identity clamps MUST match bulkUpsert's, or this mirror
+         * UPDATE would silently miss the row the upsert just wrote
+         * under the truncated name. controllerDeploymentName /
+         * controllerCronJobName are assigned columns, so an oversized
+         * value there aborts the chunk exactly like the INSERT path.
+         */
         params.push(
-          m.kind,
-          m.namespaceKey,
-          m.name,
+          truncateShortText(m.kind),
+          truncateShortText(m.namespaceKey),
+          truncateLongText(m.name),
           m.cpuPercent !== null && m.cpuPercent !== undefined
             ? m.cpuPercent
             : null,
@@ -464,8 +510,8 @@ export class Service extends DatabaseService<Model> {
             ? m.memoryPercent
             : null,
           m.observedAt,
-          m.controllerDeploymentName ?? null,
-          m.controllerCronJobName ?? null,
+          truncateLongText(m.controllerDeploymentName ?? null),
+          truncateShortText(m.controllerCronJobName ?? null),
         );
       }
 

package/Server/Services/LabelService.ts

@@ -1,14 +1,20 @@
 import CreateBy from "../Types/Database/CreateBy";
-import { OnCreate } from "../Types/Database/Hooks";
+import DeleteBy from "../Types/Database/DeleteBy";
+import { OnCreate, OnDelete } from "../Types/Database/Hooks";
 import QueryHelper from "../Types/Database/QueryHelper";
 import DatabaseService from "./DatabaseService";
+import ServiceLevelObjectiveMonitorRuleEngineService from "./ServiceLevelObjectiveMonitorRuleEngineService";
+import ServiceLevelObjectiveService from "./ServiceLevelObjectiveService";
+import ServiceLevelObjective from "../../Models/DatabaseModels/ServiceLevelObjective";
 import BadDataException from "../../Types/Exception/BadDataException";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
 import ObjectID from "../../Types/ObjectID";
 import Color from "../../Types/Color";
 import { BrightColors } from "../../Types/BrandColors";
 import GlobalCache from "../Infrastructure/GlobalCache";
 import Model from "../../Models/DatabaseModels/Label";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import logger from "../Utils/Logger";
 
 const AUTO_LABEL_ID_CACHE_NAMESPACE: string = "auto-label-id";
 const AUTO_LABEL_ID_CACHE_TTL_SECONDS: number = 24 * 60 * 60;
@@ -57,6 +63,105 @@ export class Service extends DatabaseService<Model> {
     return Promise.resolve({ createBy, carryForward: null });
   }
 
+  /*
+   * Deleting a label cascades its join rows away at the database level, so no
+   * service hook ever fires for the SLO label rules that referenced it. Those
+   * SLOs would keep the monitors the rule attached, with nothing left to
+   * explain why. Note the ids down while the label still exists, and re-run
+   * the (now smaller) rules once it is gone.
+   */
+  @CaptureSpan()
+  protected override async onBeforeDelete(
+    deleteBy: DeleteBy<Model>,
+  ): Promise<OnDelete<Model>> {
+    let serviceLevelObjectiveIds: Array<ObjectID> = [];
+
+    try {
+      const labels: Array<Model> = await this.findBy({
+        query: deleteBy.query,
+        select: {
+          _id: true,
+        },
+        limit: LIMIT_MAX,
+        skip: 0,
+        props: {
+          isRoot: true,
+        },
+      });
+
+      const labelIds: Array<ObjectID> = labels
+        .map((label: Model) => {
+          return label.id;
+        })
+        .filter((id: ObjectID | null): id is ObjectID => {
+          return Boolean(id);
+        });
+
+      if (labelIds.length > 0) {
+        const slos: Array<ServiceLevelObjective> =
+          await ServiceLevelObjectiveService.findBy({
+            query: {
+              monitorLabels: labelIds,
+            },
+            select: {
+              _id: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: {
+              isRoot: true,
+            },
+          });
+
+        serviceLevelObjectiveIds = slos
+          .map((slo: ServiceLevelObjective) => {
+            return slo.id;
+          })
+          .filter((id: ObjectID | null): id is ObjectID => {
+            return Boolean(id);
+          });
+      }
+    } catch (err) {
+      logger.error(
+        `Error collecting SLO label rules affected by a label delete: ${err}`,
+      );
+    }
+
+    return {
+      deleteBy,
+      carryForward: {
+        serviceLevelObjectiveIds: serviceLevelObjectiveIds,
+      },
+    };
+  }
+
+  @CaptureSpan()
+  protected override async onDeleteSuccess(
+    onDelete: OnDelete<Model>,
+    _itemIdsBeforeDelete: Array<ObjectID>,
+  ): Promise<OnDelete<Model>> {
+    const serviceLevelObjectiveIds: Array<ObjectID> =
+      (
+        onDelete.carryForward as {
+          serviceLevelObjectiveIds?: Array<ObjectID>;
+        } | null
+      )?.serviceLevelObjectiveIds || [];
+
+    for (const serviceLevelObjectiveId of serviceLevelObjectiveIds) {
+      try {
+        await ServiceLevelObjectiveMonitorRuleEngineService.syncMonitorsForSlo({
+          serviceLevelObjectiveId: serviceLevelObjectiveId,
+        });
+      } catch (err) {
+        logger.error(
+          `Error re-applying the monitor label rule for SLO ${serviceLevelObjectiveId.toString()} after a label delete: ${err}`,
+        );
+      }
+    }
+
+    return onDelete;
+  }
+
   /**
    * Find existing labels by name (case-insensitive) or auto-create
    * them, returning the ObjectIDs in the same order as the input.

package/Server/Services/MonitorService.ts

@@ -14,6 +14,7 @@ import MonitorOwnerTeamService from "./MonitorOwnerTeamService";
 import MonitorOwnerUserService from "./MonitorOwnerUserService";
 import MonitorProbeService from "./MonitorProbeService";
 import MonitorStatusService from "./MonitorStatusService";
+import ServiceLevelObjectiveMonitorRuleEngineService from "./ServiceLevelObjectiveMonitorRuleEngineService";
 import NetworkSiteService from "./NetworkSiteService";
 import MonitorStatusTimelineService, {
   MONITOR_STATUS_SAME_AS_PREVIOUS_ERROR_MESSAGE,
@@ -612,6 +613,37 @@ export class Service extends DatabaseService<Model> {
       }
     }
 
+    /*
+     * Labels decide SLO membership, so a label added or removed here can pull
+     * this monitor into an SLO's error budget or push it out of one. Keyed on
+     * `!== undefined` rather than on a non-empty array: clearing every label
+     * arrives as `[]`, and that is precisely the edit that should detach the
+     * monitor from every rule-driven SLO.
+     */
+    if (
+      onUpdate.updateBy.data.labels !== undefined &&
+      updatedItemIds.length > 0
+    ) {
+      for (const monitorId of updatedItemIds) {
+        try {
+          await ServiceLevelObjectiveMonitorRuleEngineService.syncSlosForMonitor(
+            {
+              monitorId: monitorId,
+              projectId: onUpdate.updateBy.props.tenantId as ObjectID,
+            },
+          );
+        } catch (error) {
+          logger.error(
+            "Syncing SLO label rules failed in MonitorService.onUpdateSuccess",
+            {
+              monitorId: monitorId?.toString(),
+            } as LogAttributes,
+          );
+          logger.error(error as Error);
+        }
+      }
+    }
+
     return onUpdate;
   }
 
@@ -983,6 +1015,31 @@ ${createdItem.description?.trim() || "No description provided."}
         }
         return Promise.resolve();
       })
+      .then(async () => {
+        /*
+         * Runs after the label rules above so a monitor created with no labels
+         * of its own, but given some by a MonitorLabelRule, still lands in the
+         * SLOs those labels imply.
+         */
+        try {
+          await ServiceLevelObjectiveMonitorRuleEngineService.syncSlosForMonitor(
+            {
+              monitorId: createdItem.id!,
+              projectId: createdItem.projectId!,
+            },
+          );
+        } catch (error) {
+          logger.error(
+            "Syncing SLO label rules failed in MonitorService.onCreateSuccess",
+            {
+              projectId: createdItem.projectId?.toString(),
+              monitorId: createdItem.id?.toString(),
+            } as LogAttributes,
+          );
+          logger.error(error as Error);
+        }
+        return Promise.resolve();
+      })
       .then(async () => {
         try {
           return await this.refreshMonitorProbeStatus(createdItem.id!);