@oneuptime/common 12.0.1 → 12.0.3

package/Server/Services/KubernetesClusterService.ts

@@ -5,6 +5,7 @@ import Model from "../../Models/DatabaseModels/KubernetesCluster";
 import Label from "../../Models/DatabaseModels/Label";
 import { OnCreate } from "../Types/Database/Hooks";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ResourceHeartbeat from "../Utils/Telemetry/ResourceHeartbeat";
 import ObjectID from "../../Types/ObjectID";
 import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
@@ -145,7 +146,6 @@ export class Service extends DatabaseService<Model> {
       agentVersion?: string | undefined;
     },
   ): Promise<void> {
-    const cacheKey: string = clusterId.toString();
     const extrasFingerprint: string = crypto
       .createHash("sha1")
       .update(
@@ -155,52 +155,34 @@ export class Service extends DatabaseService<Model> {
       )
       .digest("hex");
 
-    let cached: string | null = null;
-    try {
-      cached = await GlobalCache.getString(LAST_SEEN_CACHE_NAMESPACE, cacheKey);
-    } catch {
-      /*
-       * Cache unavailable — fail open and refresh lastSeenAt anyway. A
-       * cache error must never skip the DB write below, otherwise the
-       * resource is wrongly marked "disconnected" while telemetry is
-       * still flowing. Mirrors shouldRunMaintenance's fail-open stance.
-       */
-      cached = null;
-    }
-
-    if (cached === extrasFingerprint) {
-      return; // same data was written recently
-    }
-
-    try {
-      await GlobalCache.setString(
-        LAST_SEEN_CACHE_NAMESPACE,
-        cacheKey,
-        extrasFingerprint,
-        { expiresInSeconds: LAST_SEEN_THROTTLE_SECONDS },
-      );
-    } catch {
-      // Best-effort throttle write; proceed with the DB update regardless.
-    }
-
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    const data: any = {
+    const liveness: any = {
       lastSeenAt: OneUptimeDate.getCurrentDate(),
       otelCollectorStatus: "connected",
     };
 
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const metadata: any = {};
+
     if (extra?.agentVersion) {
-      data.agentVersion = extra.agentVersion;
+      metadata.agentVersion = extra.agentVersion;
     }
 
     /*
-     * Heartbeat write: a single-statement UPDATE with no hooks and no
-     * `version` bump, avoiding the hot-row Postgres lock convoy that the
-     * full updateOneById pipeline causes. See ServiceService.updateLastSeen.
+     * One gated, non-blocking heartbeat write. The gates, the fail-open /
+     * fail-closed split and the liveness-only fallback all live in
+     * ResourceHeartbeat — see there for why this row's throttle used to
+     * provide no throttling at all.
      */
-    await this.updateColumnsByIdWithoutHooks({
+    await ResourceHeartbeat.write({
+      service: this,
       id: clusterId,
-      data: data,
+      cacheNamespace: LAST_SEEN_CACHE_NAMESPACE,
+      throttleInSeconds: LAST_SEEN_THROTTLE_SECONDS,
+      liveness: liveness,
+      metadata: metadata,
+      fingerprint: extrasFingerprint,
+      describe: `kubernetes cluster ${clusterId.toString()}`,
     });
   }
 

package/Server/Services/KubernetesContainerService.ts

@@ -1,7 +1,12 @@
 import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/KubernetesContainer";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ColumnLength from "../../Types/Database/ColumnLength";
 import ObjectID from "../../Types/ObjectID";
+import {
+  truncateLongText,
+  truncateShortText,
+} from "../Utils/Database/TruncateColumnValue";
 import logger from "../Utils/Logger";
 
 /*
@@ -61,6 +66,29 @@ const UPSERT_COLUMNS: Array<string> = [
   "version",
 ];
 
+/*
+ * KubernetesContainer's text columns are bounded varchars — podName and
+ * image at 500 chars, podNamespaceKey / name / state / reason at 100.
+ * The bulk paths below go through manager.query(), which skips the
+ * length validation DatabaseService applies to ordinary writes, so a
+ * single oversized value aborts the whole 500-row INSERT chunk it rides
+ * in. Clamp per value instead, so one pathological pod can never drop
+ * the rest of the snapshot.
+ */
+function sanitizeContainer(
+  c: ParsedKubernetesContainer,
+): ParsedKubernetesContainer {
+  return {
+    ...c,
+    podNamespaceKey: truncateShortText(c.podNamespaceKey),
+    podName: truncateLongText(c.podName),
+    name: truncateShortText(c.name),
+    image: truncateLongText(c.image),
+    state: truncateShortText(c.state),
+    reason: truncateShortText(c.reason),
+  };
+}
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
@@ -86,12 +114,20 @@ export class Service extends DatabaseService<Model> {
       return;
     }
 
-    for (
-      let i: number = 0;
-      i < data.containers.length;
-      i += UPSERT_BATCH_SIZE
-    ) {
-      const chunk: Array<ParsedKubernetesContainer> = data.containers.slice(
+    const containers: Array<ParsedKubernetesContainer> = data.containers.map(
+      (c: ParsedKubernetesContainer) => {
+        const sanitized: ParsedKubernetesContainer = sanitizeContainer(c);
+        if (sanitized.podName !== c.podName) {
+          logger.warn(
+            `KubernetesContainer podName exceeds ${ColumnLength.LongText} chars; truncated to "${sanitized.podName}" (cluster ${data.kubernetesClusterId.toString()}).`,
+          );
+        }
+        return sanitized;
+      },
+    );
+
+    for (let i: number = 0; i < containers.length; i += UPSERT_BATCH_SIZE) {
+      const chunk: Array<ParsedKubernetesContainer> = containers.slice(
         i,
         i + UPSERT_BATCH_SIZE,
       );
@@ -224,10 +260,15 @@ export class Service extends DatabaseService<Model> {
         valueFragments.push(
           `($${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}::numeric, $${paramIndex++}::bigint, $${paramIndex++}::timestamptz)`,
         );
+        /*
+         * The identity clamps MUST match bulkUpsert's, or this mirror
+         * UPDATE would silently miss the row the upsert just wrote
+         * under the truncated podName.
+         */
         params.push(
-          m.podNamespaceKey,
-          m.podName,
-          m.name,
+          truncateShortText(m.podNamespaceKey),
+          truncateLongText(m.podName),
+          truncateShortText(m.name),
           m.cpuPercent !== null && m.cpuPercent !== undefined
             ? m.cpuPercent
             : null,

package/Server/Services/KubernetesResourceService.ts

@@ -1,9 +1,14 @@
 import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/KubernetesResource";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import ColumnLength from "../../Types/Database/ColumnLength";
 import ObjectID from "../../Types/ObjectID";
 import OneUptimeDate from "../../Types/Date";
 import { ParsedKubernetesResource } from "../../Types/Kubernetes/KubernetesInventoryExtractor";
+import {
+  truncateLongText,
+  truncateShortText,
+} from "../Utils/Database/TruncateColumnValue";
 import logger from "../Utils/Logger";
 
 /*
@@ -317,6 +322,28 @@ const UPSERT_COLUMNS: Array<keyof ParsedKubernetesResource | string> = [
   "version",
 ];
 
+/*
+ * KubernetesResource's text columns are bounded varchars — name and
+ * controllerDeploymentName at 500 chars, kind / namespaceKey / uid /
+ * phase at 100. The bulk paths below go through manager.query(), which
+ * skips the length validation DatabaseService applies to ordinary
+ * writes, so a single oversized value aborts the whole 500-row INSERT
+ * chunk it rides in. Clamp per value instead, so one pathological
+ * object can never drop the rest of the snapshot.
+ */
+function sanitizeResource(
+  r: ParsedKubernetesResource,
+): ParsedKubernetesResource {
+  return {
+    ...r,
+    kind: truncateShortText(r.kind),
+    namespaceKey: truncateShortText(r.namespaceKey),
+    name: truncateLongText(r.name),
+    uid: truncateShortText(r.uid),
+    phase: truncateShortText(r.phase),
+  };
+}
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
@@ -338,9 +365,21 @@ export class Service extends DatabaseService<Model> {
       return;
     }
 
+    const resources: Array<ParsedKubernetesResource> = data.resources.map(
+      (r: ParsedKubernetesResource) => {
+        const sanitized: ParsedKubernetesResource = sanitizeResource(r);
+        if (sanitized.name !== r.name) {
+          logger.warn(
+            `KubernetesResource name exceeds ${ColumnLength.LongText} chars; truncated to "${sanitized.name}" (cluster ${data.kubernetesClusterId.toString()}).`,
+          );
+        }
+        return sanitized;
+      },
+    );
+
     // Chunk to keep individual statement parameter counts reasonable.
-    for (let i: number = 0; i < data.resources.length; i += UPSERT_BATCH_SIZE) {
-      const chunk: Array<ParsedKubernetesResource> = data.resources.slice(
+    for (let i: number = 0; i < resources.length; i += UPSERT_BATCH_SIZE) {
+      const chunk: Array<ParsedKubernetesResource> = resources.slice(
         i,
         i + UPSERT_BATCH_SIZE,
       );
@@ -450,10 +489,17 @@ export class Service extends DatabaseService<Model> {
         valueFragments.push(
           `($${paramIndex++}, $${paramIndex++}, $${paramIndex++}, $${paramIndex++}::numeric, $${paramIndex++}::bigint, $${paramIndex++}::numeric, $${paramIndex++}::timestamptz, $${paramIndex++}, $${paramIndex++})`,
         );
+        /*
+         * The identity clamps MUST match bulkUpsert's, or this mirror
+         * UPDATE would silently miss the row the upsert just wrote
+         * under the truncated name. controllerDeploymentName /
+         * controllerCronJobName are assigned columns, so an oversized
+         * value there aborts the chunk exactly like the INSERT path.
+         */
         params.push(
-          m.kind,
-          m.namespaceKey,
-          m.name,
+          truncateShortText(m.kind),
+          truncateShortText(m.namespaceKey),
+          truncateLongText(m.name),
           m.cpuPercent !== null && m.cpuPercent !== undefined
             ? m.cpuPercent
             : null,
@@ -464,8 +510,8 @@ export class Service extends DatabaseService<Model> {
             ? m.memoryPercent
             : null,
           m.observedAt,
-          m.controllerDeploymentName ?? null,
-          m.controllerCronJobName ?? null,
+          truncateLongText(m.controllerDeploymentName ?? null),
+          truncateShortText(m.controllerCronJobName ?? null),
         );
       }
 

package/Server/Services/LabelService.ts

@@ -1,14 +1,23 @@
 import CreateBy from "../Types/Database/CreateBy";
-import { OnCreate } from "../Types/Database/Hooks";
+import DeleteBy from "../Types/Database/DeleteBy";
+import { OnCreate, OnDelete } from "../Types/Database/Hooks";
 import QueryHelper from "../Types/Database/QueryHelper";
 import DatabaseService from "./DatabaseService";
+import ServiceLevelObjectiveMonitorRuleEngineService from "./ServiceLevelObjectiveMonitorRuleEngineService";
+import ServiceLevelObjectiveService from "./ServiceLevelObjectiveService";
+import StatusPageMonitorRuleEngineService from "./StatusPageMonitorRuleEngineService";
+import StatusPageMonitorRuleService from "./StatusPageMonitorRuleService";
+import ServiceLevelObjective from "../../Models/DatabaseModels/ServiceLevelObjective";
+import StatusPageMonitorRule from "../../Models/DatabaseModels/StatusPageMonitorRule";
 import BadDataException from "../../Types/Exception/BadDataException";
+import LIMIT_MAX from "../../Types/Database/LimitMax";
 import ObjectID from "../../Types/ObjectID";
 import Color from "../../Types/Color";
 import { BrightColors } from "../../Types/BrandColors";
 import GlobalCache from "../Infrastructure/GlobalCache";
 import Model from "../../Models/DatabaseModels/Label";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import logger from "../Utils/Logger";
 
 const AUTO_LABEL_ID_CACHE_NAMESPACE: string = "auto-label-id";
 const AUTO_LABEL_ID_CACHE_TTL_SECONDS: number = 24 * 60 * 60;
@@ -57,6 +66,149 @@ export class Service extends DatabaseService<Model> {
     return Promise.resolve({ createBy, carryForward: null });
   }
 
+  /*
+   * Deleting a label cascades its join rows away at the database level, so no
+   * service hook ever fires for the SLO label rules or status page monitor
+   * rules that referenced it. Those would keep the monitors the rule attached,
+   * with nothing left to explain why. Note the ids down while the label still
+   * exists, and re-run the (now smaller) rules once it is gone.
+   */
+  @CaptureSpan()
+  protected override async onBeforeDelete(
+    deleteBy: DeleteBy<Model>,
+  ): Promise<OnDelete<Model>> {
+    let serviceLevelObjectiveIds: Array<ObjectID> = [];
+    let statusPageMonitorRuleIds: Array<ObjectID> = [];
+
+    try {
+      const labels: Array<Model> = await this.findBy({
+        query: deleteBy.query,
+        select: {
+          _id: true,
+        },
+        limit: LIMIT_MAX,
+        skip: 0,
+        props: {
+          isRoot: true,
+        },
+      });
+
+      const labelIds: Array<ObjectID> = labels
+        .map((label: Model) => {
+          return label.id;
+        })
+        .filter((id: ObjectID | null): id is ObjectID => {
+          return Boolean(id);
+        });
+
+      if (labelIds.length > 0) {
+        const slos: Array<ServiceLevelObjective> =
+          await ServiceLevelObjectiveService.findBy({
+            query: {
+              monitorLabels: labelIds,
+            },
+            select: {
+              _id: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: {
+              isRoot: true,
+            },
+          });
+
+        serviceLevelObjectiveIds = slos
+          .map((slo: ServiceLevelObjective) => {
+            return slo.id;
+          })
+          .filter((id: ObjectID | null): id is ObjectID => {
+            return Boolean(id);
+          });
+
+        const statusPageMonitorRules: Array<StatusPageMonitorRule> =
+          await StatusPageMonitorRuleService.findBy({
+            query: {
+              monitorLabels: labelIds,
+            },
+            select: {
+              _id: true,
+            },
+            limit: LIMIT_MAX,
+            skip: 0,
+            props: {
+              isRoot: true,
+            },
+          });
+
+        statusPageMonitorRuleIds = statusPageMonitorRules
+          .map((rule: StatusPageMonitorRule) => {
+            return rule.id;
+          })
+          .filter((id: ObjectID | null): id is ObjectID => {
+            return Boolean(id);
+          });
+      }
+    } catch (err) {
+      logger.error(
+        `Error collecting label rules affected by a label delete: ${err}`,
+      );
+    }
+
+    return {
+      deleteBy,
+      carryForward: {
+        serviceLevelObjectiveIds: serviceLevelObjectiveIds,
+        statusPageMonitorRuleIds: statusPageMonitorRuleIds,
+      },
+    };
+  }
+
+  @CaptureSpan()
+  protected override async onDeleteSuccess(
+    onDelete: OnDelete<Model>,
+    _itemIdsBeforeDelete: Array<ObjectID>,
+  ): Promise<OnDelete<Model>> {
+    const carryForward: {
+      serviceLevelObjectiveIds?: Array<ObjectID>;
+      statusPageMonitorRuleIds?: Array<ObjectID>;
+    } | null = onDelete.carryForward as {
+      serviceLevelObjectiveIds?: Array<ObjectID>;
+      statusPageMonitorRuleIds?: Array<ObjectID>;
+    } | null;
+
+    const serviceLevelObjectiveIds: Array<ObjectID> =
+      carryForward?.serviceLevelObjectiveIds || [];
+
+    for (const serviceLevelObjectiveId of serviceLevelObjectiveIds) {
+      try {
+        await ServiceLevelObjectiveMonitorRuleEngineService.syncMonitorsForSlo({
+          serviceLevelObjectiveId: serviceLevelObjectiveId,
+        });
+      } catch (err) {
+        logger.error(
+          `Error re-applying the monitor label rule for SLO ${serviceLevelObjectiveId.toString()} after a label delete: ${err}`,
+        );
+      }
+    }
+
+    const statusPageMonitorRuleIds: Array<ObjectID> =
+      carryForward?.statusPageMonitorRuleIds || [];
+
+    for (const statusPageMonitorRuleId of statusPageMonitorRuleIds) {
+      try {
+        await StatusPageMonitorRuleEngineService.syncResourcesForRule({
+          statusPageMonitorRuleId: statusPageMonitorRuleId,
+        });
+      } catch (err) {
+        logger.error(
+          `Error re-applying status page monitor rule ${statusPageMonitorRuleId.toString()} after a label delete: ${err}`,
+        );
+      }
+    }
+
+    return onDelete;
+  }
+
   /**
    * Find existing labels by name (case-insensitive) or auto-create
    * them, returning the ObjectIDs in the same order as the input.

package/Server/Services/MetricTypeService.ts

@@ -1,10 +1,94 @@
 import DatabaseService from "./DatabaseService";
 import Model from "../../Models/DatabaseModels/MetricType";
+import ObjectID from "../../Types/ObjectID";
+import BadDataException from "../../Types/Exception/BadDataException";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import { RelationMetadata } from "typeorm/metadata/RelationMetadata";
 
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
   }
+
+  /*
+   * Additively associate services with a metric type, in ONE statement.
+   *
+   * This replaces routing the association through `updateOneById` with the
+   * whole `services` array in the payload. That looked innocuous and was not:
+   * `services` is a `TableColumnType.EntityArray`, and its mere PRESENCE as a
+   * key flips `hasRelationUpdates` in DatabaseService, which routes the write
+   * to `getRepository().save()` — a real BEGIN/COMMIT that reloads the entity,
+   * loads the relation ids, bumps `version`, and DELETEs then re-INSERTs
+   * junction rows, holding the MetricType row's write lock across every one of
+   * those round trips. On the ingest path, where this runs per metric name per
+   * batch with no backpressure, it was the only multi-round-trip lock hold in
+   * the pipeline.
+   *
+   * The whole-array write was also silently LOSING data. `save()` diffs the
+   * array it is given against what is currently in the database and deletes
+   * anything missing — but each ingest worker only knows the services in ITS
+   * batch. Two workers with different batches therefore deleted each other's
+   * associations and re-inserted them on the next batch: permanent junction
+   * churn, and real service-to-metric links disappearing from the UI in
+   * between. Additive insert makes that impossible to express.
+   *
+   * ON CONFLICT DO NOTHING is well-defined here: the junction's primary key is
+   * exactly (metricTypeId, serviceId), so a concurrent writer adding the same
+   * association is a no-op rather than a unique violation.
+   */
+  @CaptureSpan()
+  public async attachServices(data: {
+    metricTypeId: ObjectID;
+    serviceIds: Array<ObjectID>;
+  }): Promise<void> {
+    if (!data.metricTypeId) {
+      throw new BadDataException("metricTypeId is required");
+    }
+
+    if (!data.serviceIds || data.serviceIds.length === 0) {
+      return;
+    }
+
+    /*
+     * Identifiers come from entity metadata, never from the caller, and every
+     * value is bound as a parameter — the same rule the other raw-SQL paths in
+     * DatabaseService follow.
+     */
+    const relation: RelationMetadata | undefined =
+      this.getRepository().metadata.findRelationWithPropertyPath("services");
+
+    const junction: string | undefined =
+      relation?.junctionEntityMetadata?.tableName;
+    const metricTypeColumn: string | undefined =
+      relation?.junctionEntityMetadata?.columns[0]?.databaseName;
+    const serviceColumn: string | undefined =
+      relation?.junctionEntityMetadata?.columns[1]?.databaseName;
+
+    if (!junction || !metricTypeColumn || !serviceColumn) {
+      throw new BadDataException(
+        "MetricTypeService.attachServices: the services relation has no junction metadata",
+      );
+    }
+
+    /*
+     * Deduplicate and SORT. Concurrent statements that insert overlapping sets
+     * acquire their row locks in the same order this way, which is what keeps
+     * two workers associating the same services from deadlocking each other.
+     */
+    const serviceIds: Array<string> = Array.from(
+      new Set(
+        data.serviceIds.map((id: ObjectID) => {
+          return id.toString();
+        }),
+      ),
+    ).sort();
+
+    await this.getRepository().manager.query(
+      `INSERT INTO "${junction}" ("${metricTypeColumn}", "${serviceColumn}") ` +
+        `SELECT $1, unnest($2::uuid[]) ON CONFLICT DO NOTHING`,
+      [data.metricTypeId.toString(), serviceIds],
+    );
+  }
 }
 
 export default new Service();