@oneuptime/common 10.8.1 → 11.0.0

package/Server/Services/TelemetryEntityService.ts

@@ -0,0 +1,246 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/TelemetryEntity";
+import ObjectID from "../../Types/ObjectID";
+import OneUptimeDate from "../../Types/Date";
+import ColumnLength from "../../Types/Database/ColumnLength";
+import QueryDeepPartialEntity from "../../Types/Database/PartialEntity";
+import { JSONObject } from "../../Types/JSON";
+import EntityType from "../../Types/Telemetry/EntityType";
+import logger from "../Utils/Logger";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import { ExtractedEntity } from "../Utils/Telemetry/TelemetryEntity";
+import {
+  getEntityBudget,
+  reconcileByNaturalKey,
+  REGISTRY_PROMOTED_TYPES,
+  shouldWarnEntityBudgetOnce,
+} from "../Utils/Telemetry/EntityRegistry";
+
+export class TelemetryEntityService extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  /**
+   * Forward-only registry reconciliation: upsert a catalog row per
+   * discovered entity, bump `lastSeenAt`, merge descriptive attributes
+   * (last-writer-wins) and union labels. High-churn membership-only
+   * types (container / process / service.instance / telemetry.sdk — see
+   * `REGISTRY_PROMOTED_TYPES`) are skipped: their keys still flow into
+   * the `entityKeys` column on signals, they just never mint registry
+   * rows. New rows are additionally gated by a per-(project, type)
+   * entity budget. Resilient by design — a registry failure must never break
+   * signal ingest, so every error is swallowed (logged). Callers are
+   * expected to throttle this (it should not run on every batch); see
+   * `reconcileEntityRegistryThrottled` in
+   * `Common/Server/Utils/Telemetry/EntityRegistry`.
+   */
+  @CaptureSpan()
+  public async reconcileEntities(data: {
+    projectId: ObjectID;
+    entities: Array<ExtractedEntity>;
+  }): Promise<void> {
+    /*
+     * Per-reconcile budget count cache: at most one countBy per entity
+     * type per reconcile (the fence already throttles reconciles to one
+     * per entity-set per window, so this stays cheap). Creates within
+     * this reconcile increment the cached count so a single batch cannot
+     * blow past the budget.
+     */
+    const countByType: Map<EntityType, number> = new Map<EntityType, number>();
+
+    for (const entity of data.entities) {
+      if (!REGISTRY_PROMOTED_TYPES.has(entity.entityType)) {
+        continue;
+      }
+      try {
+        await this.upsertEntity({
+          projectId: data.projectId,
+          entity,
+          countByType,
+        });
+      } catch (err) {
+        logger.error(
+          `TelemetryEntityService: failed to upsert entity ${entity.entityType}/${entity.entityKey}:`,
+        );
+        logger.error(err as Error);
+      }
+    }
+  }
+
+  private async upsertEntity(data: {
+    projectId: ObjectID;
+    entity: ExtractedEntity;
+    countByType: Map<EntityType, number>;
+  }): Promise<void> {
+    const { projectId, entity, countByType } = data;
+    const now: Date = OneUptimeDate.getCurrentDate();
+
+    await reconcileByNaturalKey({
+      service: this,
+      query: {
+        projectId,
+        entityType: entity.entityType,
+        entityKey: entity.entityKey,
+      },
+      lastSeenAt: now,
+      describe: `entity ${entity.entityType}/${entity.entityKey}`,
+      select: { descriptiveAttributes: true, labels: true },
+      buildUpdate: (existing: Model): QueryDeepPartialEntity<Model> => {
+        return TelemetryEntityService.buildDescriptiveUpdate(entity, existing);
+      },
+      /*
+       * Entity budget (doc §Edge Cases): existing rows always get their
+       * bump above; only NEW rows are gated. Counted via the
+       * (projectId, entityType) index; over budget the row is skipped —
+       * its key still flows in `entityKeys` on signals.
+       */
+      beforeCreate: async (): Promise<boolean> => {
+        const budget: number = getEntityBudget(entity.entityType);
+
+        let count: number | undefined = countByType.get(entity.entityType);
+        if (count === undefined) {
+          count = (
+            await this.countBy({
+              query: { projectId, entityType: entity.entityType },
+              props: { isRoot: true },
+            })
+          ).toNumber();
+          countByType.set(entity.entityType, count);
+        }
+
+        if (count < budget) {
+          countByType.set(entity.entityType, count + 1);
+          return true;
+        }
+
+        if (
+          await shouldWarnEntityBudgetOnce({
+            projectId,
+            entityType: entity.entityType,
+          })
+        ) {
+          logger.warn(
+            `TelemetryEntityService: entity budget reached for project ${projectId.toString()} type ${entity.entityType} (${count} >= ${budget}); skipping new registry rows. Membership keys still flow on signals.`,
+          );
+        }
+        return false;
+      },
+      buildModel: () => {
+        const model: Model = new Model();
+        model.projectId = projectId;
+        model.entityType = entity.entityType;
+        model.entityKey = entity.entityKey;
+        model.identifyingAttributes = entity.identifyingAttributes;
+        if (
+          entity.descriptiveAttributes &&
+          Object.keys(entity.descriptiveAttributes).length > 0
+        ) {
+          model.descriptiveAttributes = entity.descriptiveAttributes;
+        }
+        if (entity.labels && entity.labels.length > 0) {
+          model.labels = TelemetryEntityService.normalizeLabels(entity.labels);
+        }
+        // varchar(ShortText) column; k8s names can run to 253 chars.
+        model.displayName = TelemetryEntityService.deriveDisplayName(
+          entity,
+        ).substring(0, ColumnLength.ShortText);
+        model.firstSeenAt = now;
+        model.lastSeenAt = now;
+        return model;
+      },
+    });
+  }
+
+  /**
+   * Extra fields for the `lastSeenAt` bump on an existing row:
+   * descriptive attributes merge last-writer-wins per key, labels merge
+   * as a set union. Both are written ONLY when the merge actually changes
+   * the stored value, so a stable resource keeps its bump a one-column
+   * update (no jsonb churn).
+   */
+  private static buildDescriptiveUpdate(
+    entity: ExtractedEntity,
+    existing: Model,
+  ): QueryDeepPartialEntity<Model> {
+    /*
+     * Built as a plain shape and cast once on return: assigning into
+     * QueryDeepPartialEntity<Model> directly forces TS to instantiate the
+     * recursive JSONObject mapping (TS2589).
+     */
+    const update: {
+      descriptiveAttributes?: JSONObject;
+      labels?: Array<string>;
+    } = {};
+
+    const incoming: Record<string, string> = entity.descriptiveAttributes || {};
+    if (Object.keys(incoming).length > 0) {
+      const current: Record<string, unknown> =
+        (existing.descriptiveAttributes as Record<string, unknown>) || {};
+
+      const differs: boolean = Object.keys(incoming).some((key: string) => {
+        return current[key] !== incoming[key];
+      });
+
+      if (differs) {
+        update.descriptiveAttributes = {
+          ...current,
+          ...incoming,
+        } as JSONObject;
+      }
+    }
+
+    const incomingLabels: Array<string> = entity.labels || [];
+    if (incomingLabels.length > 0) {
+      const currentLabels: Array<string> = existing.labels || [];
+      const currentSet: Set<string> = new Set<string>(currentLabels);
+
+      const hasNew: boolean = incomingLabels.some((label: string) => {
+        return !currentSet.has(label);
+      });
+
+      if (hasNew) {
+        update.labels = TelemetryEntityService.normalizeLabels([
+          ...currentLabels,
+          ...incomingLabels,
+        ]);
+      }
+    }
+
+    return update as unknown as QueryDeepPartialEntity<Model>;
+  }
+
+  /** Sorted + deduped for a stable stored value. */
+  private static normalizeLabels(labels: Array<string>): Array<string> {
+    return Array.from(new Set<string>(labels)).sort();
+  }
+
+  /**
+   * Best-effort human-readable name for the entity explorer: prefer the
+   * most specific `*.name` identifying attribute (e.g. k8s.pod.name over
+   * k8s.cluster.name), else the last identifying value, else the key.
+   */
+  public static deriveDisplayName(entity: ExtractedEntity): string {
+    const id: Record<string, string> = entity.identifyingAttributes || {};
+    const nameKeys: Array<string> = Object.keys(id)
+      .filter((k: string) => {
+        return k.endsWith(".name");
+      })
+      .sort((a: string, b: string) => {
+        return b.length - a.length;
+      });
+
+    if (nameKeys.length > 0 && id[nameKeys[0]!]) {
+      return id[nameKeys[0]!]!;
+    }
+
+    const keys: Array<string> = Object.keys(id);
+    if (keys.length > 0 && id[keys[keys.length - 1]!]) {
+      return id[keys[keys.length - 1]!]!;
+    }
+
+    return entity.entityKey;
+  }
+}
+
+export default new TelemetryEntityService();

package/Server/Services/TelemetryExceptionService.ts

@@ -25,10 +25,10 @@ export interface DashboardServiceSummary {
   /*
    * Polymorphic: a real Service, a Host/DockerHost/KubernetesCluster id, or
    * the projectId for unattributed telemetry. The client resolves the
-   * display name per serviceType.
+   * display name per primaryEntityType.
    */
-  serviceId: string;
-  serviceType: ServiceType | null;
+  primaryEntityId: string;
+  primaryEntityType: ServiceType | null;
   unresolvedCount: number;
   totalOccurrences: number;
 }
@@ -61,7 +61,7 @@ export class Service extends DatabaseService<Model> {
         projectId: true,
         message: true,
         stackTrace: true,
-        serviceId: true,
+        primaryEntityId: true,
         exceptionType: true,
       },
       props,
@@ -186,9 +186,9 @@ export class Service extends DatabaseService<Model> {
       lastSeenAt: true,
       firstSeenAt: true,
       environment: true,
-      // serviceId is polymorphic; the client resolves it per serviceType.
-      serviceId: true,
-      serviceType: true,
+      // primaryEntityId is polymorphic; the client resolves it per primaryEntityType.
+      primaryEntityId: true,
+      primaryEntityType: true,
     };
 
     const [
@@ -285,8 +285,8 @@ export class Service extends DatabaseService<Model> {
     );
 
     interface AggregateRow {
-      serviceId: string | null;
-      serviceType: string | null;
+      primaryEntityId: string | null;
+      primaryEntityType: string | null;
       unresolvedCount: string;
       totalOccurrences: string | null;
     }
@@ -294,8 +294,11 @@ export class Service extends DatabaseService<Model> {
     const rows: Array<AggregateRow> = (await this.getQueryBuilder(
       "TelemetryException",
     )
-      .select(`"TelemetryException"."serviceId"`, "serviceId")
-      .addSelect(`"TelemetryException"."serviceType"`, "serviceType")
+      .select(`"TelemetryException"."primaryEntityId"`, "primaryEntityId")
+      .addSelect(
+        `"TelemetryException"."primaryEntityType"`,
+        "primaryEntityType",
+      )
       .addSelect(`COUNT(*)`, "unresolvedCount")
       .addSelect(
         `COALESCE(SUM("TelemetryException"."occuranceCount"), 0)`,
@@ -307,9 +310,9 @@ export class Service extends DatabaseService<Model> {
       .andWhere(`"TelemetryException"."isResolved" = false`)
       .andWhere(`"TelemetryException"."isArchived" = false`)
       .andWhere(`"TelemetryException"."deletedAt" IS NULL`)
-      .andWhere(`"TelemetryException"."serviceId" IS NOT NULL`)
-      .groupBy(`"TelemetryException"."serviceId"`)
-      .addGroupBy(`"TelemetryException"."serviceType"`)
+      .andWhere(`"TelemetryException"."primaryEntityId" IS NOT NULL`)
+      .groupBy(`"TelemetryException"."primaryEntityId"`)
+      .addGroupBy(`"TelemetryException"."primaryEntityType"`)
       .orderBy(`"unresolvedCount"`, "DESC")
       .getRawMany()) as Array<AggregateRow>;
 
@@ -318,21 +321,22 @@ export class Service extends DatabaseService<Model> {
     }
 
     /*
-     * serviceId is polymorphic — do NOT resolve it to a Service here. The
-     * old code looked each serviceId up in the Service table and dropped
+     * primaryEntityId is polymorphic — do NOT resolve it to a Service here. The
+     * old code looked each primaryEntityId up in the Service table and dropped
      * any that didn't match, which silently excluded Host / DockerHost /
      * KubernetesCluster and unattributed (Unknown) telemetry. Return the
-     * raw (serviceId, serviceType) + counts; the client resolves the
-     * display name per serviceType.
+     * raw (primaryEntityId, primaryEntityType) + counts; the client resolves the
+     * display name per primaryEntityType.
      */
     const summaries: Array<DashboardServiceSummary> = [];
     for (const row of rows) {
-      if (!row.serviceId) {
+      if (!row.primaryEntityId) {
         continue;
       }
       summaries.push({
-        serviceId: row.serviceId,
-        serviceType: (row.serviceType as ServiceType | null) ?? null,
+        primaryEntityId: row.primaryEntityId,
+        primaryEntityType:
+          (row.primaryEntityType as ServiceType | null) ?? null,
         unresolvedCount: parseInt(row.unresolvedCount, 10) || 0,
         totalOccurrences: parseInt(row.totalOccurrences || "0", 10) || 0,
       });
@@ -360,8 +364,8 @@ export class Service extends DatabaseService<Model> {
       metadata.errorMessage = telemetryException.message;
     }
 
-    if (telemetryException.serviceId) {
-      metadata.serviceId = telemetryException.serviceId.toString();
+    if (telemetryException.primaryEntityId) {
+      metadata.serviceId = telemetryException.primaryEntityId.toString();
     }
 
     return metadata;

package/Server/Services/TelemetryUsageBillingService.ts

@@ -124,33 +124,33 @@ export class Service extends DatabaseService<Model> {
     const endOfDay: Date = OneUptimeDate.getEndOfDay(usageDate);
 
     /*
-     * Enumerate usage from ClickHouse by (serviceId, serviceType) in a
+     * Enumerate usage from ClickHouse by (primaryEntityId, primaryEntityType) in a
      * single aggregation scan per analytics table. Unlike the old
      * per-Service-row loop, this surfaces EVERY resource that emitted
      * telemetry — real Services AND Hosts / Docker hosts / Kubernetes
-     * clusters / unattributed (serviceId = projectId) — so infrastructure
+     * clusters / unattributed (primaryEntityId = projectId) — so infrastructure
      * telemetry is no longer ingested for free. `byteSize(*)` gives the
      * estimated ingested bytes per group; the configured average row size
      * is only a fallback if the engine returns 0.
      */
     type ServiceUsage = {
-      serviceType: string | null;
+      primaryEntityType: string | null;
       bytes: number;
     };
     const usageByServiceId: Map<string, ServiceUsage> = new Map();
 
     const addUsage: (
       rows: Array<{
-        serviceId: string;
-        serviceType: string | null;
+        primaryEntityId: string;
+        primaryEntityType: string | null;
         rowCount: number;
         estimatedBytes: number;
       }>,
       fallbackRowSizeInBytes: number,
     ) => void = (
       rows: Array<{
-        serviceId: string;
-        serviceType: string | null;
+        primaryEntityId: string;
+        primaryEntityType: string | null;
         rowCount: number;
         estimatedBytes: number;
       }>,
@@ -162,16 +162,16 @@ export class Service extends DatabaseService<Model> {
             ? row.estimatedBytes
             : row.rowCount * fallbackRowSizeInBytes;
         const existing: ServiceUsage | undefined = usageByServiceId.get(
-          row.serviceId,
+          row.primaryEntityId,
         );
         if (existing) {
           existing.bytes += bytes;
-          if (!existing.serviceType && row.serviceType) {
-            existing.serviceType = row.serviceType;
+          if (!existing.primaryEntityType && row.primaryEntityType) {
+            existing.primaryEntityType = row.primaryEntityType;
           }
         } else {
-          usageByServiceId.set(row.serviceId, {
-            serviceType: row.serviceType,
+          usageByServiceId.set(row.primaryEntityId, {
+            primaryEntityType: row.primaryEntityType,
             bytes,
           });
         }
@@ -243,7 +243,13 @@ export class Service extends DatabaseService<Model> {
         `Failed to aggregate telemetry usage for project ${data.projectId.toString()} (${data.productType}):`,
       );
       logger.error(error as Error);
-      return;
+      /*
+       * Rethrow so the failure is loud: nothing has been staged for this
+       * day yet (staging below is skipped entirely), so a later retry of
+       * the cron can re-stage the full day. Swallowing here would
+       * silently bill the day as zero.
+       */
+      throw error;
     }
 
     if (usageByServiceId.size === 0) {
@@ -264,7 +270,7 @@ export class Service extends DatabaseService<Model> {
     /*
      * Skip serviceIds already staged for this day so re-runs of the cron
      * don't double-count (updateUsageBilling accumulates into the day's
-     * row). One query instead of a findOneBy per serviceId.
+     * row). One query instead of a findOneBy per primaryEntityId.
      */
     const alreadyStaged: Array<Model> = await this.findBy({
       query: {
@@ -273,7 +279,7 @@ export class Service extends DatabaseService<Model> {
         day: usageDayString,
       },
       select: {
-        serviceId: true,
+        primaryEntityId: true,
       },
       skip: 0,
       limit: LIMIT_MAX,
@@ -284,7 +290,7 @@ export class Service extends DatabaseService<Model> {
     const stagedServiceIds: Set<string> = new Set(
       alreadyStaged
         .map((row: Model) => {
-          return row.serviceId?.toString();
+          return row.primaryEntityId?.toString();
         })
         .filter((id: string | undefined): id is string => {
           return Boolean(id);
@@ -298,9 +304,9 @@ export class Service extends DatabaseService<Model> {
        * neither is charged as ingested telemetry volume.
        */
       if (
-        usage.serviceType === ServiceType.Monitor ||
-        usage.serviceType === ServiceType.Alert ||
-        usage.serviceType === ServiceType.Incident
+        usage.primaryEntityType === ServiceType.Monitor ||
+        usage.primaryEntityType === ServiceType.Alert ||
+        usage.primaryEntityType === ServiceType.Incident
       ) {
         continue;
       }
@@ -318,18 +324,19 @@ export class Service extends DatabaseService<Model> {
         retentionByServiceId.get(serviceIdStr) ?? projectDefaultRetentionInDays;
 
       /*
-       * Legacy rows (pre-discriminator) and any null serviceType are
-       * treated as OpenTelemetry — historically every billed serviceId was
+       * Legacy rows (pre-discriminator) and any null primaryEntityType are
+       * treated as OpenTelemetry — historically every billed primaryEntityId was
        * a real Service.
        */
-      const serviceType: ServiceType =
-        (usage.serviceType as ServiceType | null) ?? ServiceType.OpenTelemetry;
+      const primaryEntityType: ServiceType =
+        (usage.primaryEntityType as ServiceType | null) ??
+        ServiceType.OpenTelemetry;
 
       await this.updateUsageBilling({
         projectId: data.projectId,
         productType: data.productType,
-        serviceId: new ObjectID(serviceIdStr),
-        serviceType: serviceType,
+        primaryEntityId: new ObjectID(serviceIdStr),
+        primaryEntityType: primaryEntityType,
         dataIngestedInGB: estimatedGigabytes,
         retentionInDays: retentionInDays,
         usageDate: usageDate,
@@ -436,8 +443,8 @@ export class Service extends DatabaseService<Model> {
   public async updateUsageBilling(data: {
     projectId: ObjectID;
     productType: ProductType;
-    serviceId: ObjectID;
-    serviceType?: ServiceType | undefined;
+    primaryEntityId: ObjectID;
+    primaryEntityType?: ServiceType | undefined;
     dataIngestedInGB: number;
     retentionInDays: number;
     usageDate?: Date;
@@ -474,7 +481,7 @@ export class Service extends DatabaseService<Model> {
       query: {
         projectId: data.projectId,
         productType: data.productType,
-        serviceId: data.serviceId,
+        primaryEntityId: data.primaryEntityId,
         isReportedToBillingProvider: false,
         day: usageDayString,
       },
@@ -523,9 +530,9 @@ export class Service extends DatabaseService<Model> {
       usageBilling.projectId = data.projectId;
       usageBilling.productType = data.productType;
       usageBilling.dataIngestedInGB = new Decimal(data.dataIngestedInGB);
-      usageBilling.serviceId = data.serviceId;
-      if (data.serviceType) {
-        usageBilling.serviceType = data.serviceType;
+      usageBilling.primaryEntityId = data.primaryEntityId;
+      if (data.primaryEntityType) {
+        usageBilling.primaryEntityType = data.primaryEntityType;
       }
       usageBilling.retainTelemetryDataForDays = data.retentionInDays;
       usageBilling.isReportedToBillingProvider = false;