@oneuptime/common 10.8.2 → 11.0.0

package/Server/Services/AlertService.ts

@@ -1317,7 +1317,7 @@ ${alertSeverity.name}
     // delete all the alert metrics with this alert id because it's a refresh
     await MetricService.deleteBy({
       query: {
-        serviceId: data.alertId,
+        primaryEntityId: data.alertId,
       },
       props: {
         isRoot: true,
@@ -1342,8 +1342,8 @@ ${alertSeverity.name}
     const alertCountMetric: Metric = new Metric();
 
     alertCountMetric.projectId = alert.projectId;
-    alertCountMetric.serviceId = alert.id!;
-    alertCountMetric.serviceType = ServiceType.Alert;
+    alertCountMetric.primaryEntityId = alert.id!;
+    alertCountMetric.primaryEntityType = ServiceType.Alert;
     alertCountMetric.name = AlertMetricType.AlertCount;
     alertCountMetric.value = 1;
     alertCountMetric.attributes = {
@@ -1391,8 +1391,8 @@ ${alertSeverity.name}
         const timeToAcknowledgeMetric: Metric = new Metric();
 
         timeToAcknowledgeMetric.projectId = alert.projectId;
-        timeToAcknowledgeMetric.serviceId = alert.id!;
-        timeToAcknowledgeMetric.serviceType = ServiceType.Alert;
+        timeToAcknowledgeMetric.primaryEntityId = alert.id!;
+        timeToAcknowledgeMetric.primaryEntityType = ServiceType.Alert;
         timeToAcknowledgeMetric.name = AlertMetricType.TimeToAcknowledge;
         timeToAcknowledgeMetric.value = OneUptimeDate.getDifferenceInSeconds(
           ackAlertStateTimeline?.startsAt || OneUptimeDate.getCurrentDate(),
@@ -1447,8 +1447,8 @@ ${alertSeverity.name}
         const timeToResolveMetric: Metric = new Metric();
 
         timeToResolveMetric.projectId = alert.projectId;
-        timeToResolveMetric.serviceId = alert.id!;
-        timeToResolveMetric.serviceType = ServiceType.Alert;
+        timeToResolveMetric.primaryEntityId = alert.id!;
+        timeToResolveMetric.primaryEntityType = ServiceType.Alert;
         timeToResolveMetric.name = AlertMetricType.TimeToResolve;
         timeToResolveMetric.value = OneUptimeDate.getDifferenceInSeconds(
           resolvedAlertStateTimeline?.startsAt ||
@@ -1498,8 +1498,8 @@ ${alertSeverity.name}
         lastAlertStateTimeline.startsAt || OneUptimeDate.getCurrentDate();
 
       alertDurationMetric.projectId = alert.projectId;
-      alertDurationMetric.serviceId = alert.id!;
-      alertDurationMetric.serviceType = ServiceType.Alert;
+      alertDurationMetric.primaryEntityId = alert.id!;
+      alertDurationMetric.primaryEntityType = ServiceType.Alert;
       alertDurationMetric.name = AlertMetricType.AlertDuration;
       alertDurationMetric.value = OneUptimeDate.getDifferenceInSeconds(
         alertEndsAt,

package/Server/Services/AnalyticsDatabaseService.ts

@@ -28,11 +28,18 @@ import Select from "../Types/AnalyticsDatabase/Select";
 import UpdateBy from "../Types/AnalyticsDatabase/UpdateBy";
 import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement";
 import StatementGenerator from "../Utils/AnalyticsDatabase/StatementGenerator";
+import { getQuerySettings } from "../Utils/AnalyticsDatabase/QuerySettingsHelper";
 import logger, { LogAttributes } from "../Utils/Logger";
 import Realtime from "../Utils/Realtime";
 import StreamUtil from "../Utils/Stream";
 import BaseService from "./BaseService";
-import { ExecResult, ResponseJSON, ResultSet } from "@clickhouse/client";
+import {
+  ClickHouseSettings,
+  ExecResult,
+  ResponseJSON,
+  ResultSet,
+} from "@clickhouse/client";
+import { AsyncLocalStorage } from "node:async_hooks";
 import AnalyticsBaseModel from "../../Models/AnalyticsModels/AnalyticsBaseModel/AnalyticsBaseModel";
 import { WorkflowRoute } from "../../ServiceRoute";
 import Protocol from "../../Types/API/Protocol";
@@ -64,6 +71,65 @@ export type DbJSONResponse = ResponseJSON<{
   data?: Array<JSONObject>;
 }>;
 
+/*
+ * Re-exported so callers outside Common (e.g. App data migrations) can type
+ * per-call settings without depending on @clickhouse/client directly.
+ */
+export type { ClickHouseSettings } from "@clickhouse/client";
+
+/**
+ * Optional per-call knobs for `execute` / `executeQuery`, threaded into
+ * `client.exec` / `client.query`. Long-running statements (e.g. the
+ * telemetry V3 backfill's INSERT...SELECT chunks) need per-call
+ * `clickhouse_settings` — notably `send_progress_in_http_headers`, which
+ * keeps the HTTP socket non-idle so the client's `request_timeout`
+ * (enforced as a socket *idle* timer, see ClickhouseConfig.ts) never
+ * destroys a healthy request — and a deterministic `query_id` so a retry
+ * can find a still-running or already-finished predecessor in
+ * `system.processes` / `system.query_log`. Additive: callers that pass
+ * nothing get the exact pre-existing behavior.
+ */
+export interface ClickhouseExecuteOptions {
+  clickhouseSettings?: ClickHouseSettings | undefined;
+  queryId?: string | undefined;
+}
+
+/**
+ * Ambient context that makes ClickHouse inserts idempotent across queue
+ * retries. The telemetry queue worker wraps each job in
+ * `runWithInsertDedup(jobId, ...)`; every insertJsonRows call inside the
+ * job then stamps `insert_deduplication_token =
+ * "<tokenBase>:<table>:<chunkIndex>"` plus async_insert_deduplicate=1 /
+ * wait_for_async_insert=1, so a stalled-job retry that re-processes the
+ * same payload re-issues byte-identical tokens and ClickHouse drops the
+ * duplicate blocks (on replicated tables; on plain MergeTree the token is
+ * ignored unless non_replicated_deduplication_window is set — no harm
+ * either way). The chunk counter is per table because one job inserts
+ * into several tables (e.g. Span + ExceptionInstance) in a deterministic
+ * order.
+ *
+ * HTTP-path inserts run outside the context and keep the fire-and-forget
+ * async insert (wait_for_async_insert=0) — dedup waiting is only
+ * affordable off the request thread.
+ */
+export interface InsertDedupContextStore {
+  tokenBase: string;
+  chunkIndexByTable: Map<string, number>;
+}
+
+const insertDedupContext: AsyncLocalStorage<InsertDedupContextStore> =
+  new AsyncLocalStorage<InsertDedupContextStore>();
+
+export function runWithInsertDedup<T>(
+  tokenBase: string,
+  fn: () => Promise<T>,
+): Promise<T> {
+  return insertDedupContext.run(
+    { tokenBase, chunkIndexByTable: new Map<string, number>() },
+    fn,
+  );
+}
+
 export default class AnalyticsDatabaseService<
   TBaseModel extends AnalyticsBaseModel,
 > extends BaseService {
@@ -105,7 +171,20 @@ export default class AnalyticsDatabaseService<
   }
 
   @CaptureSpan()
-  public async insertJsonRows(rows: Array<JSONObject>): Promise<void> {
+  public async insertJsonRows(
+    rows: Array<JSONObject>,
+    options?: {
+      /**
+       * Explicit deduplication token for this insert. Overrides the
+       * ambient runWithInsertDedup context. Callers must guarantee the
+       * token is stable across retries of the same logical insert and
+       * unique otherwise.
+       */
+      dedupToken?: string | undefined;
+      /** Extra per-insert ClickHouse settings, merged last. */
+      clickhouseSettings?: ClickHouseSettings | undefined;
+    },
+  ): Promise<void> {
     if (!rows || rows.length === 0) {
       return;
     }
@@ -121,15 +200,51 @@ export default class AnalyticsDatabaseService<
       );
     }
 
+    let dedupToken: string | undefined = options?.dedupToken;
+
+    if (!dedupToken) {
+      const dedupStore: InsertDedupContextStore | undefined =
+        insertDedupContext.getStore();
+      if (dedupStore) {
+        const chunkIndex: number =
+          dedupStore.chunkIndexByTable.get(tableName) ?? 0;
+        dedupStore.chunkIndexByTable.set(tableName, chunkIndex + 1);
+        dedupToken = `${dedupStore.tokenBase}:${tableName}:${chunkIndex}`;
+      }
+    }
+
+    let clickhouseSettings: ClickHouseSettings = {
+      async_insert: 1,
+      wait_for_async_insert: 0,
+    };
+
+    if (dedupToken) {
+      /*
+       * wait_for_async_insert=1 so the worker only acks the job after
+       * the block actually landed (or was deduplicated) — otherwise a
+       * crash between buffer-write and flush loses data with no retry.
+       */
+      clickhouseSettings = {
+        async_insert: 1,
+        wait_for_async_insert: 1,
+        async_insert_deduplicate: 1,
+        insert_deduplication_token: dedupToken,
+      };
+    }
+
+    if (options?.clickhouseSettings) {
+      clickhouseSettings = {
+        ...clickhouseSettings,
+        ...options.clickhouseSettings,
+      };
+    }
+
     try {
       await client.insert({
         table: tableName,
         values: rows,
         format: "JSONEachRow",
-        clickhouse_settings: {
-          async_insert: 1,
-          wait_for_async_insert: 0,
-        },
+        clickhouse_settings: clickhouseSettings,
       });
 
       logger.debug(
@@ -441,13 +556,13 @@ export default class AnalyticsDatabaseService<
   }
 
   /**
-   * Group telemetry rows by (serviceId, serviceType) for a project over a
+   * Group telemetry rows by (primaryEntityId, primaryEntityType) for a project over a
    * time window, returning the row count and an estimate of the ingested
    * byte size (ClickHouse `byteSize(*)`, the uncompressed in-memory size of
    * each row's columns). This is the enumeration source for usage billing:
    * a single aggregation scan surfaces EVERY resource that emitted
    * telemetry — real Services, Hosts, Docker hosts, Kubernetes clusters,
-   * Monitors and unattributed (serviceId = projectId) — without needing a
+   * Monitors and unattributed (primaryEntityId = projectId) — without needing a
    * Postgres row per resource. The caller decides which serviceTypes to
    * bill and how to attribute retention.
    */
@@ -459,8 +574,8 @@ export default class AnalyticsDatabaseService<
     endDate: Date;
   }): Promise<
     Array<{
-      serviceId: string;
-      serviceType: string | null;
+      primaryEntityId: string;
+      primaryEntityType: string | null;
       rowCount: number;
       estimatedBytes: number;
     }>
@@ -479,7 +594,7 @@ export default class AnalyticsDatabaseService<
     const databaseName: string =
       this.database!.getDatasourceOptions().database!;
 
-    const statement: Statement = SQL`SELECT serviceId AS serviceId, serviceType AS serviceType, count() AS rowCount, sum(byteSize(*)) AS estimatedBytes FROM ${databaseName}.${this.model.tableName} WHERE projectId = ${{
+    const statement: Statement = SQL`SELECT primaryEntityId AS primaryEntityId, primaryEntityType AS primaryEntityType, count() AS rowCount, sum(byteSize(*)) AS estimatedBytes FROM ${databaseName}.${this.model.tableName} WHERE projectId = ${{
       type: TableColumnType.ObjectID,
       value: data.projectId,
     }} AND ${timestampColumnName} >= ${{
@@ -488,11 +603,16 @@ export default class AnalyticsDatabaseService<
     }} AND ${timestampColumnName} <= ${{
       type: TableColumnType.DateTime64,
       value: data.endDate,
-    }} GROUP BY serviceId, serviceType`;
+    }} GROUP BY primaryEntityId, primaryEntityType`;
 
-    statement.append(
-      " SETTINGS max_execution_time = 60, timeout_overflow_mode = 'break'",
-    );
+    /*
+     * Billing scan: deliberately NO timeout_overflow_mode='break'. A
+     * partial aggregation here silently undercounts usage (rows that
+     * weren't scanned before the cap simply never get billed). Failing
+     * loudly lets the staging cron retry instead; the cap is raised to
+     * compensate for the full-day scan on large projects.
+     */
+    statement.append(getQuerySettings({ maxExecutionTimeInSeconds: 120 }));
 
     const dbResult: ResultSet<"JSON"> = await this.executeQuery(statement);
     const responseJSON: ResponseJSON<JSONObject> =
@@ -500,25 +620,25 @@ export default class AnalyticsDatabaseService<
     const items: Array<JSONObject> = responseJSON.data ? responseJSON.data : [];
 
     const results: Array<{
-      serviceId: string;
-      serviceType: string | null;
+      primaryEntityId: string;
+      primaryEntityType: string | null;
       rowCount: number;
       estimatedBytes: number;
     }> = [];
 
     for (const item of items) {
-      const serviceId: string = (item["serviceId"] as string) || "";
-      if (!serviceId) {
+      const primaryEntityId: string = (item["primaryEntityId"] as string) || "";
+      if (!primaryEntityId) {
         continue;
       }
-      const serviceTypeRaw: unknown = item["serviceType"];
-      const serviceType: string | null =
+      const serviceTypeRaw: unknown = item["primaryEntityType"];
+      const primaryEntityType: string | null =
         typeof serviceTypeRaw === "string" && serviceTypeRaw.trim()
           ? serviceTypeRaw
           : null;
       results.push({
-        serviceId,
-        serviceType,
+        primaryEntityId,
+        primaryEntityType,
         rowCount: Number(item["rowCount"]) || 0,
         estimatedBytes: Number(item["estimatedBytes"]) || 0,
       });
@@ -847,6 +967,25 @@ export default class AnalyticsDatabaseService<
     return Promise.resolve({ findBy, carryForward: null });
   }
 
+  /**
+   * Read-side retention filter. TTL is `retentionDate DELETE` with
+   * ttl_only_drop_parts=1, so a part survives until EVERY row in it has
+   * expired — rows past their per-service retention stay on disk (and
+   * were queryable) for up to a partition's worth of extra time. For
+   * models that carry a retentionDate column, every centrally generated
+   * read appends this predicate so expired rows become invisible the
+   * moment they expire rather than when their part finally drops.
+   *
+   * Returns the raw SQL fragment (server-evaluated now(), no parameter)
+   * or "" when the model has no retentionDate column.
+   */
+  protected getRetentionReadFilter(): string {
+    if (!this.model.getTableColumn("retentionDate")) {
+      return "";
+    }
+    return " AND retentionDate >= now()";
+  }
+
   public toCountStatement(countBy: CountBy<TBaseModel>): Statement {
     if (!this.database) {
       this.useDefaultDatabase();
@@ -877,7 +1016,8 @@ export default class AnalyticsDatabaseService<
             FROM ${databaseName}.${this.model.tableName}
             WHERE TRUE `
       )
-      .append(whereStatement);
+      .append(whereStatement)
+      .append(this.getRetentionReadFilter());
 
     if (countBy.limit) {
       statement.append(SQL`
@@ -906,7 +1046,10 @@ export default class AnalyticsDatabaseService<
      * throwing, which is acceptable for pagination display.
      */
     statement.append(
-      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+      getQuerySettings({
+        maxExecutionTimeInSeconds: 45,
+        timeoutOverflowMode: "break",
+      }),
     );
 
     logger.debug(`${this.model.tableName} Count Statement`, { tableName: this.model.tableName } as LogAttributes);
@@ -941,11 +1084,13 @@ export default class AnalyticsDatabaseService<
     const statement: Statement = SQL`
             SELECT 1 as existsFlag
             FROM ${databaseName}.${this.model.tableName}
-            WHERE TRUE `.append(whereStatement);
+            WHERE TRUE `
+      .append(whereStatement)
+      .append(this.getRetentionReadFilter());
 
     statement.append(SQL` LIMIT 1`);
 
-    statement.append(" SETTINGS max_execution_time = 45");
+    statement.append(getQuerySettings({ maxExecutionTimeInSeconds: 45 }));
 
     logger.debug(`${this.model.tableName} Exists Statement`, { tableName: this.model.tableName } as LogAttributes);
     logger.debug(statement, { tableName: this.model.tableName } as LogAttributes);
@@ -978,7 +1123,10 @@ export default class AnalyticsDatabaseService<
 
     statement.append(SQL`SELECT `.append(select.statement));
     statement.append(SQL` FROM ${databaseName}.${this.model.tableName}`);
-    statement.append(SQL` WHERE TRUE `).append(whereStatement);
+    statement
+      .append(SQL` WHERE TRUE `)
+      .append(whereStatement)
+      .append(this.getRetentionReadFilter());
 
     statement
       .append(SQL` GROUP BY `)
@@ -1025,7 +1173,13 @@ export default class AnalyticsDatabaseService<
      *   ranges, but cluster headroom is preserved under burst.
      */
     statement.append(
-      ` SETTINGS optimize_aggregation_in_order=1, optimize_move_to_prewhere=1, max_threads=4`,
+      getQuerySettings({
+        additionalSettings: {
+          optimize_aggregation_in_order: 1,
+          optimize_move_to_prewhere: 1,
+          max_threads: 4,
+        },
+      }),
     );
 
     logger.debug(`${this.model.tableName} Aggregate Statement`, { tableName: this.model.tableName } as LogAttributes);
@@ -1071,7 +1225,10 @@ export default class AnalyticsDatabaseService<
 
     statement.append(SQL`SELECT `.append(select.statement));
     statement.append(SQL` FROM ${databaseName}.${this.model.tableName}`);
-    statement.append(SQL` WHERE TRUE `).append(whereStatement);
+    statement
+      .append(SQL` WHERE TRUE `)
+      .append(whereStatement)
+      .append(this.getRetentionReadFilter());
 
     if (groupByStatement) {
       statement.append(SQL` GROUP BY `).append(groupByStatement);
@@ -1100,7 +1257,10 @@ export default class AnalyticsDatabaseService<
      * partial results rather than throwing.
      */
     statement.append(
-      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+      getQuerySettings({
+        maxExecutionTimeInSeconds: 45,
+        timeoutOverflowMode: "break",
+      }),
     );
 
     logger.debug(`${this.model.tableName} Find Statement`, { tableName: this.model.tableName } as LogAttributes);
@@ -1276,7 +1436,8 @@ export default class AnalyticsDatabaseService<
 
   @CaptureSpan()
   public async execute(
-    statement: Statement | string
+    statement: Statement | string,
+    options?: ClickhouseExecuteOptions
   ): Promise<ExecResult<Stream>> {
     const client: ClickhouseClient = this.getDatabaseClient();
 
@@ -1288,12 +1449,17 @@ export default class AnalyticsDatabaseService<
     return (await client.exec({
       query: query,
       query_params: queryParams || (undefined as any), // undefined is not specified in the type for query_params, but its ok to pass undefined.
+      ...(options?.clickhouseSettings
+        ? { clickhouse_settings: options.clickhouseSettings }
+        : {}),
+      ...(options?.queryId ? { query_id: options.queryId } : {}),
     })) as ExecResult<Stream>;
   }
 
   @CaptureSpan()
   public async executeQuery(
-    statement: Statement | string
+    statement: Statement | string,
+    options?: ClickhouseExecuteOptions
   ): Promise<ResultSet<"JSON">> {
     const client: ClickhouseClient = this.getDatabaseClient();
 
@@ -1306,6 +1472,10 @@ export default class AnalyticsDatabaseService<
       query: query,
       format: "JSON",
       query_params: queryParams || (undefined as any), // undefined is not specified in the type for query_params, but its ok to pass undefined.
+      ...(options?.clickhouseSettings
+        ? { clickhouse_settings: options.clickhouseSettings }
+        : {}),
+      ...(options?.queryId ? { query_id: options.queryId } : {}),
     });
   }
 
@@ -1599,11 +1769,10 @@ export default class AnalyticsDatabaseService<
     data: TBaseModel
   ): TBaseModel {
     if (!data.id) {
-      data.id = ObjectID.generate();
+      data.id = ObjectID.generateTimeOrdered();
     }
 
     data.createdAt = OneUptimeDate.getCurrentDate();
-    data.updatedAt = OneUptimeDate.getCurrentDate();
 
     return data;
   }

package/Server/Services/ExceptionAggregationService.ts

@@ -1,4 +1,5 @@
 import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement";
+import { getQuerySettings } from "../Utils/AnalyticsDatabase/QuerySettingsHelper";
 import ExceptionInstanceService from "./ExceptionInstanceService";
 import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
 import { JSONObject } from "../../Types/JSON";
@@ -52,7 +53,7 @@ export class ExceptionAggregationService {
   private static readonly TABLE_NAME: string =
     AnalyticsTableName.ExceptionInstance;
   private static readonly TOP_LEVEL_COLUMNS: Set<string> = new Set([
-    "serviceId",
+    "primaryEntityId",
     "exceptionType",
     "environment",
     "fingerprint",
@@ -63,8 +64,8 @@ export class ExceptionAggregationService {
   ]);
   /*
    * Virtual facet keys — same scheme as TraceAggregationService /
-   * LogAggregationService. The `serviceId` slot is reused for host /
-   * docker host / k8s cluster ids, disambiguated by the `serviceType`
+   * LogAggregationService. The `primaryEntityId` slot is reused for host /
+   * docker host / k8s cluster ids, disambiguated by the `primaryEntityType`
    * discriminator column on each ExceptionInstance row.
    */
   private static readonly RESOURCE_FACET_KEYS: Map<string, ServiceType> =
@@ -164,6 +165,14 @@ export class ExceptionAggregationService {
         }}
     `;
 
+    /*
+     * Read-side retention filter: rows past their per-service retention
+     * stay in their part until the whole part drops (ttl_only_drop_parts).
+     * Raw `time` predicates already make these queries ineligible for the
+     * proj_exception_group projection, so the extra predicate is free.
+     */
+    statement.append(" AND retentionDate >= now()");
+
     ExceptionAggregationService.appendCommonFilters(statement, request);
 
     statement.append(" GROUP BY bucket, escaped ORDER BY bucket ASC");
@@ -174,13 +183,21 @@ export class ExceptionAggregationService {
      * rather than throwing, which is acceptable for a density viz.
      */
     statement.append(
-      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+      getQuerySettings({
+        maxExecutionTimeInSeconds: 45,
+        timeoutOverflowMode: "break",
+      }),
     );
 
     return statement;
   }
 
   private static buildFacetStatement(request: FacetRequest): Statement {
+    // Pre-rename alias from stale clients; the V3 column is primaryEntityId.
+    if (request.facetKey === "serviceId") {
+      request.facetKey = "primaryEntityId";
+    }
+
     const limit: number =
       request.limit ?? ExceptionAggregationService.DEFAULT_FACET_LIMIT;
 
@@ -197,11 +214,12 @@ export class ExceptionAggregationService {
 
     if (isResourceFacet) {
       /*
-       * Virtual facet — group serviceId values whose row carries the matching
-       * ServiceType discriminator (Host / DockerHost / KubernetesCluster).
+       * Virtual facet — group primaryEntityId values whose row carries the
+       * matching ServiceType discriminator (Host / DockerHost /
+       * KubernetesCluster).
        */
       statement.append(
-        SQL`SELECT toString(serviceId) AS val, count() AS cnt FROM ${ExceptionAggregationService.TABLE_NAME}`,
+        SQL`SELECT toString(primaryEntityId) AS val, count() AS cnt FROM ${ExceptionAggregationService.TABLE_NAME}`,
       );
     } else if (isTopLevelColumn) {
       statement.append(
@@ -209,10 +227,10 @@ export class ExceptionAggregationService {
       );
     } else {
       statement.append(
-        SQL`SELECT JSONExtractRaw(attributes, ${{
+        SQL`SELECT attributes[${{
           type: TableColumnType.Text,
           value: request.facetKey,
-        }}) AS val, count() AS cnt FROM ${ExceptionAggregationService.TABLE_NAME}`,
+        }}] AS val, count() AS cnt FROM ${ExceptionAggregationService.TABLE_NAME}`,
       );
     }
 
@@ -231,32 +249,34 @@ export class ExceptionAggregationService {
 
     if (isResourceFacet) {
       statement.append(
-        SQL` AND serviceType = ${{
+        SQL` AND primaryEntityType = ${{
           type: TableColumnType.Text,
           value: resourceServiceType as string,
         }}`,
       );
-    } else if (request.facetKey === "serviceId") {
+    } else if (request.facetKey === "primaryEntityId") {
       /*
        * Constrain the canonical Services facet to rows that actually
-       * belong to a Service. NULL / empty serviceType covers legacy rows
-       * ingested before the discriminator existed.
+       * belong to a Service. NULL / empty primaryEntityType covers legacy
+       * rows ingested before the discriminator existed.
        */
       statement.append(
-        SQL` AND (serviceType = '' OR serviceType = ${{
+        SQL` AND (primaryEntityType = '' OR primaryEntityType = ${{
           type: TableColumnType.Text,
           value: ServiceType.OpenTelemetry as string,
         }})`,
       );
     } else if (!isTopLevelColumn) {
       statement.append(
-        SQL` AND JSONHas(attributes, ${{
+        SQL` AND mapContains(attributes, ${{
           type: TableColumnType.Text,
           value: request.facetKey,
-        }}) = 1`,
+        }})`,
       );
     }
 
+    statement.append(" AND retentionDate >= now()");
+
     ExceptionAggregationService.appendCommonFilters(statement, request);
 
     statement.append(
@@ -271,7 +291,10 @@ export class ExceptionAggregationService {
      * 60s proxy_read_timeout so a slow facet never starves the endpoint.
      */
     statement.append(
-      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+      getQuerySettings({
+        maxExecutionTimeInSeconds: 45,
+        timeoutOverflowMode: "break",
+      }),
     );
 
     return statement;
@@ -283,7 +306,7 @@ export class ExceptionAggregationService {
   ): void {
     if (request.serviceIds && request.serviceIds.length > 0) {
       statement.append(
-        SQL` AND serviceId IN (${{
+        SQL` AND primaryEntityId IN (${{
           type: TableColumnType.ObjectID,
           value: new Includes(
             request.serviceIds.map((id: ObjectID) => {

package/Server/Services/HostService.ts

@@ -11,6 +11,7 @@ import OneUptimeDate from "../../Types/Date";
 import LIMIT_MAX from "../../Types/Database/LimitMax";
 import GlobalCache from "../Infrastructure/GlobalCache";
 import logger, { LogAttributes } from "../Utils/Logger";
+import { canonicalizeEntityValue } from "../../Utils/Telemetry/EntityKey";
 import crypto from "crypto";
 
 const LAST_SEEN_CACHE_NAMESPACE: string = "host-last-seen";
@@ -71,7 +72,7 @@ export class Service extends DatabaseService<Model> {
      * host.name (OtelIngestBaseService.normalizeHostNameAttributesInPlace);
      * we repeat it here so the method is correct for any caller.
      */
-    const hostIdentifier: string = data.hostIdentifier.trim().toLowerCase();
+    const hostIdentifier: string = canonicalizeEntityValue(data.hostIdentifier);
 
     /*
      * Look up case-insensitively. The unique guard on name/hostIdentifier