@oneuptime/common 10.5.1 → 10.5.2

package/Server/Services/ExceptionAggregationService.ts

@@ -8,6 +8,7 @@ import Includes from "../../Types/BaseDatabase/Includes";
 import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { DbJSONResponse, Results } from "./AnalyticsDatabaseService";
+import ServiceType from "../../Types/Telemetry/ServiceType";
 
 export interface HistogramBucket {
   time: string;
@@ -35,6 +36,7 @@ export interface HistogramRequest extends ExceptionFilters {
 export interface FacetValue {
   value: string;
   count: number;
+  displayName?: string | undefined;
 }
 
 export interface FacetRequest extends ExceptionFilters {
@@ -59,6 +61,18 @@ export class ExceptionAggregationService {
     "escaped",
     "release",
   ]);
+  /*
+   * Virtual facet keys — same scheme as TraceAggregationService /
+   * LogAggregationService. The `serviceId` slot is reused for host /
+   * docker host / k8s cluster ids, disambiguated by the `serviceType`
+   * discriminator column on each ExceptionInstance row.
+   */
+  private static readonly RESOURCE_FACET_KEYS: Map<string, ServiceType> =
+    new Map([
+      ["hostId", ServiceType.Host],
+      ["dockerHostId", ServiceType.DockerHost],
+      ["kubernetesClusterId", ServiceType.KubernetesCluster],
+    ]);
   private static readonly ATTRIBUTE_KEY_PATTERN: RegExp = /^[a-zA-Z0-9._:/-]+$/;
   private static readonly MAX_FACET_KEY_LENGTH: number = 256;
 
@@ -169,12 +183,24 @@ export class ExceptionAggregationService {
 
     ExceptionAggregationService.validateFacetKey(request.facetKey);
 
+    const resourceServiceType: ServiceType | undefined =
+      ExceptionAggregationService.RESOURCE_FACET_KEYS.get(request.facetKey);
+    const isResourceFacet: boolean = resourceServiceType !== undefined;
     const isTopLevelColumn: boolean =
+      isResourceFacet ||
       ExceptionAggregationService.isTopLevelColumn(request.facetKey);
 
     const statement: Statement = new Statement();
 
-    if (isTopLevelColumn) {
+    if (isResourceFacet) {
+      /*
+       * Virtual facet — group serviceId values whose row carries the matching
+       * ServiceType discriminator (Host / DockerHost / KubernetesCluster).
+       */
+      statement.append(
+        SQL`SELECT toString(serviceId) AS val, count() AS cnt FROM ${ExceptionAggregationService.TABLE_NAME}`,
+      );
+    } else if (isTopLevelColumn) {
       statement.append(
         SQL`SELECT toString(${request.facetKey}) AS val, count() AS cnt FROM ${ExceptionAggregationService.TABLE_NAME}`,
       );
@@ -200,7 +226,26 @@ export class ExceptionAggregationService {
       }}`,
     );
 
-    if (!isTopLevelColumn) {
+    if (isResourceFacet) {
+      statement.append(
+        SQL` AND serviceType = ${{
+          type: TableColumnType.Text,
+          value: resourceServiceType as string,
+        }}`,
+      );
+    } else if (request.facetKey === "serviceId") {
+      /*
+       * Constrain the canonical Services facet to rows that actually
+       * belong to a Service. NULL / empty serviceType covers legacy rows
+       * ingested before the discriminator existed.
+       */
+      statement.append(
+        SQL` AND (serviceType = '' OR serviceType = ${{
+          type: TableColumnType.Text,
+          value: ServiceType.OpenTelemetry as string,
+        }})`,
+      );
+    } else if (!isTopLevelColumn) {
       statement.append(
         SQL` AND JSONHas(attributes, ${{
           type: TableColumnType.Text,
@@ -322,7 +367,10 @@ export class ExceptionAggregationService {
       throw new BadDataException("Invalid facetKey");
     }
 
-    if (ExceptionAggregationService.isTopLevelColumn(facetKey)) {
+    if (
+      ExceptionAggregationService.isTopLevelColumn(facetKey) ||
+      ExceptionAggregationService.RESOURCE_FACET_KEYS.has(facetKey)
+    ) {
       return;
     }
 

package/Server/Services/LogAggregationService.ts

@@ -32,6 +32,7 @@ export interface HistogramRequest {
 export interface FacetValue {
   value: string;
   count: number;
+  displayName?: string | undefined;
 }
 
 export interface FacetRequest {

package/Server/Services/MetricAggregationService.ts

@@ -0,0 +1,227 @@
+import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement";
+import MetricService from "./MetricService";
+import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
+import { JSONObject } from "../../Types/JSON";
+import ObjectID from "../../Types/ObjectID";
+import BadDataException from "../../Types/Exception/BadDataException";
+import Includes from "../../Types/BaseDatabase/Includes";
+import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+import { DbJSONResponse, Results } from "./AnalyticsDatabaseService";
+import ServiceType from "../../Types/Telemetry/ServiceType";
+
+export interface FacetValue {
+  value: string;
+  count: number;
+  displayName?: string | undefined;
+}
+
+export interface MetricFilters {
+  serviceIds?: Array<ObjectID> | undefined;
+  metricNames?: Array<string> | undefined;
+}
+
+export interface FacetRequest extends MetricFilters {
+  projectId: ObjectID;
+  startTime: Date;
+  endTime: Date;
+  facetKey: string;
+  limit?: number | undefined;
+}
+
+/*
+ * Facet aggregation for the Metrics page sidebar. Same shape as
+ * TraceAggregationService / LogAggregationService — per-facet GROUP BY on
+ * the analytics table, with a `serviceType` discriminator that lets the
+ * `serviceId` column carry Host / DockerHost / KubernetesCluster ids for
+ * the corresponding virtual facets.
+ */
+export class MetricAggregationService {
+  private static readonly DEFAULT_FACET_LIMIT: number = 500;
+  private static readonly TABLE_NAME: string = AnalyticsTableName.Metric;
+  private static readonly TOP_LEVEL_COLUMNS: Set<string> = new Set([
+    "serviceId",
+    "name",
+  ]);
+  private static readonly RESOURCE_FACET_KEYS: Map<string, ServiceType> =
+    new Map([
+      ["hostId", ServiceType.Host],
+      ["dockerHostId", ServiceType.DockerHost],
+      ["kubernetesClusterId", ServiceType.KubernetesCluster],
+    ]);
+  private static readonly ATTRIBUTE_KEY_PATTERN: RegExp = /^[a-zA-Z0-9._:/-]+$/;
+  private static readonly MAX_FACET_KEY_LENGTH: number = 256;
+
+  @CaptureSpan()
+  public static async getFacetValues(
+    request: FacetRequest,
+  ): Promise<Array<FacetValue>> {
+    const statement: Statement =
+      MetricAggregationService.buildFacetStatement(request);
+
+    const dbResult: Results = await MetricService.executeQuery(statement);
+    const response: DbJSONResponse = await dbResult.json<{
+      data?: Array<JSONObject>;
+    }>();
+
+    const rows: Array<JSONObject> = response.data || [];
+
+    return rows
+      .map((row: JSONObject): FacetValue => {
+        return {
+          value: String(row["val"] || ""),
+          count: Number(row["cnt"] || 0),
+        };
+      })
+      .filter((facet: FacetValue): boolean => {
+        return facet.value.length > 0;
+      });
+  }
+
+  private static buildFacetStatement(request: FacetRequest): Statement {
+    const limit: number =
+      request.limit ?? MetricAggregationService.DEFAULT_FACET_LIMIT;
+
+    MetricAggregationService.validateFacetKey(request.facetKey);
+
+    const resourceServiceType: ServiceType | undefined =
+      MetricAggregationService.RESOURCE_FACET_KEYS.get(request.facetKey);
+    const isResourceFacet: boolean = resourceServiceType !== undefined;
+    const isTopLevelColumn: boolean =
+      isResourceFacet ||
+      MetricAggregationService.isTopLevelColumn(request.facetKey);
+
+    const statement: Statement = new Statement();
+
+    if (isResourceFacet) {
+      statement.append(
+        SQL`SELECT toString(serviceId) AS val, count() AS cnt FROM ${MetricAggregationService.TABLE_NAME}`,
+      );
+    } else if (isTopLevelColumn) {
+      statement.append(
+        SQL`SELECT toString(${request.facetKey}) AS val, count() AS cnt FROM ${MetricAggregationService.TABLE_NAME}`,
+      );
+    } else {
+      statement.append(
+        SQL`SELECT JSONExtractRaw(attributes, ${{
+          type: TableColumnType.Text,
+          value: request.facetKey,
+        }}) AS val, count() AS cnt FROM ${MetricAggregationService.TABLE_NAME}`,
+      );
+    }
+
+    statement.append(
+      SQL` WHERE projectId = ${{
+        type: TableColumnType.ObjectID,
+        value: request.projectId,
+      }} AND time >= ${{
+        type: TableColumnType.Date,
+        value: request.startTime,
+      }} AND time <= ${{
+        type: TableColumnType.Date,
+        value: request.endTime,
+      }}`,
+    );
+
+    if (isResourceFacet) {
+      statement.append(
+        SQL` AND serviceType = ${{
+          type: TableColumnType.Text,
+          value: resourceServiceType as string,
+        }}`,
+      );
+    } else if (request.facetKey === "serviceId") {
+      statement.append(
+        SQL` AND (serviceType = '' OR serviceType = ${{
+          type: TableColumnType.Text,
+          value: ServiceType.OpenTelemetry as string,
+        }})`,
+      );
+    } else if (!isTopLevelColumn) {
+      statement.append(
+        SQL` AND JSONHas(attributes, ${{
+          type: TableColumnType.Text,
+          value: request.facetKey,
+        }}) = 1`,
+      );
+    }
+
+    MetricAggregationService.appendCommonFilters(statement, request);
+
+    statement.append(
+      SQL` GROUP BY val ORDER BY cnt DESC LIMIT ${{
+        type: TableColumnType.Number,
+        value: limit,
+      }}`,
+    );
+
+    /*
+     * Defense in depth: cap runtime below nginx's 60s proxy_read_timeout
+     * so a slow facet never starves the endpoint.
+     */
+    statement.append(
+      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+    );
+
+    return statement;
+  }
+
+  private static appendCommonFilters(
+    statement: Statement,
+    request: MetricFilters,
+  ): void {
+    if (request.serviceIds && request.serviceIds.length > 0) {
+      statement.append(
+        SQL` AND serviceId IN (${{
+          type: TableColumnType.ObjectID,
+          value: new Includes(
+            request.serviceIds.map((id: ObjectID) => {
+              return id.toString();
+            }),
+          ),
+        }})`,
+      );
+    }
+
+    if (request.metricNames && request.metricNames.length > 0) {
+      statement.append(
+        SQL` AND name IN (${{
+          type: TableColumnType.Text,
+          value: new Includes(request.metricNames),
+        }})`,
+      );
+    }
+  }
+
+  private static isTopLevelColumn(key: string): boolean {
+    return MetricAggregationService.TOP_LEVEL_COLUMNS.has(key);
+  }
+
+  private static validateFacetKey(
+    facetKey: unknown,
+  ): asserts facetKey is string {
+    if (typeof facetKey !== "string") {
+      throw new BadDataException("Invalid facetKey");
+    }
+
+    if (
+      facetKey.length === 0 ||
+      facetKey.length > MetricAggregationService.MAX_FACET_KEY_LENGTH
+    ) {
+      throw new BadDataException("Invalid facetKey");
+    }
+
+    if (
+      MetricAggregationService.isTopLevelColumn(facetKey) ||
+      MetricAggregationService.RESOURCE_FACET_KEYS.has(facetKey)
+    ) {
+      return;
+    }
+
+    if (!MetricAggregationService.ATTRIBUTE_KEY_PATTERN.test(facetKey)) {
+      throw new BadDataException("Invalid facetKey");
+    }
+  }
+}
+
+export default MetricAggregationService;

package/Server/Services/OpenTelemetryIngestService.ts

@@ -23,6 +23,7 @@ import KubernetesCluster from "../../Models/DatabaseModels/KubernetesCluster";
 import HostService from "./HostService";
 import DockerHostService from "./DockerHostService";
 import KubernetesClusterService from "./KubernetesClusterService";
+import GlobalCache from "../Infrastructure/GlobalCache";
 
 export enum OtelAggregationTemporality {
   Cumulative = "AGGREGATION_TEMPORALITY_CUMULATIVE",
@@ -52,11 +53,83 @@ interface ProjectRetentionContext {
   projectRetentionInDays: number;
 }
 
+/*
+ * Per-process memo holding the project's telemetry retention
+ * context for the duration of the cache TTL. The L2 GlobalCache
+ * (Redis, shared across workers) is the source of truth for
+ * cross-process freshness; this L1 in-process Map exists so the
+ * dozens of getProjectRetentionContext calls per worker batch
+ * (one per resource span -> Service/Host/DockerHost/Kubernetes
+ * resolution) collapse to zero network round-trips after the
+ * first one warms it. Both layers TTL out together.
+ */
+interface CachedRetentionContext {
+  context: ProjectRetentionContext;
+  expiresAtMs: number;
+}
+
+const PROJECT_RETENTION_CACHE_NAMESPACE: string = "project-retention-context";
+/*
+ * 5-minute TTL is long enough that steady-state ingest sees ~100%
+ * cache hits and short enough that an admin retention change in
+ * the UI propagates without us having to wire up cross-process
+ * invalidation (which would need pub/sub — GlobalCache has no
+ * delete primitive today). Admins changing retention should
+ * expect up to 5 minutes of lag before new rows pick up the new
+ * config; existing rows keep whatever retentionDate they were
+ * stamped with at ingest time and aren't affected either way.
+ */
+const PROJECT_RETENTION_CACHE_TTL_SECONDS: number = 5 * 60;
+const projectRetentionInProcessCache: Map<string, CachedRetentionContext> =
+  new Map();
+
 export default class OTelIngestService {
   @CaptureSpan()
   private static async getProjectRetentionContext(
     projectId: ObjectID,
   ): Promise<ProjectRetentionContext> {
+    const projectIdStr: string = projectId.toString();
+    const now: number = Date.now();
+
+    // L1: in-process memo. Zero network cost.
+    const memoed: CachedRetentionContext | undefined =
+      projectRetentionInProcessCache.get(projectIdStr);
+    if (memoed && memoed.expiresAtMs > now) {
+      return memoed.context;
+    }
+
+    // L2: Redis. Single round-trip; shared across workers.
+    try {
+      const cached: JSONObject | null = await GlobalCache.getJSONObject(
+        PROJECT_RETENTION_CACHE_NAMESPACE,
+        projectIdStr,
+      );
+      if (cached) {
+        const context: ProjectRetentionContext = {
+          projectRetentionConfig:
+            (cached[
+              "projectRetentionConfig"
+            ] as TelemetryRetentionConfig | null) ?? null,
+          projectRetentionInDays:
+            (cached["projectRetentionInDays"] as number) ||
+            DEFAULT_RETENTION_IN_DAYS,
+        };
+        projectRetentionInProcessCache.set(projectIdStr, {
+          context,
+          expiresAtMs: now + PROJECT_RETENTION_CACHE_TTL_SECONDS * 1000,
+        });
+        return context;
+      }
+    } catch (err) {
+      // Cache outage must never fail ingest. Fall through to Postgres.
+      logger.warn(
+        `Project retention cache read failed for project ${projectIdStr}; falling back to Postgres: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+    }
+
+    // Cold path: hit Postgres and warm both caches.
     const project: Project | null = await ProjectService.findOneById({
       id: projectId,
       select: {
@@ -68,11 +141,38 @@ export default class OTelIngestService {
       },
     });
 
-    return {
+    const context: ProjectRetentionContext = {
       projectRetentionConfig: project?.telemetryRetentionConfig ?? null,
       projectRetentionInDays:
         project?.defaultTelemetryRetentionInDays || DEFAULT_RETENTION_IN_DAYS,
     };
+
+    projectRetentionInProcessCache.set(projectIdStr, {
+      context,
+      expiresAtMs: now + PROJECT_RETENTION_CACHE_TTL_SECONDS * 1000,
+    });
+
+    try {
+      await GlobalCache.setJSON(
+        PROJECT_RETENTION_CACHE_NAMESPACE,
+        projectIdStr,
+        {
+          projectRetentionConfig: (context.projectRetentionConfig ??
+            null) as unknown as JSONObject,
+          projectRetentionInDays: context.projectRetentionInDays,
+        },
+        { expiresInSeconds: PROJECT_RETENTION_CACHE_TTL_SECONDS },
+      );
+    } catch (err) {
+      // Best-effort warm. Don't fail the request.
+      logger.warn(
+        `Project retention cache write failed for project ${projectIdStr}: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+    }
+
+    return context;
   }
 
   @CaptureSpan()

package/Server/Services/TraceAggregationService.ts

@@ -38,6 +38,7 @@ export interface HistogramRequest extends TraceFilters {
 export interface FacetValue {
   value: string;
   count: number;
+  displayName?: string | undefined;
 }
 
 export interface FacetRequest extends TraceFilters {

package/Server/Utils/Monitor/MonitorLogUtil.ts

@@ -7,6 +7,29 @@ import ObjectID from "../../../Types/ObjectID";
 import { JSONObject } from "../../../Types/JSON";
 import DataToProcess from "./DataToProcess";
 
+/*
+ * Maximum rows held in memory before we force a flush, and the
+ * maximum time we hold a row in the buffer before flushing.
+ *
+ * Sizing rationale: 10,000 rows at a few KB each is ~tens of MB
+ * of peak heap per process, which is fine for the API/worker
+ * processes that run monitor probes. 5 seconds is a tight enough
+ * worst-case latency that the dashboard "last log" view feels
+ * live, and loose enough that even a single very chatty monitor
+ * coalesces dozens to thousands of inserts into one.
+ *
+ * The legacy implementation called MonitorLogService.insertJsonRows
+ * with a one-element array per monitor probe, fire-and-forget.
+ * ClickHouse's async_insert deduplicates these into part files,
+ * but every call still pays an HTTP round-trip and the
+ * @clickhouse/client pool can saturate when probe traffic spikes
+ * (e.g. a 10k-monitor project running concurrent probes). Batching
+ * collapses N round-trips into N / 10_000 (size-bounded) or
+ * N / (5s × per-second-rate) (time-bounded), whichever comes first.
+ */
+const MONITOR_LOG_FLUSH_BATCH_SIZE: number = 10_000;
+const MONITOR_LOG_FLUSH_INTERVAL_MS: number = 5_000;
+
 export default class MonitorLogUtil {
   // Default retention in days if GlobalConfig is not set
   private static readonly DEFAULT_RETENTION_DAYS: number = 1;
@@ -16,6 +39,18 @@ export default class MonitorLogUtil {
   private static lastCacheRefresh: Date | null = null;
   private static readonly CACHE_TTL_MS: number = 5 * 60 * 1000; // 5 minutes
 
+  /*
+   * In-process write buffer for MonitorLog rows. Rows accumulate
+   * here until either MONITOR_LOG_FLUSH_BATCH_SIZE rows arrive
+   * (size trigger) or MONITOR_LOG_FLUSH_INTERVAL_MS elapses since
+   * the first row entered an empty buffer (time trigger),
+   * whichever comes first. On graceful shutdown the SIGTERM /
+   * SIGINT hook below drains the buffer before the process exits.
+   */
+  private static buffer: Array<JSONObject> = [];
+  private static flushTimer: NodeJS.Timeout | null = null;
+  private static shutdownHooksRegistered: boolean = false;
+
   private static async getRetentionDays(): Promise<number> {
     const now: Date = OneUptimeDate.getCurrentDate();
 
@@ -83,7 +118,7 @@ export default class MonitorLogUtil {
       return;
     }
 
-    // Fire-and-forget: fetch retention config then insert
+    // Fire-and-forget: fetch retention config then enqueue
     this.getRetentionDays()
       .then((retentionDays: number) => {
         const logIngestionDate: Date = OneUptimeDate.getCurrentDate();
@@ -106,14 +141,119 @@ export default class MonitorLogUtil {
           retentionDate: OneUptimeDate.toClickhouseDateTime(retentionDate),
         };
 
-        MonitorLogService.insertJsonRows([monitorLogRow]).catch(
-          (err: Error) => {
-            logger.error(err);
-          },
-        );
+        this.enqueueRow(monitorLogRow);
       })
       .catch((err: Error) => {
         logger.error(err);
       });
   }
+
+  /*
+   * Append a row to the buffer, then decide whether to flush
+   * immediately (size threshold) or schedule the timed flush
+   * (first row in an empty buffer). The timer is intentionally
+   * NOT reset on every row — we want strict "at most 5 s of
+   * staleness", not "at most 5 s of idle". Resetting per-row
+   * could indefinitely delay flush under steady ingest.
+   */
+  private static enqueueRow(row: JSONObject): void {
+    this.ensureShutdownHooks();
+
+    this.buffer.push(row);
+
+    if (this.buffer.length >= MONITOR_LOG_FLUSH_BATCH_SIZE) {
+      // Size trigger — flush immediately.
+      this.triggerFlush();
+      return;
+    }
+
+    if (!this.flushTimer && this.buffer.length === 1) {
+      // First row in an empty buffer — start the time-based flush.
+      this.flushTimer = setTimeout(() => {
+        this.triggerFlush();
+      }, MONITOR_LOG_FLUSH_INTERVAL_MS);
+    }
+  }
+
+  /*
+   * Synchronous fire-and-forget flush used by the normal hot
+   * path: callers must not be blocked waiting for ClickHouse.
+   * Swap the buffer out first so new arrivals during the network
+   * round-trip land in a fresh array and are not double-flushed.
+   */
+  private static triggerFlush(): void {
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+
+    if (this.buffer.length === 0) {
+      return;
+    }
+
+    const toFlush: Array<JSONObject> = this.buffer;
+    this.buffer = [];
+
+    MonitorLogService.insertJsonRows(toFlush).catch((err: Error) => {
+      logger.error(
+        `MonitorLog batch insert failed for ${toFlush.length} rows; batch dropped.`,
+      );
+      logger.error(err);
+    });
+  }
+
+  /*
+   * Awaitable flush used by the SIGTERM / SIGINT shutdown hook
+   * so we don't lose buffered rows when Kubernetes / Docker
+   * sends the process to bed. Errors are logged but swallowed —
+   * a failing flush must not block the rest of the shutdown
+   * chain (other handlers may still need to run).
+   */
+  private static async flushAndWait(): Promise<void> {
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = null;
+    }
+
+    if (this.buffer.length === 0) {
+      return;
+    }
+
+    const toFlush: Array<JSONObject> = this.buffer;
+    this.buffer = [];
+
+    try {
+      await MonitorLogService.insertJsonRows(toFlush);
+    } catch (err) {
+      logger.error(
+        `MonitorLog shutdown flush failed for ${toFlush.length} rows; batch dropped.`,
+      );
+      logger.error(err);
+    }
+  }
+
+  /*
+   * Register SIGTERM / SIGINT handlers exactly once, lazily on
+   * first ingest. We avoid registering at module-load time so
+   * tooling that imports this file (e.g. migration runners,
+   * CLI scripts) doesn't end up with stray process listeners.
+   */
+  private static ensureShutdownHooks(): void {
+    if (this.shutdownHooksRegistered) {
+      return;
+    }
+    this.shutdownHooksRegistered = true;
+
+    const flushOnShutdown: () => Promise<void> = async (): Promise<void> => {
+      try {
+        await this.flushAndWait();
+      } catch (err) {
+        logger.error("Error flushing MonitorLog buffer on shutdown:");
+        logger.error(err);
+      }
+    };
+
+    process.on("SIGTERM", flushOnShutdown);
+    process.on("SIGINT", flushOnShutdown);
+  }
 }