@oneuptime/common 10.0.58 → 10.0.59

package/Models/AnalyticsModels/Span.ts

@@ -565,6 +565,37 @@ export default class Span extends AnalyticsBaseModel {
       },
     });
 
+    const isRootSpanColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
+      key: "isRootSpan",
+      title: "Is Root Span",
+      description:
+        "Whether this span is a root span (has no parent), populated at ingest time for fast root-only filtering",
+      required: true,
+      defaultValue: false,
+      type: TableColumnType.Boolean,
+      skipIndex: {
+        name: "idx_is_root_span",
+        type: SkipIndexType.Set,
+        params: [2],
+        granularity: 4,
+      },
+      accessControl: {
+        read: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.ReadTelemetryServiceTraces,
+        ],
+        create: [
+          Permission.ProjectOwner,
+          Permission.ProjectAdmin,
+          Permission.ProjectMember,
+          Permission.CreateTelemetryServiceTraces,
+        ],
+        update: [],
+      },
+    });
+
     const retentionDateColumn: AnalyticsTableColumn = new AnalyticsTableColumn({
       key: "retentionDate",
       title: "Retention Date",
@@ -628,6 +659,7 @@ export default class Span extends AnalyticsBaseModel {
         nameColumn,
         kindColumn,
         hasExceptionColumn,
+        isRootSpanColumn,
         retentionDateColumn,
       ],
       projections: [
@@ -641,6 +673,11 @@ export default class Span extends AnalyticsBaseModel {
           query:
             "SELECT projectId, traceId, startTime, serviceId, spanId, parentSpanId, name, durationUnixNano, statusCode, hasException ORDER BY (projectId, traceId, startTime)",
         },
+        {
+          name: "proj_hist_by_minute",
+          query:
+            "SELECT projectId, toStartOfMinute(startTime) AS minute, serviceId, statusCode, isRootSpan, count() AS cnt GROUP BY projectId, minute, serviceId, statusCode, isRootSpan",
+        },
       ],
       sortKeys: ["projectId", "startTime", "serviceId", "traceId"],
       primaryKeys: ["projectId", "startTime", "serviceId", "traceId"],
@@ -809,6 +846,14 @@ export default class Span extends AnalyticsBaseModel {
     this.setColumnValue("hasException", v);
   }
 
+  public get isRootSpan(): boolean | undefined {
+    return this.getColumnValue("isRootSpan") as boolean | undefined;
+  }
+
+  public set isRootSpan(v: boolean | undefined) {
+    this.setColumnValue("isRootSpan", v);
+  }
+
   public get retentionDate(): Date | undefined {
     return this.getColumnValue("retentionDate") as Date | undefined;
   }

package/Server/API/BaseAnalyticsAPI.ts

@@ -255,21 +255,22 @@ export default class BaseAnalyticsAPI<
     const databaseProps: DatabaseCommonInteractionProps =
       await CommonAPI.getDatabaseCommonInteractionProps(req);
 
-    const list: Array<AnalyticsDataModel> = await this.service.findBy({
-      query,
-      select,
-      skip: skip,
-      limit: limit,
-      sort: sort,
-      groupBy: groupBy,
-      props: databaseProps,
-    });
-
-    const count: PositiveNumber = await this.service.countBy({
-      query,
-      groupBy: groupBy,
-      props: databaseProps,
-    });
+    const [list, count] = await Promise.all([
+      this.service.findBy({
+        query,
+        select,
+        skip: skip,
+        limit: limit,
+        sort: sort,
+        groupBy: groupBy,
+        props: databaseProps,
+      }),
+      this.service.countBy({
+        query,
+        groupBy: groupBy,
+        props: databaseProps,
+      }),
+    ]);
 
     return Response.sendEntityArrayResponse(
       req,

package/Server/Middleware/TelemetryIngest.ts

@@ -41,6 +41,13 @@ export default class TelemetryIngest {
           | undefined;
       }
 
+      // if x-oneuptime-ingestion-key header is present then use that as token.
+      if (!oneuptimeToken) {
+        oneuptimeToken = req.headers["x-oneuptime-ingestion-key"] as
+          | string
+          | undefined;
+      }
+
       if (!oneuptimeToken) {
         logger.error(
           "Missing header: x-oneuptime-token",

package/Server/Services/AnalyticsDatabaseService.ts

@@ -211,18 +211,33 @@ export default class AnalyticsDatabaseService<
         tableName: this.model.tableName,
       } as LogAttributes);
 
-      const resultInJSON: ResponseJSON<JSONObject> =
-        await dbResult.json<JSONObject>();
-
       let countPositive: PositiveNumber = new PositiveNumber(0);
-      if (
-        resultInJSON.data &&
-        resultInJSON.data[0] &&
-        resultInJSON.data[0]["count"] &&
-        typeof resultInJSON.data[0]["count"] === "string"
-      ) {
-        countPositive = new PositiveNumber(
-          resultInJSON.data[0]["count"] as string,
+
+      try {
+        const resultInJSON: ResponseJSON<JSONObject> =
+          await dbResult.json<JSONObject>();
+
+        if (
+          resultInJSON.data &&
+          resultInJSON.data[0] &&
+          resultInJSON.data[0]["count"] &&
+          typeof resultInJSON.data[0]["count"] === "string"
+        ) {
+          countPositive = new PositiveNumber(
+            resultInJSON.data[0]["count"] as string,
+          );
+        }
+      } catch {
+        /*
+         * When max_execution_time fires with timeout_overflow_mode='break',
+         * ClickHouse may return a truncated response for count() queries
+         * (the aggregation has no partial row to emit). Treat this as
+         * "count unavailable" rather than a fatal error — the list query
+         * itself still succeeds.
+         */
+        logger.warn(
+          `${this.model.tableName} count query returned unparseable response, defaulting to 0`,
+          { tableName: this.model.tableName } as LogAttributes,
         );
       }
 
@@ -689,6 +704,19 @@ export default class AnalyticsDatabaseService<
             }}
             `);
     }
+
+    /*
+     * Cap count query runtime below the ClickHouse client's 58s
+     * request_timeout. Wide time-range queries on large tables (e.g. Span)
+     * can scan billions of rows; without a cap the query runs until the
+     * HTTP client disconnects, wasting ClickHouse resources. With 'break'
+     * mode ClickHouse returns a partial (lower-bound) count rather than
+     * throwing, which is acceptable for pagination display.
+     */
+    statement.append(
+      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+    );
+
     logger.debug(`${this.model.tableName} Count Statement`, { tableName: this.model.tableName } as LogAttributes);
     logger.debug(statement, { tableName: this.model.tableName } as LogAttributes);
 
@@ -813,6 +841,17 @@ export default class AnalyticsDatabaseService<
     }}
         `);
 
+    /*
+     * Defense in depth: cap find-query runtime below the ClickHouse
+     * client's 58s request_timeout. The LIMIT clause keeps most queries
+     * fast, but complex WHERE filters (e.g. parentSpanId IS NULL) on
+     * wide time ranges can still cause long scans. 'break' mode returns
+     * partial results rather than throwing.
+     */
+    statement.append(
+      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+    );
+
     logger.debug(`${this.model.tableName} Find Statement`, { tableName: this.model.tableName } as LogAttributes);
     logger.debug(statement, { tableName: this.model.tableName } as LogAttributes);
 

package/Server/Services/SpanService.ts

@@ -1,11 +1,187 @@
 import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
 import AnalyticsDatabaseService from "./AnalyticsDatabaseService";
 import Span from "../../Models/AnalyticsModels/Span";
+import CountBy from "../Types/AnalyticsDatabase/CountBy";
+import { SQL, Statement } from "../Utils/AnalyticsDatabase/Statement";
+import TableColumnType from "../../Types/AnalyticsDatabase/TableColumnType";
+import InBetween from "../../Types/BaseDatabase/InBetween";
+import Includes from "../../Types/BaseDatabase/Includes";
+import ObjectID from "../../Types/ObjectID";
+
+/**
+ * Columns the proj_hist_by_minute projection can answer with. If a count
+ * query touches only these columns, we can route it through the projection
+ * for a 100x+ speedup over scanning the raw 1.8B-row span table. Any other
+ * column (kind, name, traceId, attributes, …) forces a fallback to the
+ * generic table scan because the projection doesn't store those values.
+ */
+const PROJECTION_ELIGIBLE_KEYS: Set<string> = new Set([
+  "projectId",
+  "startTime",
+  "isRootSpan",
+  "serviceId",
+  "statusCode",
+]);
 
 export class SpanService extends AnalyticsDatabaseService<Span> {
   public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
     super({ modelType: Span, database: clickhouseDatabase });
   }
+
+  /**
+   * Normalize a JSON-deserialized date value to a Date instance. When a query
+   * crosses the API boundary, InBetween's startValue/endValue come back as ISO
+   * strings (or numeric epoch ms) rather than Date objects. Returns null if
+   * the value is unusable.
+   */
+  private static coerceToDate(value: unknown): Date | null {
+    if (value instanceof Date) {
+      return isNaN(value.getTime()) ? null : value;
+    }
+    if (typeof value === "string" || typeof value === "number") {
+      const parsed: Date = new Date(value);
+      return isNaN(parsed.getTime()) ? null : parsed;
+    }
+    return null;
+  }
+
+  /**
+   * Override the count statement to route eligible queries through the
+   * proj_hist_by_minute projection. The projection is keyed on
+   * (projectId, toStartOfMinute(startTime), serviceId, statusCode, isRootSpan)
+   * so its WHERE clause must reference the projection's exact expressions —
+   * filtering on raw `startTime` won't trigger projection use.
+   *
+   * Trade-off: time bounds get rounded to the minute, so the count can be
+   * inflated by spans that started in the same minute as the boundary. For
+   * pagination this is acceptable.
+   */
+  public override toCountStatement(countBy: CountBy<Span>): Statement {
+    const projectionStatement: Statement | null =
+      this.tryBuildProjectionCountStatement(countBy);
+    if (projectionStatement) {
+      return projectionStatement;
+    }
+    return super.toCountStatement(countBy);
+  }
+
+  private tryBuildProjectionCountStatement(
+    countBy: CountBy<Span>,
+  ): Statement | null {
+    if (countBy.groupBy && Object.keys(countBy.groupBy).length > 0) {
+      // GROUP BY count needs the raw table; projection can't help.
+      return null;
+    }
+
+    const query: Record<string, unknown> = (countBy.query ||
+      {}) as unknown as Record<string, unknown>;
+
+    // Bail out if the query references any column the projection doesn't store.
+    for (const key of Object.keys(query)) {
+      if (!PROJECTION_ELIGIBLE_KEYS.has(key)) {
+        return null;
+      }
+    }
+
+    const projectId: ObjectID | undefined = query["projectId"] as
+      | ObjectID
+      | undefined;
+    const startTimeFilter: unknown = query["startTime"];
+
+    /*
+     * Projection only helps when both projectId and a time range are bound —
+     * these are the partition pruning / primary key conditions the optimizer
+     * needs to see in projection-form.
+     */
+    if (!projectId || !(startTimeFilter instanceof InBetween)) {
+      return null;
+    }
+
+    if (!this.database) {
+      this.useDefaultDatabase();
+    }
+    const databaseName: string = this.database.getDatasourceOptions().database!;
+
+    const startValue: Date | null = SpanService.coerceToDate(
+      startTimeFilter.startValue,
+    );
+    const endValue: Date | null = SpanService.coerceToDate(
+      startTimeFilter.endValue,
+    );
+    if (!startValue || !endValue) {
+      return null;
+    }
+
+    const statement: Statement = SQL`SELECT count() AS count FROM ${databaseName}.${this.model.tableName} WHERE projectId = ${{
+      type: TableColumnType.ObjectID,
+      value: projectId,
+    }} AND toStartOfMinute(startTime) >= toStartOfMinute(${{
+      type: TableColumnType.Date,
+      value: startValue,
+    }}) AND toStartOfMinute(startTime) <= toStartOfMinute(${{
+      type: TableColumnType.Date,
+      value: endValue,
+    }})`;
+
+    if (query["isRootSpan"] !== undefined) {
+      statement.append(
+        SQL` AND isRootSpan = ${{
+          type: TableColumnType.Boolean,
+          value: Boolean(query["isRootSpan"]),
+        }}`,
+      );
+    }
+
+    const serviceIdValue: unknown = query["serviceId"];
+    if (serviceIdValue instanceof ObjectID) {
+      statement.append(
+        SQL` AND serviceId = ${{
+          type: TableColumnType.ObjectID,
+          value: serviceIdValue,
+        }}`,
+      );
+    } else if (serviceIdValue instanceof Includes) {
+      statement.append(
+        SQL` AND serviceId IN (${{
+          type: TableColumnType.ObjectID,
+          value: serviceIdValue,
+        }})`,
+      );
+    } else if (serviceIdValue !== undefined) {
+      // Unrecognized serviceId form — let the generic path handle it.
+      return null;
+    }
+
+    const statusCodeValue: unknown = query["statusCode"];
+    if (typeof statusCodeValue === "number") {
+      statement.append(
+        SQL` AND statusCode = ${{
+          type: TableColumnType.Number,
+          value: statusCodeValue,
+        }}`,
+      );
+    } else if (statusCodeValue instanceof Includes) {
+      statement.append(
+        SQL` AND statusCode IN (${{
+          type: TableColumnType.Number,
+          value: statusCodeValue,
+        }})`,
+      );
+    } else if (statusCodeValue !== undefined) {
+      return null;
+    }
+
+    /*
+     * optimize_use_projections is on by default in modern ClickHouse but we
+     * set it explicitly to make the intent obvious. The 45s cap is defense
+     * in depth — projection scans should complete in <1s.
+     */
+    statement.append(
+      " SETTINGS optimize_use_projections = 1, max_execution_time = 45, timeout_overflow_mode = 'break'",
+    );
+
+    return statement;
+  }
 }
 
 export default new SpanService();

package/Server/Services/TraceAggregationService.ts

@@ -8,6 +8,7 @@ import Includes from "../../Types/BaseDatabase/Includes";
 import AnalyticsTableName from "../../Types/AnalyticsDatabase/AnalyticsTableName";
 import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import { DbJSONResponse, Results } from "./AnalyticsDatabaseService";
+import logger from "../Utils/Logger";
 
 export interface HistogramBucket {
   time: string;
@@ -66,6 +67,7 @@ export class TraceAggregationService {
     "name",
     "kind",
     "statusCode",
+    "isRootSpan",
   ]);
   private static readonly ATTRIBUTE_KEY_PATTERN: RegExp = /^[a-zA-Z0-9._:/-]+$/;
   private static readonly MAX_FACET_KEY_LENGTH: number = 256;
@@ -78,11 +80,23 @@ export class TraceAggregationService {
       TraceAggregationService.buildHistogramStatement(request);
 
     const dbResult: Results = await SpanService.executeQuery(statement);
-    const response: DbJSONResponse = await dbResult.json<{
-      data?: Array<JSONObject>;
-    }>();
 
-    const rows: Array<JSONObject> = response.data || [];
+    let rows: Array<JSONObject> = [];
+    try {
+      const response: DbJSONResponse = await dbResult.json<{
+        data?: Array<JSONObject>;
+      }>();
+      rows = response.data || [];
+    } catch {
+      /*
+       * When max_execution_time fires with timeout_overflow_mode='break',
+       * ClickHouse may return a truncated JSON response. Return an empty
+       * histogram rather than failing — the user still sees the span list.
+       */
+      logger.warn(
+        "Histogram query returned unparseable response, returning empty result",
+      );
+    }
 
     return rows.map((row: JSONObject): HistogramBucket => {
       return {
@@ -289,41 +303,60 @@ export class TraceAggregationService {
   private static buildHistogramStatement(request: HistogramRequest): Statement {
     const intervalSeconds: number = request.bucketSizeInMinutes * 60;
 
+    /*
+     * Two-stage aggregation. The inner query groups by minute + statusCode,
+     * which exactly matches the proj_hist_by_minute projection. With
+     * optimize_use_projections=1 (default in modern ClickHouse), the inner
+     * scan reads pre-aggregated rows instead of the 1.8B-row span table —
+     * even for multi-week ranges. The outer query then re-buckets the tiny
+     * minute-level result to the requested bucket size.
+     *
+     * If any non-projection filter (kind, name, traceId, nameSearchText,
+     * attributes) is active, ClickHouse transparently falls back to
+     * scanning the main table for the inner query — same cost as before.
+     */
     const statement: Statement = SQL`
       SELECT
-        toStartOfInterval(startTime, INTERVAL ${{
+        toStartOfInterval(minute, INTERVAL ${{
           type: TableColumnType.Number,
           value: intervalSeconds,
         }} SECOND) AS bucket,
         statusCode,
-        count() AS cnt
-      FROM ${TraceAggregationService.TABLE_NAME}
-      WHERE projectId = ${{
-        type: TableColumnType.ObjectID,
-        value: request.projectId,
-      }}
-        AND startTime >= ${{
-          type: TableColumnType.Date,
-          value: request.startTime,
-        }}
-        AND startTime <= ${{
-          type: TableColumnType.Date,
-          value: request.endTime,
+        sum(cnt_minute) AS cnt
+      FROM (
+        SELECT
+          toStartOfMinute(startTime) AS minute,
+          statusCode,
+          count() AS cnt_minute
+        FROM ${TraceAggregationService.TABLE_NAME}
+        WHERE projectId = ${{
+          type: TableColumnType.ObjectID,
+          value: request.projectId,
         }}
+          AND startTime >= ${{
+            type: TableColumnType.Date,
+            value: request.startTime,
+          }}
+          AND startTime <= ${{
+            type: TableColumnType.Date,
+            value: request.endTime,
+          }}
     `;
 
     TraceAggregationService.appendCommonFilters(statement, request);
 
-    statement.append(" GROUP BY bucket, statusCode ORDER BY bucket ASC");
+    statement.append(
+      " GROUP BY minute, statusCode ) GROUP BY bucket, statusCode ORDER BY bucket ASC",
+    );
 
     /*
      * Defense in depth: cap histogram runtime below nginx's 60s
      * proxy_read_timeout. ClickHouse returns partial aggregated results
      * with 'break' mode rather than throwing, which is acceptable for
-     * a density visualization.
+     * a density visualization. Explicitly enable projection use.
      */
     statement.append(
-      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+      " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break', optimize_use_projections = 1",
     );
 
     return statement;
@@ -401,7 +434,7 @@ export class TraceAggregationService {
     request: TraceFilters,
   ): void {
     if (request.rootOnly) {
-      statement.append(" AND (parentSpanId = '' OR parentSpanId IS NULL)");
+      statement.append(" AND isRootSpan = 1");
     }
 
     if (request.serviceIds && request.serviceIds.length > 0) {

package/Tests/Server/Services/AnalyticsDatabaseService.test.ts

@@ -106,7 +106,8 @@ describe("AnalyticsDatabaseService", () => {
         "SELECT\n" +
           "    count() as count\n" +
           "FROM {p0:Identifier}.{p1:Identifier}\n" +
-          "WHERE TRUE <where-statement>",
+          "WHERE TRUE <where-statement>" +
+          " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
       );
       expect(statement.query_params).toStrictEqual({
         p0: "oneuptime",
@@ -126,7 +127,8 @@ describe("AnalyticsDatabaseService", () => {
           "    count() as count\n" +
           "FROM {p0:Identifier}.{p1:Identifier}\n" +
           "WHERE TRUE <where-statement>\n" +
-          "LIMIT {p2:Int32}",
+          "LIMIT {p2:Int32}\n" +
+          " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
       );
       expect(statement.query_params).toStrictEqual({
         p0: "oneuptime",
@@ -147,7 +149,8 @@ describe("AnalyticsDatabaseService", () => {
           "    count() as count\n" +
           "FROM {p0:Identifier}.{p1:Identifier}\n" +
           "WHERE TRUE <where-statement>\n" +
-          "OFFSET {p2:Int32}",
+          "OFFSET {p2:Int32}\n" +
+          " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
       );
       expect(statement.query_params).toStrictEqual({
         p0: "oneuptime",
@@ -211,7 +214,8 @@ describe("AnalyticsDatabaseService", () => {
       });
 
       expect(statement.query).toBe(
-        "SELECT <select-statement> FROM {p0:Identifier}.{p1:Identifier} WHERE TRUE <where-statement> ORDER BY <sort-statement> LIMIT {p2:Int32} OFFSET {p3:Int32}",
+        "SELECT <select-statement> FROM {p0:Identifier}.{p1:Identifier} WHERE TRUE <where-statement> ORDER BY <sort-statement> LIMIT {p2:Int32} OFFSET {p3:Int32}\n" +
+          "SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
       );
       expect(statement.query_params).toStrictEqual({
         p0: "oneuptime",

package/UI/Components/TelemetryViewer/TelemetryViewer.tsx

@@ -107,7 +107,7 @@ function TelemetryViewerInner<T>(props: TelemetryViewerProps<T>): ReactElement {
   const showHistogram: boolean = props.showHistogram ?? true;
 
   return (
-    <div className="flex h-full w-full flex-col gap-3">
+    <div className="flex min-h-0 w-full flex-1 flex-col gap-3">
       {/* Toolbar */}
       <div className="flex flex-wrap items-center gap-2">
         <div className="min-w-0 flex-1">

package/build/dist/Models/AnalyticsModels/Span.js

@@ -523,6 +523,35 @@ export default class Span extends AnalyticsBaseModel {
                 update: [],
             },
         });
+        const isRootSpanColumn = new AnalyticsTableColumn({
+            key: "isRootSpan",
+            title: "Is Root Span",
+            description: "Whether this span is a root span (has no parent), populated at ingest time for fast root-only filtering",
+            required: true,
+            defaultValue: false,
+            type: TableColumnType.Boolean,
+            skipIndex: {
+                name: "idx_is_root_span",
+                type: SkipIndexType.Set,
+                params: [2],
+                granularity: 4,
+            },
+            accessControl: {
+                read: [
+                    Permission.ProjectOwner,
+                    Permission.ProjectAdmin,
+                    Permission.ProjectMember,
+                    Permission.ReadTelemetryServiceTraces,
+                ],
+                create: [
+                    Permission.ProjectOwner,
+                    Permission.ProjectAdmin,
+                    Permission.ProjectMember,
+                    Permission.CreateTelemetryServiceTraces,
+                ],
+                update: [],
+            },
+        });
         const retentionDateColumn = new AnalyticsTableColumn({
             key: "retentionDate",
             title: "Retention Date",
@@ -584,6 +613,7 @@ export default class Span extends AnalyticsBaseModel {
                 nameColumn,
                 kindColumn,
                 hasExceptionColumn,
+                isRootSpanColumn,
                 retentionDateColumn,
             ],
             projections: [
@@ -595,6 +625,10 @@ export default class Span extends AnalyticsBaseModel {
                     name: "proj_trace_by_id",
                     query: "SELECT projectId, traceId, startTime, serviceId, spanId, parentSpanId, name, durationUnixNano, statusCode, hasException ORDER BY (projectId, traceId, startTime)",
                 },
+                {
+                    name: "proj_hist_by_minute",
+                    query: "SELECT projectId, toStartOfMinute(startTime) AS minute, serviceId, statusCode, isRootSpan, count() AS cnt GROUP BY projectId, minute, serviceId, statusCode, isRootSpan",
+                },
             ],
             sortKeys: ["projectId", "startTime", "serviceId", "traceId"],
             primaryKeys: ["projectId", "startTime", "serviceId", "traceId"],
@@ -722,6 +756,12 @@ export default class Span extends AnalyticsBaseModel {
     set hasException(v) {
         this.setColumnValue("hasException", v);
     }
+    get isRootSpan() {
+        return this.getColumnValue("isRootSpan");
+    }
+    set isRootSpan(v) {
+        this.setColumnValue("isRootSpan", v);
+    }
     get retentionDate() {
         return this.getColumnValue("retentionDate");
     }