@oneuptime/common 10.2.11 → 10.2.13

package/Server/API/DashboardAPI.ts

@@ -24,6 +24,8 @@ import { DASHBOARD_MASTER_PASSWORD_INVALID_MESSAGE } from "../../Types/Dashboard
 import NotAuthenticatedException from "../../Types/Exception/NotAuthenticatedException";
 import ForbiddenException from "../../Types/Exception/ForbiddenException";
 import JSONFunctions from "../../Types/JSONFunctions";
+import TelemetryAttributeService from "../Services/TelemetryAttributeService";
+import TelemetryType from "../../Types/Telemetry/TelemetryType";
 
 export default class DashboardAPI extends BaseAPI<
   Dashboard,
@@ -302,6 +304,100 @@ export default class DashboardAPI extends BaseAPI<
       },
     );
 
+    /*
+     * Public attribute-value lookup for dashboard variables.
+     *
+     * The private `/telemetry/metrics/get-attribute-values` route requires
+     * a logged-in session; public dashboards have no session, so we mirror
+     * the behaviour here scoped to the dashboard's owning projectId.
+     * Authorization reuses DashboardService.hasReadAccess (public flag, IP
+     * whitelist, master password) — never falls back to project-wide read.
+     */
+    this.router.post(
+      `${new this.entityType()
+        .getCrudApiPath()
+        ?.toString()}/attribute-values/:dashboardId`,
+      UserMiddleware.getUserMiddleware,
+      async (req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
+        try {
+          const dashboardId: ObjectID = new ObjectID(
+            req.params["dashboardId"] as string,
+          );
+
+          const accessResult: {
+            hasReadAccess: boolean;
+            error?: NotAuthenticatedException | ForbiddenException;
+          } = await DashboardService.hasReadAccess({
+            dashboardId,
+            req,
+          });
+
+          if (!accessResult.hasReadAccess) {
+            throw (
+              accessResult.error ||
+              new BadDataException("Access denied to this dashboard.")
+            );
+          }
+
+          const attributeKey: string | undefined =
+            req.body && (req.body["attributeKey"] as string);
+
+          if (!attributeKey || !attributeKey.trim()) {
+            throw new BadDataException("attributeKey is required.");
+          }
+
+          const metricNameRaw: string | undefined =
+            req.body && (req.body["metricName"] as string);
+          const metricName: string | undefined =
+            metricNameRaw && metricNameRaw.trim() ? metricNameRaw : undefined;
+
+          const telemetryTypeRaw: string | undefined =
+            req.body && (req.body["telemetryType"] as string);
+          let telemetryType: TelemetryType = TelemetryType.Metric;
+          if (telemetryTypeRaw) {
+            const match: TelemetryType | undefined = (
+              Object.values(TelemetryType) as Array<string>
+            ).includes(telemetryTypeRaw)
+              ? (telemetryTypeRaw as TelemetryType)
+              : undefined;
+            if (match) {
+              telemetryType = match;
+            }
+          }
+
+          const dashboard: Dashboard | null =
+            await DashboardService.findOneById({
+              id: dashboardId,
+              select: {
+                _id: true,
+                projectId: true,
+              },
+              props: {
+                isRoot: true,
+              },
+            });
+
+          if (!dashboard || !dashboard.projectId) {
+            throw new NotFoundException("Dashboard not found");
+          }
+
+          const values: Array<string> =
+            await TelemetryAttributeService.fetchAttributeValues({
+              projectId: dashboard.projectId,
+              telemetryType,
+              attributeKey: attributeKey.trim(),
+              metricName,
+            });
+
+          return Response.sendJsonObjectResponse(req, res, {
+            values,
+          });
+        } catch (err) {
+          next(err);
+        }
+      },
+    );
+
     this.router.post(
       `${new this.entityType()
         .getCrudApiPath()

package/Server/Services/TelemetryAttributeService.ts

@@ -35,10 +35,20 @@ type TelemetryAttributesCacheEntry = {
 
 export class TelemetryAttributeService {
   private static readonly ATTRIBUTES_LIMIT: number = 5000;
-  private static readonly ROW_SCAN_LIMIT: number = 10000;
   private static readonly CACHE_NAMESPACE: string = "telemetry-attributes";
-  private static readonly CACHE_STALE_AFTER_MINUTES: number = 5;
-  private static readonly LOOKBACK_WINDOW_IN_DAYS: number = 30;
+  /*
+   * Attribute keys change rarely. Cache for an hour so the (still O(seconds))
+   * ClickHouse scan only runs once per project per hour rather than on every
+   * dashboard / metrics-explorer load.
+   */
+  private static readonly CACHE_STALE_AFTER_MINUTES: number = 60;
+  /*
+   * The previous 30-day window forced a 100M+ row scan with an in-CTE
+   * ORDER BY time DESC that pushed this query to 30-60s on busy projects.
+   * Attribute keys rotate slowly, so a 1-day window covers virtually every
+   * active key while keeping the scan tractable.
+   */
+  private static readonly LOOKBACK_WINDOW_IN_DAYS: number = 1;
 
   private getTelemetrySource(
     telemetryType: TelemetryType,
@@ -247,76 +257,56 @@ export class TelemetryAttributeService {
       TelemetryAttributeService.getLookbackStartDate();
 
     /*
-     * If the source has a denormalized attributeKeys array column, prefer it
-     * (avoids materializing every row's map). Otherwise fall back to
-     * mapKeys(attributes) — slower but works for tables that don't carry
-     * the precomputed array (e.g. ExceptionInstance).
+     * Two notable choices here:
+     *
+     * 1. We aggregate with `groupUniqArrayArray` (or `mapKeys`+`groupUniqArray`
+     *    for tables that lack the denormalized array column) instead of
+     *    `arrayJoin` + outer `DISTINCT`. That avoids materializing one row
+     *    per attribute key across millions of source rows.
+     *
+     * 2. The previous implementation wrapped the scan in
+     *    `ORDER BY time DESC LIMIT 10000` to "cap" the work. With `arrayJoin`
+     *    that LIMIT applied AFTER expansion so it didn't actually bound rows
+     *    read, but it did force ClickHouse to sort every matching row by
+     *    time — the dominant cost on busy projects. Bounded by lookback
+     *    instead, the aggregate-and-flatten approach finishes in seconds.
      */
     const statement: Statement = data.attributeKeysColumn
       ? SQL`
-      WITH filtered AS (
-        SELECT arrayJoin(
-            if(
-              empty(${data.attributeKeysColumn}),
-              mapKeys(${data.attributesColumn}),
-              ${data.attributeKeysColumn}
-            )
-          ) AS attribute
-        FROM ${data.tableName}
-        WHERE projectId = ${{
-          type: TableColumnType.ObjectID,
-          value: data.projectId,
-        }}
-          AND (
-            NOT empty(${data.attributeKeysColumn}) OR
-            NOT empty(${data.attributesColumn})
-          )
-          AND ${data.timeColumn} >= ${{
-            type: TableColumnType.Date,
-            value: lookbackStartDate,
-          }}`
+      SELECT arrayDistinct(arrayFlatten(groupUniqArrayArray(${data.attributeKeysColumn}))) AS keys
+      FROM ${data.tableName}
+      WHERE projectId = ${{
+        type: TableColumnType.ObjectID,
+        value: data.projectId,
+      }}
+        AND NOT empty(${data.attributeKeysColumn})
+        AND ${data.timeColumn} >= ${{
+          type: TableColumnType.Date,
+          value: lookbackStartDate,
+        }}`
       : SQL`
-      WITH filtered AS (
-        SELECT arrayJoin(mapKeys(${data.attributesColumn})) AS attribute
-        FROM ${data.tableName}
-        WHERE projectId = ${{
-          type: TableColumnType.ObjectID,
-          value: data.projectId,
-        }}
-          AND NOT empty(${data.attributesColumn})
-          AND ${data.timeColumn} >= ${{
-            type: TableColumnType.Date,
-            value: lookbackStartDate,
-          }}`;
+      SELECT groupUniqArray(arrayJoin(mapKeys(${data.attributesColumn}))) AS keys
+      FROM ${data.tableName}
+      WHERE projectId = ${{
+        type: TableColumnType.ObjectID,
+        value: data.projectId,
+      }}
+        AND NOT empty(${data.attributesColumn})
+        AND ${data.timeColumn} >= ${{
+          type: TableColumnType.Date,
+          value: lookbackStartDate,
+        }}`;
 
     if (data.metricName) {
       statement.append(
         SQL`
-          AND name = ${{
-            type: TableColumnType.Text,
-            value: data.metricName,
-          }}`,
+        AND name = ${{
+          type: TableColumnType.Text,
+          value: data.metricName,
+        }}`,
       );
     }
 
-    statement.append(
-      SQL`
-        ORDER BY ${data.timeColumn} DESC
-        LIMIT ${{
-          type: TableColumnType.Number,
-          value: TelemetryAttributeService.ROW_SCAN_LIMIT,
-        }}
-      )
-      SELECT DISTINCT attribute
-      FROM filtered
-      WHERE attribute IS NOT NULL AND attribute != ''
-      ORDER BY attribute ASC
-      LIMIT ${{
-        type: TableColumnType.Number,
-        value: TelemetryAttributeService.ATTRIBUTES_LIMIT,
-      }}`,
-    );
-
     return statement;
   }
 
@@ -341,17 +331,29 @@ export class TelemetryAttributeService {
     }>();
 
     const rows: Array<JSONObject> = response.data || [];
+    const firstRow: JSONObject | undefined = rows[0];
+    const rawKeys: unknown = firstRow ? firstRow["keys"] : null;
 
-    const attributeKeys: Array<string> = rows
-      .map((row: JSONObject) => {
-        const attribute: unknown = row["attribute"];
+    if (!Array.isArray(rawKeys)) {
+      return [];
+    }
+
+    const attributeKeys: Array<string> = rawKeys
+      .map((attribute: unknown): string | null => {
         return typeof attribute === "string" ? attribute.trim() : null;
       })
       .filter((attribute: string | null): attribute is string => {
         return Boolean(attribute);
       });
 
-    return Array.from(new Set(attributeKeys));
+    const distinctKeys: Array<string> = Array.from(new Set(attributeKeys));
+    distinctKeys.sort((a: string, b: string): number => {
+      return a.localeCompare(b);
+    });
+    if (distinctKeys.length > TelemetryAttributeService.ATTRIBUTES_LIMIT) {
+      distinctKeys.length = TelemetryAttributeService.ATTRIBUTES_LIMIT;
+    }
+    return distinctKeys;
   }
 
   private static readonly ATTRIBUTE_VALUES_LIMIT: number = 100;

package/Server/Utils/AnalyticsDatabase/StatementGenerator.ts

@@ -20,6 +20,7 @@ import GreaterThan from "../../../Types/BaseDatabase/GreaterThan";
 import GreaterThanOrEqual from "../../../Types/BaseDatabase/GreaterThanOrEqual";
 import InBetween from "../../../Types/BaseDatabase/InBetween";
 import Includes from "../../../Types/BaseDatabase/Includes";
+import ObjectID from "../../../Types/ObjectID";
 import IsNull from "../../../Types/BaseDatabase/IsNull";
 import LessThan from "../../../Types/BaseDatabase/LessThan";
 import LessThanOrEqual from "../../../Types/BaseDatabase/LessThanOrEqual";
@@ -711,6 +712,34 @@ export default class StatementGenerator<TBaseModel extends AnalyticsBaseModel> {
             continue;
           }
 
+          /*
+           * Multi-value selection (dashboard variables, ad-hoc filters):
+           * an empty `Includes` would expand to `IN ()`, which ClickHouse
+           * treats as "match nothing" and is never the user's intent
+           * here — skip the predicate instead so a cleared multi-select
+           * behaves like "All".
+           */
+          if (mapEntry instanceof Includes) {
+            const includesValues: Array<string> = (
+              (mapEntry as Includes).values || []
+            ).map((v: string | ObjectID | number) => {
+              return String(v);
+            });
+            if (includesValues.length === 0) {
+              continue;
+            }
+            whereStatement.append(
+              SQL`AND ${key}[${{
+                value: mapKey,
+                type: TableColumnType.Text,
+              }}] IN ${{
+                value: new Includes(includesValues),
+                type: TableColumnType.Text,
+              }}`,
+            );
+            continue;
+          }
+
           // Bare string/number/boolean — direct Map subscript.
           whereStatement.append(
             SQL`AND ${key}[${{

package/Tests/Server/Utils/AnalyticsDatabase/StatementGenerator.test.ts

@@ -18,6 +18,7 @@ import NotEqual from "../../../../Types/BaseDatabase/NotEqual";
 import IsNull from "../../../../Types/BaseDatabase/IsNull";
 import NotNull from "../../../../Types/BaseDatabase/NotNull";
 import GreaterThan from "../../../../Types/BaseDatabase/GreaterThan";
+import Includes from "../../../../Types/BaseDatabase/Includes";
 import Search from "../../../../Types/BaseDatabase/Search";
 import StartsWith from "../../../../Types/BaseDatabase/StartsWith";
 
@@ -323,6 +324,41 @@ describe("StatementGenerator", () => {
         expect(statement.query).toContain("lowerUTF8");
         expect(statement.query).toContain("ILIKE");
       });
+
+      test("emits direct map subscript IN(...) for Includes wrapper", () => {
+        const statement: Statement = mapGenerator.toWhereStatement({
+          attributes: {
+            "k8s.cluster.name": new Includes(["prod-east", "prod-west"]),
+          },
+        } as any);
+        /*
+         * Multi-value dashboard variables emit Includes on a map column;
+         * the generator must produce an O(1) Map subscript followed by
+         * IN, matching the fast-path used for bare-value equality.
+         */
+        expect(statement.query).toBe(
+          "AND {p0:Identifier}[{p1:String}] IN {p2:Array(String)}",
+        );
+        expect(statement.query_params).toStrictEqual({
+          p0: "attributes",
+          p1: "k8s.cluster.name",
+          p2: ["prod-east", "prod-west"],
+        });
+      });
+
+      test("drops empty Includes wrapper instead of producing IN ()", () => {
+        const statement: Statement = mapGenerator.toWhereStatement({
+          attributes: {
+            "k8s.cluster.name": new Includes([]),
+          },
+        } as any);
+        /*
+         * An empty multi-select is the user's "All" — must not emit
+         * `IN ()` (which ClickHouse treats as match-nothing).
+         */
+        expect(statement.query).toBe("");
+        expect(statement.query_params).toStrictEqual({});
+      });
     });
   });
 

package/Types/Dashboard/DashboardVariable.ts

@@ -2,6 +2,7 @@ export enum DashboardVariableType {
   CustomList = "Custom List",
   Query = "Query",
   TextInput = "Text Input",
+  TelemetryAttribute = "Telemetry Attribute",
 }
 
 export default interface DashboardVariable {
@@ -13,6 +14,22 @@ export default interface DashboardVariable {
   customListValues?: string | undefined;
   // For Query: a ClickHouse query to populate options
   query?: string | undefined;
+  /*
+   * For TelemetryAttribute: the OpenTelemetry attribute key this
+   * variable binds to (e.g. "k8s.cluster.name"). At render time the
+   * selected value is injected into any widget filter that targets
+   * this attribute key. Options are fetched from the distinct values
+   * of this attribute across the current time range.
+   */
+  attributeKey?: string | undefined;
+  /*
+   * For TelemetryAttribute: optional metric name to scope the option
+   * lookup to. When set, the dropdown lists distinct attribute values
+   * observed on that metric only (e.g. distinct `k8s.pod.name` values
+   * emitted by `k8s.container.cpu_usage`). Leave undefined to list
+   * values across every metric in the project.
+   */
+  metricName?: string | undefined;
   // Current selected value(s)
   selectedValue?: string | undefined;
   selectedValues?: Array<string> | undefined;

package/Types/Metrics/MetricsQuery.ts

@@ -1,10 +1,11 @@
+import Includes from "../BaseDatabase/Includes";
 import Search from "../BaseDatabase/Search";
 import Dictionary from "../Dictionary";
 import MetricsAggregationType from "./MetricsAggregationType";
 
 export default interface MetricsQuery {
   metricName: string;
-  attributes: Dictionary<string | boolean | number | Search<string>>;
+  attributes: Dictionary<string | boolean | number | Search<string> | Includes>;
   aggegationType: MetricsAggregationType;
   aggregateBy: Dictionary<boolean>;
 

package/Utils/Dashboard/VariableInterpolation.ts

@@ -0,0 +1,169 @@
+import Includes from "../../Types/BaseDatabase/Includes";
+import DashboardVariable, {
+  DashboardVariableType,
+} from "../../Types/Dashboard/DashboardVariable";
+import MetricQueryConfigData from "../../Types/Metrics/MetricQueryConfigData";
+
+export interface ResolvedVariableValue {
+  /*
+   * Single-value selection. Becomes a `attributes[key] = '...'` predicate
+   * server-side.
+   */
+  scalar?: string | undefined;
+  /*
+   * Multi-value selection. Becomes a `attributes[key] IN (...)` predicate.
+   * Always non-empty when present; an empty multi-select is treated as
+   * "All" and produces neither field.
+   */
+  multi?: Array<string> | undefined;
+}
+
+/*
+ * Applies dashboard variables to a metric query at render time.
+ *
+ * For each TelemetryAttribute variable with a selected value, the
+ * variable's bound attribute key is injected into the query's attribute
+ * filter. An empty/unset selection means "All" — no filter is applied,
+ * and any previously-set filter on that key is removed so the widget
+ * shows the cross-cluster view by default.
+ *
+ * Multi-select variables emit an `Includes` operator on the same map
+ * key so the server's WHERE-builder produces `attributes[key] IN (...)`.
+ */
+export default class DashboardVariableInterpolation {
+  public static resolveValue(
+    variable: DashboardVariable,
+  ): ResolvedVariableValue | undefined {
+    if (variable.isMultiSelect) {
+      const values: Array<string> = (variable.selectedValues || []).filter(
+        (v: string) => {
+          return Boolean(v);
+        },
+      );
+      if (values.length > 0) {
+        return { multi: values };
+      }
+      // No multi-select picks. Fall through to default value handling.
+    }
+
+    const v: string | undefined =
+      variable.selectedValue ?? variable.defaultValue ?? undefined;
+    if (v === undefined || v === null || v === "") {
+      return undefined;
+    }
+    return { scalar: v };
+  }
+
+  /*
+   * Generic helper: take a plain attributes map and return a new map
+   * with all TelemetryAttribute variables applied. Used by every widget
+   * type whose filter is shaped like `{ [attrKey]: value | operator }`
+   * (metric charts, log streams). Returns the original reference when
+   * nothing changes so React.memo can short-circuit.
+   */
+  public static applyToAttributes(
+    attributes: Record<string, unknown> | undefined,
+    variables: Array<DashboardVariable> | undefined,
+  ): Record<string, unknown> {
+    const source: Record<string, unknown> = attributes || {};
+    if (!variables || variables.length === 0) {
+      return source;
+    }
+
+    const telemetryAttributeVariables: Array<DashboardVariable> =
+      variables.filter((v: DashboardVariable) => {
+        return (
+          v.type === DashboardVariableType.TelemetryAttribute &&
+          Boolean(v.attributeKey)
+        );
+      });
+
+    if (telemetryAttributeVariables.length === 0) {
+      return source;
+    }
+
+    const next: Record<string, unknown> = { ...source };
+    let changed: boolean = false;
+
+    for (const variable of telemetryAttributeVariables) {
+      const key: string = variable.attributeKey as string;
+      const resolved: ResolvedVariableValue | undefined =
+        DashboardVariableInterpolation.resolveValue(variable);
+
+      if (!resolved) {
+        if (key in next) {
+          delete next[key];
+          changed = true;
+        }
+        continue;
+      }
+
+      if (resolved.multi && resolved.multi.length > 0) {
+        next[key] = new Includes(resolved.multi);
+        changed = true;
+        continue;
+      }
+
+      if (resolved.scalar !== undefined && next[key] !== resolved.scalar) {
+        next[key] = resolved.scalar;
+        changed = true;
+      }
+    }
+
+    return changed ? next : source;
+  }
+
+  public static applyToQueryConfig(
+    queryConfig: MetricQueryConfigData,
+    variables: Array<DashboardVariable> | undefined,
+  ): MetricQueryConfigData {
+    if (!variables || variables.length === 0) {
+      return queryConfig;
+    }
+
+    const filterData: typeof queryConfig.metricQueryData.filterData =
+      queryConfig.metricQueryData?.filterData;
+
+    if (!filterData) {
+      return queryConfig;
+    }
+
+    const originalAttributes: Record<string, unknown> =
+      (filterData.attributes as Record<string, unknown> | undefined) || {};
+    const nextAttributes: Record<string, unknown> =
+      DashboardVariableInterpolation.applyToAttributes(
+        originalAttributes,
+        variables,
+      );
+
+    if (nextAttributes === originalAttributes) {
+      return queryConfig;
+    }
+
+    return {
+      ...queryConfig,
+      metricQueryData: {
+        ...queryConfig.metricQueryData,
+        filterData: {
+          ...filterData,
+          attributes: nextAttributes,
+        } as typeof queryConfig.metricQueryData.filterData,
+      },
+    };
+  }
+
+  public static applyToQueryConfigs(
+    queryConfigs: Array<MetricQueryConfigData>,
+    variables: Array<DashboardVariable> | undefined,
+  ): Array<MetricQueryConfigData> {
+    if (!variables || variables.length === 0) {
+      return queryConfigs;
+    }
+    return queryConfigs.map((config: MetricQueryConfigData) => {
+      return DashboardVariableInterpolation.applyToQueryConfig(
+        config,
+        variables,
+      );
+    });
+  }
+}