@oneuptime/common 11.5.9 → 11.5.11

package/Tests/Utils/Metrics/MetricFormulaEvaluator.test.ts

@@ -26,6 +26,20 @@ const buildQueryConfig: BuildQueryConfigFunction = (
   };
 };
 
+type BuildGroupedQueryConfigFunction = (
+  variable: string,
+  groupByAttributeKeys: Array<string>,
+) => MetricQueryConfigData;
+
+const buildGroupedQueryConfig: BuildGroupedQueryConfigFunction = (
+  variable: string,
+  groupByAttributeKeys: Array<string>,
+): MetricQueryConfigData => {
+  const config: MetricQueryConfigData = buildQueryConfig(variable);
+  config.metricQueryData.groupByAttributeKeys = groupByAttributeKeys;
+  return config;
+};
+
 type BuildResultFunction = (
   points: Array<{ timestamp: string; value: number }>,
 ) => AggregatedResult;
@@ -43,6 +57,38 @@ const buildResult: BuildResultFunction = (
   };
 };
 
+type BuildGroupedResultFunction = (
+  points: Array<{
+    timestamp: string;
+    value: number;
+    attributes: Record<string, string>;
+  }>,
+) => AggregatedResult;
+
+const buildGroupedResult: BuildGroupedResultFunction = (
+  points: Array<{
+    timestamp: string;
+    value: number;
+    attributes: Record<string, string>;
+  }>,
+): AggregatedResult => {
+  return {
+    data: points.map(
+      (point: {
+        timestamp: string;
+        value: number;
+        attributes: Record<string, string>;
+      }) => {
+        return {
+          timestamp: new Date(point.timestamp),
+          value: point.value,
+          attributes: point.attributes,
+        };
+      },
+    ),
+  };
+};
+
 describe("MetricFormulaEvaluator", () => {
   describe("evaluateFormula", () => {
     test("adds two series point-by-point", () => {
@@ -230,6 +276,402 @@ describe("MetricFormulaEvaluator", () => {
         });
       }).toThrow(/unknown variable/i);
     });
+
+    test("throws a structural error for malformed formulas like a**b", () => {
+      expect(() => {
+        MetricFormulaEvaluator.evaluateFormula({
+          formula: "a ** b",
+          queryConfigs: [buildQueryConfig("a"), buildQueryConfig("b")],
+          formulaConfigs: [],
+          results: [
+            buildResult([{ timestamp: "2024-01-01T00:00:00.000Z", value: 2 }]),
+            buildResult([{ timestamp: "2024-01-01T00:00:00.000Z", value: 3 }]),
+          ],
+        });
+      }).toThrow(/missing an operand/i);
+    });
+  });
+
+  describe("group-aware evaluation", () => {
+    const timestamps: Array<string> = [
+      "2024-01-01T00:00:00.000Z",
+      "2024-01-01T00:01:00.000Z",
+    ];
+
+    test("evaluates grouped formulas once per group and stamps group attributes", () => {
+      const queryConfigs: Array<MetricQueryConfigData> = [
+        buildGroupedQueryConfig("a", ["host.name"]),
+        buildGroupedQueryConfig("b", ["host.name"]),
+      ];
+
+      const results: Array<AggregatedResult> = [
+        buildGroupedResult([
+          {
+            timestamp: timestamps[0]!,
+            value: 10,
+            attributes: { "host.name": "web-1" },
+          },
+          {
+            timestamp: timestamps[0]!,
+            value: 40,
+            attributes: { "host.name": "web-2" },
+          },
+          {
+            timestamp: timestamps[1]!,
+            value: 20,
+            attributes: { "host.name": "web-1" },
+          },
+          {
+            timestamp: timestamps[1]!,
+            value: 80,
+            attributes: { "host.name": "web-2" },
+          },
+        ]),
+        buildGroupedResult([
+          {
+            timestamp: timestamps[0]!,
+            value: 2,
+            attributes: { "host.name": "web-1" },
+          },
+          {
+            timestamp: timestamps[0]!,
+            value: 4,
+            attributes: { "host.name": "web-2" },
+          },
+          {
+            timestamp: timestamps[1]!,
+            value: 2,
+            attributes: { "host.name": "web-1" },
+          },
+          {
+            timestamp: timestamps[1]!,
+            value: 4,
+            attributes: { "host.name": "web-2" },
+          },
+        ]),
+      ];
+
+      const output: AggregatedResult = MetricFormulaEvaluator.evaluateFormula({
+        formula: "a / b",
+        queryConfigs,
+        formulaConfigs: [],
+        results,
+      });
+
+      // Groups are ordered by canonical label key: web-1 first, then web-2.
+      expect(output.data).toEqual([
+        {
+          timestamp: new Date(timestamps[0]!),
+          value: 5,
+          attributes: { "host.name": "web-1" },
+        },
+        {
+          timestamp: new Date(timestamps[1]!),
+          value: 10,
+          attributes: { "host.name": "web-1" },
+        },
+        {
+          timestamp: new Date(timestamps[0]!),
+          value: 10,
+          attributes: { "host.name": "web-2" },
+        },
+        {
+          timestamp: new Date(timestamps[1]!),
+          value: 20,
+          attributes: { "host.name": "web-2" },
+        },
+      ]);
+    });
+
+    test("broadcasts an ungrouped variable across groups", () => {
+      const queryConfigs: Array<MetricQueryConfigData> = [
+        buildGroupedQueryConfig("a", ["host.name"]),
+        buildQueryConfig("b"),
+      ];
+
+      const results: Array<AggregatedResult> = [
+        buildGroupedResult([
+          {
+            timestamp: timestamps[0]!,
+            value: 10,
+            attributes: { "host.name": "web-1" },
+          },
+          {
+            timestamp: timestamps[0]!,
+            value: 30,
+            attributes: { "host.name": "web-2" },
+          },
+        ]),
+        buildResult([{ timestamp: timestamps[0]!, value: 2 }]),
+      ];
+
+      const output: AggregatedResult = MetricFormulaEvaluator.evaluateFormula({
+        formula: "a / b",
+        queryConfigs,
+        formulaConfigs: [],
+        results,
+      });
+
+      expect(output.data).toEqual([
+        {
+          timestamp: new Date(timestamps[0]!),
+          value: 5,
+          attributes: { "host.name": "web-1" },
+        },
+        {
+          timestamp: new Date(timestamps[0]!),
+          value: 15,
+          attributes: { "host.name": "web-2" },
+        },
+      ]);
+    });
+
+    test("throws a structural error when grouped variables share no groups", () => {
+      const queryConfigs: Array<MetricQueryConfigData> = [
+        buildGroupedQueryConfig("a", ["host.name"]),
+        buildGroupedQueryConfig("b", ["region"]),
+      ];
+
+      const results: Array<AggregatedResult> = [
+        buildGroupedResult([
+          {
+            timestamp: timestamps[0]!,
+            value: 1,
+            attributes: { "host.name": "web-1" },
+          },
+          {
+            timestamp: timestamps[0]!,
+            value: 2,
+            attributes: { "host.name": "web-2" },
+          },
+        ]),
+        buildGroupedResult([
+          {
+            timestamp: timestamps[0]!,
+            value: 3,
+            attributes: { region: "us-east" },
+          },
+          {
+            timestamp: timestamps[0]!,
+            value: 4,
+            attributes: { region: "eu-west" },
+          },
+        ]),
+      ];
+
+      expect(() => {
+        MetricFormulaEvaluator.evaluateFormula({
+          formula: "a + b",
+          queryConfigs,
+          formulaConfigs: [],
+          results,
+        });
+      }).toThrow(/no series groups in common/i);
+    });
+
+    test("joins a grouped variable that collapsed to a single series by group key (no cross-group broadcast)", () => {
+      /*
+       * Regression: $a and $b are both grouped by host.name, but in the
+       * charted window $b only returned rows for web-1 (web-2's
+       * denominator metric is missing). $b must NOT be treated as
+       * ungrouped-and-broadcast — that would compute a(web-2)/b(web-1)
+       * and stamp it as the web-2 series. Per the documented "groups
+       * present in ALL grouped variables" semantics, web-2 is dropped.
+       */
+      const queryConfigs: Array<MetricQueryConfigData> = [
+        buildGroupedQueryConfig("a", ["host.name"]),
+        buildGroupedQueryConfig("b", ["host.name"]),
+      ];
+
+      const results: Array<AggregatedResult> = [
+        buildGroupedResult([
+          {
+            timestamp: timestamps[0]!,
+            value: 10,
+            attributes: { "host.name": "web-1" },
+          },
+          {
+            timestamp: timestamps[0]!,
+            value: 40,
+            attributes: { "host.name": "web-2" },
+          },
+        ]),
+        buildGroupedResult([
+          {
+            timestamp: timestamps[0]!,
+            value: 2,
+            attributes: { "host.name": "web-1" },
+          },
+        ]),
+      ];
+
+      const output: AggregatedResult = MetricFormulaEvaluator.evaluateFormula({
+        formula: "a / b",
+        queryConfigs,
+        formulaConfigs: [],
+        results,
+      });
+
+      // Only web-1 (the common group) is emitted — and with web-1 math.
+      expect(output.data).toEqual([
+        {
+          timestamp: new Date(timestamps[0]!),
+          value: 5,
+          attributes: { "host.name": "web-1" },
+        },
+      ]);
+    });
+
+    test("still broadcasts a config-grouped variable whose result is EMPTY (degrades to gaps, not a structural error)", () => {
+      const queryConfigs: Array<MetricQueryConfigData> = [
+        buildGroupedQueryConfig("a", ["host.name"]),
+        buildGroupedQueryConfig("b", ["host.name"]),
+      ];
+
+      const results: Array<AggregatedResult> = [
+        buildGroupedResult([
+          {
+            timestamp: timestamps[0]!,
+            value: 10,
+            attributes: { "host.name": "web-1" },
+          },
+          {
+            timestamp: timestamps[0]!,
+            value: 40,
+            attributes: { "host.name": "web-2" },
+          },
+        ]),
+        buildGroupedResult([]),
+      ];
+
+      const output: AggregatedResult = MetricFormulaEvaluator.evaluateFormula({
+        formula: "a / b",
+        queryConfigs,
+        formulaConfigs: [],
+        results,
+      });
+
+      // Every point misses $b, so every point is skipped silently.
+      expect(output.data).toEqual([]);
+    });
+
+    test("single-series inputs stay byte-identical to ungrouped behavior", () => {
+      /*
+       * The metric-monitor worker pre-buckets per series fingerprint and
+       * calls the evaluator with one series per variable — even though
+       * the query configs still carry groupByAttributeKeys and the rows
+       * still carry full attributes. Output must be exactly the legacy
+       * shape: unstamped {timestamp, value} rows.
+       */
+      const queryConfigs: Array<MetricQueryConfigData> = [
+        buildGroupedQueryConfig("a", ["host.name"]),
+        buildGroupedQueryConfig("b", ["host.name"]),
+      ];
+
+      const results: Array<AggregatedResult> = [
+        buildGroupedResult([
+          {
+            timestamp: timestamps[0]!,
+            value: 10,
+            attributes: { "host.name": "web-1", "process.pid": "42" },
+          },
+        ]),
+        buildGroupedResult([
+          {
+            timestamp: timestamps[0]!,
+            value: 4,
+            attributes: { "host.name": "web-1", "process.pid": "43" },
+          },
+        ]),
+      ];
+
+      const output: AggregatedResult = MetricFormulaEvaluator.evaluateFormula({
+        formula: "a - b",
+        queryConfigs,
+        formulaConfigs: [],
+        results,
+      });
+
+      expect(output.data).toEqual([
+        { timestamp: new Date(timestamps[0]!), value: 6 },
+      ]);
+      expect(Object.keys(output.data[0]!)).toEqual(["timestamp", "value"]);
+    });
+
+    test("skips per-point gaps within a group silently", () => {
+      const queryConfigs: Array<MetricQueryConfigData> = [
+        buildGroupedQueryConfig("a", ["host.name"]),
+        buildGroupedQueryConfig("b", ["host.name"]),
+      ];
+
+      const results: Array<AggregatedResult> = [
+        buildGroupedResult([
+          {
+            timestamp: timestamps[0]!,
+            value: 10,
+            attributes: { "host.name": "web-1" },
+          },
+          {
+            timestamp: timestamps[1]!,
+            value: 20,
+            attributes: { "host.name": "web-1" },
+          },
+          {
+            timestamp: timestamps[0]!,
+            value: 30,
+            attributes: { "host.name": "web-2" },
+          },
+          {
+            timestamp: timestamps[1]!,
+            value: 40,
+            attributes: { "host.name": "web-2" },
+          },
+        ]),
+        buildGroupedResult([
+          // web-1 is missing the first timestamp for "b".
+          {
+            timestamp: timestamps[1]!,
+            value: 2,
+            attributes: { "host.name": "web-1" },
+          },
+          {
+            timestamp: timestamps[0]!,
+            value: 3,
+            attributes: { "host.name": "web-2" },
+          },
+          {
+            timestamp: timestamps[1]!,
+            value: 4,
+            attributes: { "host.name": "web-2" },
+          },
+        ]),
+      ];
+
+      const output: AggregatedResult = MetricFormulaEvaluator.evaluateFormula({
+        formula: "a / b",
+        queryConfigs,
+        formulaConfigs: [],
+        results,
+      });
+
+      expect(output.data).toEqual([
+        {
+          timestamp: new Date(timestamps[1]!),
+          value: 10,
+          attributes: { "host.name": "web-1" },
+        },
+        {
+          timestamp: new Date(timestamps[0]!),
+          value: 10,
+          attributes: { "host.name": "web-2" },
+        },
+        {
+          timestamp: new Date(timestamps[1]!),
+          value: 10,
+          attributes: { "host.name": "web-2" },
+        },
+      ]);
+    });
   });
 
   describe("validateFormula", () => {
@@ -265,5 +707,37 @@ describe("MetricFormulaEvaluator", () => {
       });
       expect(message).toMatch(/required/i);
     });
+
+    test("rejects doubled binary operators like a**b", () => {
+      const message: string | null = MetricFormulaEvaluator.validateFormula({
+        formula: "a ** b",
+        availableVariables: ["a", "b"],
+      });
+      expect(message).toMatch(/missing an operand/i);
+    });
+
+    test("rejects trailing operators like a+", () => {
+      const message: string | null = MetricFormulaEvaluator.validateFormula({
+        formula: "a +",
+        availableVariables: ["a"],
+      });
+      expect(message).toMatch(/missing an operand/i);
+    });
+
+    test("rejects empty parentheses", () => {
+      const message: string | null = MetricFormulaEvaluator.validateFormula({
+        formula: "()",
+        availableVariables: ["a"],
+      });
+      expect(message).toMatch(/empty/i);
+    });
+
+    test("rejects adjacent values with no operator", () => {
+      const message: string | null = MetricFormulaEvaluator.validateFormula({
+        formula: "a b",
+        availableVariables: ["a", "b"],
+      });
+      expect(message).toMatch(/single value/i);
+    });
   });
 });

package/Tests/Utils/Telemetry/HeartbeatAvailability.test.ts

@@ -234,13 +234,61 @@ describe("HeartbeatAvailabilityUtil.buildAvailabilitySeries", () => {
     expect(downPoints(result).length).toBeGreaterThanOrEqual(2);
   });
 
-  test("hour buckets are used for windows over 3 hours", () => {
+  test("5-minute buckets are used for windows over 3 hours", () => {
+    const windowEnd: Date = NOW;
+    const windowStart: Date = new Date(NOW.getTime() - 6 * 60 * MINUTE);
+    const bucketMs: number = 5 * MINUTE;
+    const base: number =
+      Math.floor(windowStart.getTime() / bucketMs) * bucketMs;
+    const rows: Array<AggregatedModel> = Array.from(
+      { length: 74 },
+      (_: unknown, i: number): AggregatedModel => {
+        return row(new Date(base + i * bucketMs));
+      },
+    );
+    const result: HeartbeatAvailabilityResult = build(
+      rows,
+      windowStart,
+      windowEnd,
+      NOW,
+    );
+    expect(result.uptimePercent).toBe(100);
+    // ~72 five-minute buckets, not ~360 minute buckets.
+    expect(result.points.length).toBeLessThanOrEqual(74);
+    expect(result.points.length).toBeGreaterThan(60);
+  });
+
+  test("15-minute buckets are used for a 24-hour window", () => {
     const windowEnd: Date = NOW;
     const windowStart: Date = new Date(NOW.getTime() - 24 * 60 * MINUTE);
+    const bucketMs: number = 15 * MINUTE;
+    const base: number =
+      Math.floor(windowStart.getTime() / bucketMs) * bucketMs;
+    const rows: Array<AggregatedModel> = Array.from(
+      { length: 98 },
+      (_: unknown, i: number): AggregatedModel => {
+        return row(new Date(base + i * bucketMs));
+      },
+    );
+    const result: HeartbeatAvailabilityResult = build(
+      rows,
+      windowStart,
+      windowEnd,
+      NOW,
+    );
+    expect(result.uptimePercent).toBe(100);
+    // ~96 fifteen-minute buckets, not ~1440 minute buckets.
+    expect(result.points.length).toBeLessThanOrEqual(98);
+    expect(result.points.length).toBeGreaterThan(90);
+  });
+
+  test("hour buckets are used for windows over 3 days", () => {
+    const windowEnd: Date = NOW;
+    const windowStart: Date = new Date(NOW.getTime() - 5 * 24 * 60 * MINUTE);
     const hourMs: number = 60 * MINUTE;
     const base: number = Math.floor(windowStart.getTime() / hourMs) * hourMs;
     const rows: Array<AggregatedModel> = Array.from(
-      { length: 25 },
+      { length: 122 },
       (_: unknown, i: number): AggregatedModel => {
         return row(new Date(base + i * hourMs));
       },
@@ -252,8 +300,8 @@ describe("HeartbeatAvailabilityUtil.buildAvailabilitySeries", () => {
       NOW,
     );
     expect(result.uptimePercent).toBe(100);
-    // ~24 hour buckets, not ~1440 minute buckets.
-    expect(result.points.length).toBeLessThanOrEqual(26);
+    // ~120 hour buckets, not ~7200 minute buckets.
+    expect(result.points.length).toBeLessThanOrEqual(122);
   });
 
   test("returns null uptime when nothing is evaluable", () => {

package/Types/AnalyticsDatabase/AnalyticsTableName.ts

@@ -4,6 +4,7 @@ enum AnalyticsTableName {
   ExceptionInstance = "ExceptionItemV3",
   Span = "SpanItemV3",
   MonitorLog = "MonitorLogV3",
+  NetworkFlow = "NetworkFlowV1",
   Profile = "ProfileItemV3",
   ProfileSample = "ProfileSampleItemV3",
   AuditLog = "AuditLogV2",
@@ -20,6 +21,16 @@ enum AnalyticsTableName {
    * hostEntityKey instead of the raw host.name spelling.
    */
   MetricItemAggMV1mByHostV2 = "MetricItemAggMV1mByHostV2",
+  /*
+   * Entity-keyed siblings of MetricItemAggMV1mByHostV2 — same layout,
+   * keyed by the other ingest-stamped scalar entity-key columns
+   * (serviceEntityKey / k8sClusterEntityKey / containerEntityKey) so
+   * entity-scoped chart queries can be served from a pre-aggregated
+   * rollup instead of scanning MetricItemV3.
+   */
+  MetricItemAggMV1mByService = "MetricItemAggMV1mByService",
+  MetricItemAggMV1mByK8sCluster = "MetricItemAggMV1mByK8sCluster",
+  MetricItemAggMV1mByContainer = "MetricItemAggMV1mByContainer",
   MetricBaselineHourly = "MetricBaselineHourly",
   MutableMetric = "MutableMetricItem",
 }

package/Types/BaseDatabase/AggregateBy.ts

@@ -10,8 +10,9 @@ export default interface AggregateBy<TBaseModel extends GenericObject> {
   aggregationType: AggregationType;
   /**
    * Optional explicit time-bucket size. When omitted (the default), the
-   * bucket size is derived from the query window (≤3h → minute, ≤7d →
-   * hour, ≤6w → day, …) — see AggregationIntervalUtil. Set this to pin a
+   * bucket size is derived from the query window (≤3h → minute, ≤12h →
+   * 5 minutes, ≤24h → 15 minutes, ≤3d → 30 minutes, ≤7d → hour, ≤6w →
+   * day, …) — see AggregationIntervalUtil. Set this to pin a
    * granularity independent of the window (e.g. `Day` buckets over a
    * one-week window), or to `Total` to aggregate the whole window into a
    * single value per group (no time bucketing). Invalid values are
@@ -38,4 +39,32 @@ export default interface AggregateBy<TBaseModel extends GenericObject> {
    * `attributes` object containing exactly these keys.
    */
   groupByAttributeKeys?: Array<string> | undefined;
+  /**
+   * Server-side Top-K group selection for grouped aggregations. When
+   * set (and the aggregation is grouped via `groupBy` /
+   * `groupByAttributeKeys`), the server ranks groups by `rankBy`
+   * (max or avg of the aggregated column) over the WHOLE query window,
+   * keeps only the top `count` groups, and reports the total number of
+   * matching groups via `AggregatedResult.totalGroups` — so a chart can
+   * plot the top 10 series without fetching every bucket of every
+   * group. Ignored for ungrouped aggregations. Plain JSON so it
+   * round-trips the API's serialize/deserialize unchanged.
+   */
+  topK?:
+    | {
+        count: number;
+        rankBy: "max" | "avg";
+      }
+    | undefined;
+  /**
+   * What ClickHouse does when the aggregate's execution-time cap fires.
+   * 'break' (the default when omitted) returns the partial buckets
+   * computed so far — acceptable for chart rendering. Callers that
+   * ALERT on the result (the metric-monitor worker) must pass 'throw'
+   * so a timed-out evaluation fails loudly instead of silently scoring
+   * partial data as if it were complete. The server allow-lists the
+   * value before it reaches SQL: anything other than the exact string
+   * "throw" is treated as 'break'.
+   */
+  timeoutOverflowMode?: "break" | "throw" | undefined;
 }

package/Types/BaseDatabase/AggregatedResult.ts

@@ -2,4 +2,26 @@ import AggregatedModel from "./AggregatedModel";
 
 export default interface AggregatedResult {
   data: Array<AggregatedModel>;
+  /**
+   * Total number of distinct groups matching the query window, counted
+   * in the Top-K ranking phase. Only present for grouped aggregations
+   * executed with `AggregateBy.topK`; compare against `topK.count` to
+   * know how many series were left out.
+   */
+  totalGroups?: number | undefined;
+  /**
+   * True when the result is known (or heuristically likely) to be
+   * incomplete: the row count hit the applied LIMIT, or Top-K dropped
+   * groups (`totalGroups > topK.count`). Additive — absent/false means
+   * the window was fully served.
+   */
+  truncated?: boolean | undefined;
+  /**
+   * Human-readable reason this result carries no data — e.g. a metric
+   * formula that failed structural validation (bad syntax, unknown
+   * variable, disjoint groups). Only set by client-side evaluation
+   * layers; never returned by the server. Chart layers should render it
+   * in place of the empty chart instead of showing a silent "no data".
+   */
+  errorMessage?: string | undefined;
 }

package/Types/BaseDatabase/AggregationInterval.ts

@@ -1,5 +1,16 @@
 export enum AggregationInterval {
   Minute = "Minute",
+  /*
+   * The sub-hour tiers are served from the same 1-minute rollup as
+   * `Minute` (any bucket that is a whole multiple of a minute is >= the
+   * MV's resolution). Unlike the calendar units they are NOT valid
+   * ClickHouse `date_trunc`/INTERVAL-1 units, so SQL builders must go
+   * through AggregateUtil.buildBucketTimestampExpression instead of
+   * lowercasing the enum value into the statement.
+   */
+  FiveMinutes = "FiveMinutes",
+  FifteenMinutes = "FifteenMinutes",
+  ThirtyMinutes = "ThirtyMinutes",
   Hour = "Hour",
   Day = "Day",
   Week = "Week",