@oneuptime/common 10.8.2 → 11.0.1

package/Tests/Server/Services/AnalyticsDatabaseService.test.ts

@@ -107,7 +107,7 @@ describe("AnalyticsDatabaseService", () => {
           "    count() as count\n" +
           "FROM {p0:Identifier}.{p1:Identifier}\n" +
           "WHERE TRUE <where-statement>" +
-          " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+          " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break', max_memory_usage = 3221225472, max_bytes_before_external_group_by = 1610612736, max_bytes_before_external_sort = 1610612736",
       );
       expect(statement.query_params).toStrictEqual({
         p0: "oneuptime",
@@ -128,7 +128,7 @@ describe("AnalyticsDatabaseService", () => {
           "FROM {p0:Identifier}.{p1:Identifier}\n" +
           "WHERE TRUE <where-statement>\n" +
           "LIMIT {p2:Int32}\n" +
-          " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+          " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break', max_memory_usage = 3221225472, max_bytes_before_external_group_by = 1610612736, max_bytes_before_external_sort = 1610612736",
       );
       expect(statement.query_params).toStrictEqual({
         p0: "oneuptime",
@@ -150,7 +150,7 @@ describe("AnalyticsDatabaseService", () => {
           "FROM {p0:Identifier}.{p1:Identifier}\n" +
           "WHERE TRUE <where-statement>\n" +
           "OFFSET {p2:Int32}\n" +
-          " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+          " SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break', max_memory_usage = 3221225472, max_bytes_before_external_group_by = 1610612736, max_bytes_before_external_sort = 1610612736",
       );
       expect(statement.query_params).toStrictEqual({
         p0: "oneuptime",
@@ -215,7 +215,7 @@ describe("AnalyticsDatabaseService", () => {
 
       expect(statement.query).toBe(
         "SELECT <select-statement> FROM {p0:Identifier}.{p1:Identifier} WHERE TRUE <where-statement> ORDER BY <sort-statement> LIMIT {p2:Int32} OFFSET {p3:Int32}\n" +
-          "SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+          "SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break', max_memory_usage = 3221225472, max_bytes_before_external_group_by = 1610612736, max_bytes_before_external_sort = 1610612736",
       );
       expect(statement.query_params).toStrictEqual({
         p0: "oneuptime",
@@ -366,4 +366,79 @@ describe("AnalyticsDatabaseService", () => {
       });
     });
   });
+
+  /*
+   * Per-call exec options (clickhouse_settings + query_id) added for the
+   * telemetry V3 backfill engine — long INSERT...SELECT statements need
+   * HTTP progress-header keepalive, dedup tokens and deterministic query
+   * ids per call. Must be strictly additive: option-less calls keep the
+   * exact pre-existing exec/query payload.
+   */
+  describe("execute / executeQuery per-call options", () => {
+    let exec: ReturnType<typeof jest.fn>;
+    let query: ReturnType<typeof jest.fn>;
+
+    beforeEach(() => {
+      exec = jest.fn(() => {
+        return Promise.resolve({});
+      });
+      query = jest.fn(() => {
+        return Promise.resolve({});
+      });
+      (service as any).database = {};
+      (service as any).databaseClient = { exec, query };
+    });
+
+    test("execute threads clickhouse settings and query id into client.exec", async () => {
+      await service.execute("INSERT INTO Foo SELECT 1", {
+        clickhouseSettings: {
+          insert_deduplication_token: "v3copy:Foo:1:202601",
+          send_progress_in_http_headers: 1,
+        },
+        queryId: "v3copy:Foo:1:202601:123",
+      });
+
+      expect(exec).toHaveBeenCalledWith({
+        query: "INSERT INTO Foo SELECT 1",
+        query_params: undefined,
+        clickhouse_settings: {
+          insert_deduplication_token: "v3copy:Foo:1:202601",
+          send_progress_in_http_headers: 1,
+        },
+        query_id: "v3copy:Foo:1:202601:123",
+      });
+    });
+
+    test("execute without options keeps the pre-existing exec payload", async () => {
+      await service.execute("SELECT 1");
+
+      expect(exec).toHaveBeenCalledWith({
+        query: "SELECT 1",
+        query_params: undefined,
+      });
+    });
+
+    test("executeQuery threads clickhouse settings into client.query", async () => {
+      await service.executeQuery("SELECT 1", {
+        clickhouseSettings: { max_execution_time: 1800 },
+      });
+
+      expect(query).toHaveBeenCalledWith({
+        query: "SELECT 1",
+        format: "JSON",
+        query_params: undefined,
+        clickhouse_settings: { max_execution_time: 1800 },
+      });
+    });
+
+    test("executeQuery without options keeps the pre-existing query payload", async () => {
+      await service.executeQuery("SELECT 1");
+
+      expect(query).toHaveBeenCalledWith({
+        query: "SELECT 1",
+        format: "JSON",
+        query_params: undefined,
+      });
+    });
+  });
 });

package/Tests/Server/Services/LogAggregationService.test.ts

@@ -31,7 +31,7 @@ describe("LogAggregationService", () => {
     });
 
     expect(statement.query).toBe(
-      "SELECT toString({p0:Identifier}) AS val, count() AS cnt FROM {p1:Identifier} WHERE projectId = {p2:String} AND time >= {p3:DateTime} AND time <= {p4:DateTime} GROUP BY val ORDER BY cnt DESC LIMIT {p5:Int32} SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+      "SELECT toString({p0:Identifier}) AS val, count() AS cnt FROM {p1:Identifier} WHERE projectId = {p2:String} AND time >= {p3:DateTime} AND time <= {p4:DateTime} AND retentionDate >= now() GROUP BY val ORDER BY cnt DESC LIMIT {p5:Int32} SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break', max_memory_usage = 3221225472, max_bytes_before_external_group_by = 1610612736, max_bytes_before_external_sort = 1610612736",
     );
 
     expect(statement.query_params).toStrictEqual({
@@ -50,8 +50,13 @@ describe("LogAggregationService", () => {
       facetKey,
     });
 
+    /*
+     * attributes is Map(String, String) — facet reads MUST use subscript
+     * access + mapContains. The previous JSONExtractRaw/JSONHas shape
+     * throws ILLEGAL_TYPE_OF_ARGUMENT (Code 43) against Map columns.
+     */
     expect(statement.query).toBe(
-      "SELECT JSONExtractRaw(attributes, {p0:String}) AS val, count() AS cnt FROM {p1:Identifier} WHERE projectId = {p2:String} AND time >= {p3:DateTime} AND time <= {p4:DateTime} AND JSONHas(attributes, {p5:String}) = 1 GROUP BY val ORDER BY cnt DESC LIMIT {p6:Int32} SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break'",
+      "SELECT attributes[{p0:String}] AS val, count() AS cnt FROM {p1:Identifier} WHERE projectId = {p2:String} AND time >= {p3:DateTime} AND time <= {p4:DateTime} AND mapContains(attributes, {p5:String}) AND retentionDate >= now() GROUP BY val ORDER BY cnt DESC LIMIT {p6:Int32} SETTINGS max_execution_time = 45, timeout_overflow_mode = 'break', max_memory_usage = 3221225472, max_bytes_before_external_group_by = 1610612736, max_bytes_before_external_sort = 1610612736",
     );
 
     expect(statement.query_params).toStrictEqual({

package/Tests/Server/Services/ProfileAggregationService.test.ts

@@ -0,0 +1,280 @@
+import ProfileAggregationService, {
+  DiffFlamegraphNode,
+  FlamegraphResult,
+  ProfileFlamegraphNode,
+} from "../../../Server/Services/ProfileAggregationService";
+import ProfileSampleDatabaseService from "../../../Server/Services/ProfileSampleService";
+import { Results } from "../../../Server/Services/AnalyticsDatabaseService";
+import { JSONObject } from "../../../Types/JSON";
+import ObjectID from "../../../Types/ObjectID";
+import { describe, expect, test, afterEach, jest } from "@jest/globals";
+
+/*
+ * Replace the ClickHouse boundary: getFlamegraph builds a Statement and
+ * hands it to ProfileSampleDatabaseService.executeQuery, then everything
+ * after that (tree building, totals) is pure. Stubbing executeQuery lets
+ * the test feed grouped stack rows directly.
+ */
+const stubGroupedRows: (rows: Array<JSONObject>) => void = (
+  rows: Array<JSONObject>,
+): void => {
+  const fakeResult: Results = {
+    json: () => {
+      return Promise.resolve({ data: rows });
+    },
+  } as unknown as Results;
+
+  jest
+    .spyOn(ProfileSampleDatabaseService, "executeQuery")
+    .mockResolvedValue(fakeResult);
+};
+
+/*
+ * Invariants every flamegraph consumer relies on: a child never exceeds
+ * its parent, and a node's total is exactly its self value plus its
+ * children's totals (no double counting, no lost samples).
+ */
+const assertTreeInvariants: (node: ProfileFlamegraphNode) => void = (
+  node: ProfileFlamegraphNode,
+): void => {
+  let childrenTotal: number = 0;
+
+  for (const child of node.children) {
+    expect(child.totalValue).toBeLessThanOrEqual(node.totalValue);
+    childrenTotal += child.totalValue;
+    assertTreeInvariants(child);
+  }
+
+  expect(node.selfValue + childrenTotal).toBe(node.totalValue);
+};
+
+const findChild: (
+  node: ProfileFlamegraphNode,
+  functionName: string,
+) => ProfileFlamegraphNode = (
+  node: ProfileFlamegraphNode,
+  functionName: string,
+): ProfileFlamegraphNode => {
+  const child: ProfileFlamegraphNode | undefined = node.children.find(
+    (candidate: ProfileFlamegraphNode) => {
+      return candidate.functionName === functionName;
+    },
+  );
+  expect(child).toBeDefined();
+  return child!;
+};
+
+describe("ProfileAggregationService.getFlamegraph", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("merges grouped stack rows into a tree with consistent parent/child totals", async () => {
+    /*
+     * Three pre-aggregated stacks sharing the "main" root frame. Stored
+     * stacktraces are LEAF-FIRST (pprof/OTLP wire order), so the leaf
+     * sits at index 0 and "main" comes last. ClickHouse's JSON output
+     * renders wide numerics as strings, so one row uses string values to
+     * cover the Number() coercion path.
+     */
+    stubGroupedRows([
+      {
+        stacktrace: ["work@main.go:22", "main@main.go:10"],
+        frameTypes: ["go", "go"],
+        totalValue: "100",
+        sampleCount: "2",
+      },
+      {
+        stacktrace: ["idle@main.go:30", "main@main.go:10"],
+        frameTypes: ["go", "go"],
+        totalValue: 50,
+        sampleCount: 1,
+      },
+      {
+        stacktrace: ["main@main.go:10"],
+        frameTypes: ["go"],
+        totalValue: 25,
+        sampleCount: 1,
+      },
+    ]);
+
+    const result: FlamegraphResult =
+      await ProfileAggregationService.getFlamegraph({
+        projectId: ObjectID.generate(),
+      });
+
+    expect(result.truncated).toBe(false);
+
+    const root: ProfileFlamegraphNode = result.flamegraph;
+    expect(root.functionName).toBe("(root)");
+    expect(root.totalValue).toBe(175);
+    expect(root.selfValue).toBe(0);
+    expect(root.children.length).toBe(1);
+
+    const main: ProfileFlamegraphNode = findChild(root, "main");
+    expect(main.fileName).toBe("main.go");
+    expect(main.lineNumber).toBe(10);
+    expect(main.frameType).toBe("go");
+    // All three stacks pass through main; only the third ends there.
+    expect(main.totalValue).toBe(175);
+    expect(main.selfValue).toBe(25);
+    expect(main.children.length).toBe(2);
+
+    const work: ProfileFlamegraphNode = findChild(main, "work");
+    expect(work.totalValue).toBe(100);
+    expect(work.selfValue).toBe(100);
+    expect(work.lineNumber).toBe(22);
+    expect(work.children.length).toBe(0);
+
+    const idle: ProfileFlamegraphNode = findChild(main, "idle");
+    expect(idle.totalValue).toBe(50);
+    expect(idle.selfValue).toBe(50);
+
+    assertTreeInvariants(root);
+  });
+
+  test("skips empty stacktraces and tolerates frames without the @file:line shape", async () => {
+    stubGroupedRows([
+      {
+        stacktrace: [],
+        frameTypes: [],
+        totalValue: 999,
+        sampleCount: 1,
+      },
+      {
+        stacktrace: ["bareframe"],
+        frameTypes: [],
+        totalValue: 10,
+        sampleCount: 1,
+      },
+    ]);
+
+    const result: FlamegraphResult =
+      await ProfileAggregationService.getFlamegraph({
+        projectId: ObjectID.generate(),
+      });
+
+    const root: ProfileFlamegraphNode = result.flamegraph;
+
+    // The empty-stack row contributes nothing — not even to the root total.
+    expect(root.totalValue).toBe(10);
+    expect(root.children.length).toBe(1);
+
+    const bare: ProfileFlamegraphNode = findChild(root, "bareframe");
+    expect(bare.fileName).toBe("");
+    expect(bare.lineNumber).toBe(0);
+    expect(bare.totalValue).toBe(10);
+
+    assertTreeInvariants(root);
+  });
+
+  test("flags truncation when the unique-stack fetch cap is hit", async () => {
+    const cappedRows: Array<JSONObject> = Array.from(
+      { length: 10000 },
+      (_unused: unknown, index: number) => {
+        return {
+          stacktrace: [`fn${index}@file.ts:1`],
+          frameTypes: ["node"],
+          totalValue: 1,
+          sampleCount: 1,
+        };
+      },
+    );
+
+    stubGroupedRows(cappedRows);
+
+    const result: FlamegraphResult =
+      await ProfileAggregationService.getFlamegraph({
+        projectId: ObjectID.generate(),
+      });
+
+    expect(result.truncated).toBe(true);
+    expect(result.flamegraph.totalValue).toBe(10000);
+    assertTreeInvariants(result.flamegraph);
+  });
+});
+
+describe("ProfileAggregationService.mergeDiffTrees", () => {
+  type MergeDiffTrees = (
+    baseline: ProfileFlamegraphNode | null,
+    comparison: ProfileFlamegraphNode | null,
+  ) => DiffFlamegraphNode;
+
+  /*
+   * mergeDiffTrees is private but pure; reaching it through a cast keeps
+   * the diff math testable without a ClickHouse round trip.
+   */
+  const mergeDiffTrees: MergeDiffTrees = (
+    ProfileAggregationService as unknown as {
+      mergeDiffTrees: MergeDiffTrees;
+    }
+  ).mergeDiffTrees.bind(ProfileAggregationService);
+
+  const makeNode: (
+    functionName: string,
+    totalValue: number,
+    selfValue: number,
+    children?: Array<ProfileFlamegraphNode>,
+  ) => ProfileFlamegraphNode = (
+    functionName: string,
+    totalValue: number,
+    selfValue: number,
+    children: Array<ProfileFlamegraphNode> = [],
+  ): ProfileFlamegraphNode => {
+    return {
+      functionName,
+      fileName: "main.go",
+      lineNumber: 1,
+      selfValue,
+      totalValue,
+      children,
+      frameType: "go",
+    };
+  };
+
+  test("computes deltas across both windows and unions children", async () => {
+    const baseline: ProfileFlamegraphNode = makeNode("(root)", 100, 0, [
+      makeNode("stable", 60, 60),
+      makeNode("removed", 40, 40),
+    ]);
+    const comparison: ProfileFlamegraphNode = makeNode("(root)", 150, 0, [
+      makeNode("stable", 90, 90),
+      makeNode("added", 60, 60),
+    ]);
+
+    const diff: DiffFlamegraphNode = mergeDiffTrees(baseline, comparison);
+
+    expect(diff.baselineValue).toBe(100);
+    expect(diff.comparisonValue).toBe(150);
+    expect(diff.delta).toBe(50);
+    expect(diff.deltaPercent).toBe(50);
+    expect(diff.children.length).toBe(3);
+
+    // Children come back sorted by comparison value, descending.
+    expect(
+      diff.children.map((child: DiffFlamegraphNode) => {
+        return child.functionName;
+      }),
+    ).toEqual(["stable", "added", "removed"]);
+
+    const stable: DiffFlamegraphNode = diff.children[0]!;
+    expect(stable.baselineValue).toBe(60);
+    expect(stable.comparisonValue).toBe(90);
+    expect(stable.delta).toBe(30);
+    expect(stable.deltaPercent).toBe(50);
+
+    // Present only in the comparison window: +100% by convention.
+    const added: DiffFlamegraphNode = diff.children[1]!;
+    expect(added.baselineValue).toBe(0);
+    expect(added.comparisonValue).toBe(60);
+    expect(added.delta).toBe(60);
+    expect(added.deltaPercent).toBe(100);
+
+    // Present only in the baseline window: the full value drops away.
+    const removed: DiffFlamegraphNode = diff.children[2]!;
+    expect(removed.baselineValue).toBe(40);
+    expect(removed.comparisonValue).toBe(0);
+    expect(removed.delta).toBe(-40);
+    expect(removed.deltaPercent).toBe(-100);
+  });
+});

package/Tests/Server/Services/ProfileBreakdown.test.ts

@@ -0,0 +1,161 @@
+import ProfileAggregationService, {
+  BreakdownResult,
+} from "../../../Server/Services/ProfileAggregationService";
+import ProfileDatabaseService from "../../../Server/Services/ProfileService";
+import { Results } from "../../../Server/Services/AnalyticsDatabaseService";
+import { JSONObject } from "../../../Types/JSON";
+import ObjectID from "../../../Types/ObjectID";
+import { describe, expect, test, afterEach, jest } from "@jest/globals";
+
+/*
+ * Replace the ClickHouse boundary: getBreakdown reads pre-grouped rows
+ * from the Profile table via ProfileDatabaseService.executeQuery (NOT
+ * the sample service — breakdowns never touch the huge ProfileSample
+ * table). Everything after the read (totals, shares, limit) is pure.
+ */
+const stubBreakdownRows: (rows: Array<JSONObject>) => void = (
+  rows: Array<JSONObject>,
+): void => {
+  const fakeResult: Results = {
+    json: () => {
+      return Promise.resolve({ data: rows });
+    },
+  } as unknown as Results;
+
+  jest
+    .spyOn(ProfileDatabaseService, "executeQuery")
+    .mockResolvedValue(fakeResult);
+};
+
+describe("ProfileAggregationService.getBreakdown", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  test("computes shares against the pre-limit total and keeps descending order", async () => {
+    /*
+     * Rows arrive sorted by summed sampleCount descending — the query's
+     * ORDER BY guarantees it, so the service preserves order instead of
+     * re-sorting. ClickHouse's JSON output renders wide numerics as
+     * strings, so one row uses string values to cover Number() coercion.
+     * The empty-value row stands in for profiles missing the dimension
+     * and must be dropped before it pollutes the total.
+     */
+    stubBreakdownRows([
+      {
+        breakdownValue: "svc-a",
+        totalSampleCount: "500",
+        profileCount: "5",
+      },
+      {
+        breakdownValue: "svc-b",
+        totalSampleCount: 300,
+        profileCount: 3,
+      },
+      {
+        breakdownValue: "svc-c",
+        totalSampleCount: 200,
+        profileCount: 2,
+      },
+      {
+        breakdownValue: "",
+        totalSampleCount: 999,
+        profileCount: 9,
+      },
+    ]);
+
+    const result: BreakdownResult =
+      await ProfileAggregationService.getBreakdown({
+        projectId: ObjectID.generate(),
+        startTime: new Date("2026-06-01T00:00:00Z"),
+        endTime: new Date("2026-06-02T00:00:00Z"),
+        breakdownBy: "service",
+        limit: 2,
+      });
+
+    /*
+     * The total covers ALL named groups (including svc-c, which the
+     * limit drops) but not the empty-value row — shares must stay
+     * percentages of the real volume, not of the visible top-N.
+     */
+    expect(result.totalSampleCount).toBe(1000);
+
+    // Limit applies AFTER the total: only the top two groups come back.
+    expect(result.items.length).toBe(2);
+    expect(result.items[0]!.value).toBe("svc-a");
+    expect(result.items[0]!.sampleCount).toBe(500);
+    expect(result.items[0]!.profileCount).toBe(5);
+    expect(result.items[0]!.share).toBe(50);
+
+    expect(result.items[1]!.value).toBe("svc-b");
+    expect(result.items[1]!.sampleCount).toBe(300);
+    expect(result.items[1]!.share).toBe(30);
+
+    // Descending order survives the trip through the service.
+    expect(result.items[0]!.sampleCount).toBeGreaterThanOrEqual(
+      result.items[1]!.sampleCount,
+    );
+
+    // Truncated shares undercount: they sum to at most 100, never more.
+    const shareSum: number = result.items.reduce(
+      (sum: number, item: { share: number }) => {
+        return sum + item.share;
+      },
+      0,
+    );
+    expect(shareSum).toBeLessThanOrEqual(100);
+    expect(shareSum).toBe(80);
+  });
+
+  test("shares sum to exactly 100 when no group is dropped by the limit", async () => {
+    stubBreakdownRows([
+      { breakdownValue: "svc-a", totalSampleCount: 500, profileCount: 5 },
+      { breakdownValue: "svc-b", totalSampleCount: 300, profileCount: 3 },
+      { breakdownValue: "svc-c", totalSampleCount: 200, profileCount: 2 },
+    ]);
+
+    const result: BreakdownResult =
+      await ProfileAggregationService.getBreakdown({
+        projectId: ObjectID.generate(),
+        startTime: new Date("2026-06-01T00:00:00Z"),
+        endTime: new Date("2026-06-02T00:00:00Z"),
+        breakdownBy: "service",
+      });
+
+    // The default limit (10) exceeds the group count — nothing dropped.
+    expect(result.items.length).toBe(3);
+    expect(result.totalSampleCount).toBe(1000);
+
+    const shareSum: number = result.items.reduce(
+      (sum: number, item: { share: number }) => {
+        return sum + item.share;
+      },
+      0,
+    );
+    expect(shareSum).toBe(100);
+  });
+
+  test("guards share math against a zero total instead of producing NaN", async () => {
+    /*
+     * A group can legitimately report zero samples (e.g. profiles
+     * ingested with empty sample sets). 0/0 would yield NaN and break
+     * every percentage formatter downstream.
+     */
+    stubBreakdownRows([
+      { breakdownValue: "svc-empty", totalSampleCount: 0, profileCount: 1 },
+    ]);
+
+    const result: BreakdownResult =
+      await ProfileAggregationService.getBreakdown({
+        projectId: ObjectID.generate(),
+        startTime: new Date("2026-06-01T00:00:00Z"),
+        endTime: new Date("2026-06-02T00:00:00Z"),
+        breakdownBy: "service",
+      });
+
+    expect(result.totalSampleCount).toBe(0);
+    expect(result.items.length).toBe(1);
+    expect(result.items[0]!.share).toBe(0);
+    expect(Number.isNaN(result.items[0]!.share)).toBe(false);
+  });
+});