@oneuptime/common 12.0.24 → 12.0.25

package/Tests/App/Dashboard/LlmCallsTableIdentity.test.tsx

@@ -0,0 +1,322 @@
+import "@testing-library/jest-dom";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  jest,
+  test,
+} from "@jest/globals";
+import { act, cleanup, render, screen } from "@testing-library/react";
+import * as React from "react";
+import { MemoryRouter } from "react-router-dom";
+
+/*
+ * ---------------------------------------------------------------------------
+ * The LLM calls table's employee columns and filters
+ * ---------------------------------------------------------------------------
+ *
+ * "Which of our engineers spent this?" is the question this table exists to
+ * answer, and everything that answers it is configuration passed as props:
+ * the User column and its email→id fallback, the Team column, the three text
+ * filters, and the llmUserId entry in selectMoreFields. Dropping any of them
+ * type-checks and renders — the table simply stops answering the question, or
+ * answers it with an empty cell for half the fleet.
+ *
+ * So the table itself is mocked to capture its props, and the captured
+ * getElement closure is then rendered directly. That also keeps this test off
+ * the network: the real AnalyticsModelTable fetches on mount.
+ */
+
+type CapturedColumn = {
+  field: Record<string, boolean>;
+  title: string;
+  isHiddenByDefault?: boolean | undefined;
+  getElement?: ((item: Span) => React.ReactElement) | undefined;
+  getExportValue?: ((item: Span) => string) | undefined;
+};
+
+type CapturedFilter = {
+  field: Record<string, boolean>;
+  title: string;
+  type: unknown;
+};
+
+type CapturedTableProps = {
+  columns?: Array<CapturedColumn>;
+  filters?: Array<CapturedFilter>;
+  selectMoreFields?: Record<string, boolean>;
+};
+
+let capturedTableProps: CapturedTableProps | null = null;
+
+jest.mock("../../../UI/Components/ModelTable/AnalyticsModelTable", () => {
+  return {
+    __esModule: true,
+    default: (props: CapturedTableProps) => {
+      capturedTableProps = props;
+      return null;
+    },
+  };
+});
+
+/*
+ * The table loads the project's telemetry services on mount purely to colour
+ * the Service cell. Stubbed to an immediately-resolving empty list so no
+ * request escapes and no act() warning follows the resolve.
+ */
+jest.mock("../../../UI/Utils/ModelAPI/ModelAPI", () => {
+  return {
+    __esModule: true,
+    default: {
+      getList: () => {
+        return Promise.resolve({ data: [], count: 0, skip: 0, limit: 0 });
+      },
+    },
+  };
+});
+
+import LlmCallsTable from "../../../../App/FeatureSet/Dashboard/src/Components/AI/LlmCallsTable";
+import Span from "../../../Models/AnalyticsModels/Span";
+import FieldType from "../../../UI/Components/Types/FieldType";
+import ObjectID from "../../../Types/ObjectID";
+import ProjectUtil from "../../../UI/Utils/Project";
+
+const PROJECT_ID: ObjectID = new ObjectID(
+  "11111111-1111-4111-8111-111111111111",
+);
+
+type ColumnByTitleFunction = (title: string) => CapturedColumn;
+
+const columnByTitle: ColumnByTitleFunction = (
+  title: string,
+): CapturedColumn => {
+  const column: CapturedColumn | undefined = capturedTableProps?.columns?.find(
+    (candidate: CapturedColumn): boolean => {
+      return candidate.title === title;
+    },
+  );
+
+  expect(column).toBeDefined();
+
+  return column!;
+};
+
+type FilterByTitleFunction = (title: string) => CapturedFilter;
+
+const filterByTitle: FilterByTitleFunction = (
+  title: string,
+): CapturedFilter => {
+  const filter: CapturedFilter | undefined = capturedTableProps?.filters?.find(
+    (candidate: CapturedFilter): boolean => {
+      return candidate.title === title;
+    },
+  );
+
+  expect(filter).toBeDefined();
+
+  return filter!;
+};
+
+type MakeSpanFunction = (data: {
+  llmUserEmail?: string | undefined;
+  llmUserId?: string | undefined;
+  llmTeam?: string | undefined;
+}) => Span;
+
+const makeSpan: MakeSpanFunction = (data: {
+  llmUserEmail?: string | undefined;
+  llmUserId?: string | undefined;
+  llmTeam?: string | undefined;
+}): Span => {
+  const span: Span = new Span();
+
+  span.llmUserEmail = data.llmUserEmail;
+  span.llmUserId = data.llmUserId;
+  span.llmTeam = data.llmTeam;
+
+  return span;
+};
+
+type RenderTableFunction = () => Promise<void>;
+
+/*
+ * Awaited inside act(): the component loads telemetry services on mount and
+ * setStates when that promise resolves, so an un-awaited render leaves a
+ * state update escaping the test.
+ */
+const renderTable: RenderTableFunction = async (): Promise<void> => {
+  await act(async (): Promise<void> => {
+    render(
+      <MemoryRouter>
+        <LlmCallsTable />
+      </MemoryRouter>,
+    );
+  });
+
+  expect(capturedTableProps).not.toBeNull();
+};
+
+describe("LlmCallsTable — employee columns", () => {
+  beforeEach(() => {
+    capturedTableProps = null;
+    jest.spyOn(ProjectUtil, "getCurrentProjectId").mockReturnValue(PROJECT_ID);
+  });
+
+  afterEach(() => {
+    cleanup();
+    jest.restoreAllMocks();
+  });
+
+  test("User and Team are shown by default", async () => {
+    await renderTable();
+
+    expect(columnByTitle("User").isHiddenByDefault).toBeFalsy();
+    expect(columnByTitle("Team").isHiddenByDefault).toBeFalsy();
+    expect(columnByTitle("Team").field).toEqual({ llmTeam: true });
+  });
+
+  test("the two identity columns did not widen the default table", async () => {
+    /*
+     * Provider and Operation moved into the column picker to pay for User and
+     * Team. Pinned because it is a deliberate trade rather than an accident:
+     * un-hiding them without hiding something else silently pushes cost and
+     * status off the side of a laptop screen.
+     */
+    await renderTable();
+
+    expect(columnByTitle("Provider").isHiddenByDefault).toBe(true);
+    expect(columnByTitle("Operation").isHiddenByDefault).toBe(true);
+
+    const visibleTitles: Array<string> = (capturedTableProps?.columns || [])
+      .filter((column: CapturedColumn): boolean => {
+        return !column.isHiddenByDefault;
+      })
+      .map((column: CapturedColumn): string => {
+        return column.title;
+      });
+
+    expect(visibleTitles).toEqual([
+      "Seen At",
+      "Service",
+      "Model",
+      "User",
+      "Team",
+      "Tokens (in / out)",
+      "Cost",
+      "Status",
+    ]);
+  });
+
+  test("the User cell prefers the email", async () => {
+    await renderTable();
+
+    const column: CapturedColumn = columnByTitle("User");
+
+    render(
+      column.getElement!(
+        makeSpan({ llmUserEmail: "ada@example.com", llmUserId: "acct-9f2" }),
+      ),
+    );
+
+    expect(screen.getByText("ada@example.com")).toBeInTheDocument();
+    expect(screen.queryByText("acct-9f2")).not.toBeInTheDocument();
+  });
+
+  test("the User cell falls back to the id when no email was reported", async () => {
+    /*
+     * The gateway population: LiteLLM stamps a key-owner id and no email. If
+     * the fallback were dropped, this whole class of emitter would render an
+     * empty User column while the table still filtered on it.
+     */
+    await renderTable();
+
+    render(
+      columnByTitle("User").getElement!(makeSpan({ llmUserId: "acct-9f2" })),
+    );
+
+    expect(screen.getByText("acct-9f2")).toBeInTheDocument();
+  });
+
+  test("the User cell reads as absent, not as a nameless person", async () => {
+    await renderTable();
+
+    // Whitespace is what an unset environment variable produces at the emitter.
+    render(
+      columnByTitle("User").getElement!(makeSpan({ llmUserEmail: "   " })),
+    );
+
+    expect(screen.getByText("—")).toBeInTheDocument();
+  });
+
+  test("the CSV export carries the same fallback the cell renders", async () => {
+    /*
+     * An Element column exports its declared field's raw value unless it says
+     * otherwise, so without getExportValue the export would drop exactly the
+     * rows whose identity came from the id.
+     */
+    await renderTable();
+
+    const column: CapturedColumn = columnByTitle("User");
+
+    expect(column.getExportValue!(makeSpan({ llmUserId: "acct-9f2" }))).toBe(
+      "acct-9f2",
+    );
+    expect(
+      column.getExportValue!(makeSpan({ llmUserEmail: "ada@example.com" })),
+    ).toBe("ada@example.com");
+    expect(column.getExportValue!(makeSpan({}))).toBe("");
+  });
+
+  test("llmUserId is selected even though no column declares it", async () => {
+    /*
+     * The User column declares llmUserEmail; the fallback reads a field
+     * nothing else asks for, so it has to be requested explicitly.
+     */
+    await renderTable();
+
+    expect(capturedTableProps?.selectMoreFields?.["llmUserId"]).toBe(true);
+  });
+});
+
+describe("LlmCallsTable — employee filters", () => {
+  beforeEach(() => {
+    capturedTableProps = null;
+    jest.spyOn(ProjectUtil, "getCurrentProjectId").mockReturnValue(PROJECT_ID);
+  });
+
+  afterEach(() => {
+    cleanup();
+    jest.restoreAllMocks();
+  });
+
+  test("email, id and team are free-text filters", async () => {
+    /*
+     * Text rather than a dropdown: the set of people who have made an LLM
+     * call is not a bounded list the page can fetch upfront, unlike services.
+     */
+    await renderTable();
+
+    expect(filterByTitle("User Email").field).toEqual({ llmUserEmail: true });
+    expect(filterByTitle("User Email").type).toBe(FieldType.Text);
+
+    expect(filterByTitle("User ID").field).toEqual({ llmUserId: true });
+    expect(filterByTitle("User ID").type).toBe(FieldType.Text);
+
+    expect(filterByTitle("Team").field).toEqual({ llmTeam: true });
+    expect(filterByTitle("Team").type).toBe(FieldType.Text);
+  });
+
+  test("email and id are separate filters, not one merged input", async () => {
+    /*
+     * A filter narrows ONE stored column and most emitters populate exactly
+     * one of the two, so a single merged "User" input would silently return
+     * nothing for whichever half of the fleet reports the other spelling.
+     */
+    await renderTable();
+
+    expect(filterByTitle("User Email").field).not.toEqual(
+      filterByTitle("User ID").field,
+    );
+  });
+});

package/Tests/App/Dashboard/LlmOverview.test.tsx

@@ -0,0 +1,335 @@
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  jest,
+  test,
+} from "@jest/globals";
+import "@testing-library/jest-dom";
+import { act, cleanup, render, screen } from "@testing-library/react";
+import * as React from "react";
+import { MemoryRouter } from "react-router-dom";
+import getJestMockFunction, { MockFunction } from "../../MockType";
+
+/*
+ * ---------------------------------------------------------------------------
+ * The AI / LLM Overview cost KPI
+ * ---------------------------------------------------------------------------
+ *
+ * The cost tile is the headline number on this page, and it has to reconcile
+ * two facts about the metric stream that nothing in a snapshot would reveal:
+ *
+ *  - there are TWO cost streams, in two units. The vendor counters report
+ *    USD; the OpenAI Codex CLI reports MILLIONTHS of a USD. They can never
+ *    share a Sum — the unit is unrecoverable after the addition, and a $3
+ *    turn folded into the USD list would surface as $3,000,000 — so the tile
+ *    queries both lists and folds them through combineCostTotals, which
+ *    applies the scale per list. Querying only the USD list (which this tile
+ *    used to do) makes a Codex-only project read $0 on the Overview while the
+ *    Usage tab and its cost budgets, which already query both, show the real
+ *    figure.
+ *  - spans are authoritative, and the two signals are NEVER summed. An
+ *    emitter producing both would otherwise have every dollar counted twice.
+ */
+
+const aggregateMock: MockFunction = getJestMockFunction();
+const countMock: MockFunction = getJestMockFunction();
+const getCurrentProjectIdMock: MockFunction = getJestMockFunction();
+
+jest.mock("../../../UI/Utils/AnalyticsModelAPI/AnalyticsModelAPI", () => {
+  return {
+    __esModule: true,
+    default: {
+      aggregate: (...args: Array<unknown>) => {
+        return aggregateMock(...args);
+      },
+      count: (...args: Array<unknown>) => {
+        return countMock(...args);
+      },
+    },
+  };
+});
+
+jest.mock("../../../UI/Utils/Project", () => {
+  return {
+    __esModule: true,
+    default: {
+      getCurrentProjectId: (...args: Array<unknown>) => {
+        return getCurrentProjectIdMock(...args);
+      },
+    },
+  };
+});
+
+/*
+ * The recent-calls table below the tiles is a real AnalyticsModelTable that
+ * fetches on mount. Stubbed out so this test stays on the KPI row and no
+ * request escapes.
+ */
+jest.mock(
+  "../../../../App/FeatureSet/Dashboard/src/Components/AI/LlmCallsTable",
+  () => {
+    return {
+      __esModule: true,
+      default: () => {
+        return null;
+      },
+    };
+  },
+);
+
+import LlmOverview from "../../../../App/FeatureSet/Dashboard/src/Components/AI/LlmOverview";
+import AggregatedModel from "../../../Types/BaseDatabase/AggregatedModel";
+import AggregatedResult from "../../../Types/BaseDatabase/AggregatedResult";
+import Includes from "../../../Types/BaseDatabase/Includes";
+import ObjectID from "../../../Types/ObjectID";
+import { JSONObject } from "../../../Types/JSON";
+import {
+  LlmCostMetricNames,
+  LlmMicroUsdCostMetricNames,
+} from "../../../Types/Telemetry/LlmMetricConventions";
+
+const PROJECT_ID: ObjectID = new ObjectID(
+  "11111111-1111-4111-8111-111111111111",
+);
+
+interface AggregateCall {
+  modelType: { new (): unknown };
+  aggregateBy: JSONObject;
+}
+
+type ModelNameFunction = (call: AggregateCall) => string;
+
+const modelNameOf: ModelNameFunction = (call: AggregateCall): string => {
+  return (call.modelType as unknown as { name: string }).name;
+};
+
+type ColumnNameFunction = (call: AggregateCall) => string;
+
+const columnOf: ColumnNameFunction = (call: AggregateCall): string => {
+  return String(call.aggregateBy["aggregateColumnName"]);
+};
+
+type MetricNamesFunction = (call: AggregateCall) => Array<string>;
+
+const metricNamesOf: MetricNamesFunction = (
+  call: AggregateCall,
+): Array<string> => {
+  const query: JSONObject = call.aggregateBy["query"] as JSONObject;
+  const names: Includes = query["name"] as unknown as Includes;
+
+  return (names?.values as Array<string>) || [];
+};
+
+type MatchesNameListFunction = (
+  call: AggregateCall,
+  list: Array<string>,
+) => boolean;
+
+const matchesNameList: MatchesNameListFunction = (
+  call: AggregateCall,
+  list: Array<string>,
+): boolean => {
+  return metricNamesOf(call).includes(list[0]!);
+};
+
+type RowFunction = (value: number) => AggregatedModel;
+
+const row: RowFunction = (value: number): AggregatedModel => {
+  return {
+    timestamp: new Date("2026-08-20T00:00:00.000Z"),
+    value: value,
+  };
+};
+
+type ResultFunction = (rows: Array<AggregatedModel>) => AggregatedResult;
+
+const result: ResultFunction = (
+  rows: Array<AggregatedModel>,
+): AggregatedResult => {
+  return { data: rows };
+};
+
+type RenderFunction = () => Promise<void>;
+
+const renderOverview: RenderFunction = async (): Promise<void> => {
+  await act(async () => {
+    render(
+      <MemoryRouter>
+        <LlmOverview />
+      </MemoryRouter>,
+    );
+  });
+};
+
+/*
+ * Span figures that keep every tile OTHER than cost on the span stream, so a
+ * "from GenAI metrics" hint on screen can only have come from the cost tile.
+ * Cost itself is zero — the precondition for the metric fallback.
+ */
+type SpanSumFunction = (call: AggregateCall) => AggregatedResult | null;
+
+const spanSumsWithZeroCost: SpanSumFunction = (
+  call: AggregateCall,
+): AggregatedResult | null => {
+  if (modelNameOf(call) !== "Span") {
+    return null;
+  }
+
+  if (columnOf(call) === "llmInputTokens") {
+    return result([row(120)]);
+  }
+
+  if (columnOf(call) === "llmOutputTokens") {
+    return result([row(30)]);
+  }
+
+  // llmCost — the span stream reports no spend at all.
+  return result([]);
+};
+
+beforeEach(() => {
+  aggregateMock.mockReset();
+  countMock.mockReset();
+  getCurrentProjectIdMock.mockReset();
+
+  getCurrentProjectIdMock.mockReturnValue(PROJECT_ID);
+  countMock.mockResolvedValue(7 as never);
+});
+
+afterEach(() => {
+  cleanup();
+});
+
+describe("LlmOverview - the metric-sourced cost tile", () => {
+  test("folds micro-USD cost metrics, so a Codex-only project does not read $0", async () => {
+    aggregateMock.mockImplementation((call: unknown) => {
+      const aggregateCall: AggregateCall = call as AggregateCall;
+
+      const spanResult: AggregatedResult | null =
+        spanSumsWithZeroCost(aggregateCall);
+
+      if (spanResult) {
+        return Promise.resolve(spanResult);
+      }
+
+      if (matchesNameList(aggregateCall, LlmMicroUsdCostMetricNames)) {
+        // 1,500,000 millionths of a dollar is $1.50, not $1,500,000.
+        return Promise.resolve(result([row(1500000)]));
+      }
+
+      // The USD cost list and the token list have nothing for this project.
+      return Promise.resolve(result([]));
+    });
+
+    await renderOverview();
+
+    expect(screen.getByText("$1.5000")).toBeInTheDocument();
+    expect(screen.queryByText("$0.0000")).not.toBeInTheDocument();
+    expect(screen.getAllByText("from GenAI metrics").length).toBeGreaterThan(0);
+  });
+
+  test("adds the USD and micro-USD streams after scaling each, never before", async () => {
+    aggregateMock.mockImplementation((call: unknown) => {
+      const aggregateCall: AggregateCall = call as AggregateCall;
+
+      const spanResult: AggregatedResult | null =
+        spanSumsWithZeroCost(aggregateCall);
+
+      if (spanResult) {
+        return Promise.resolve(spanResult);
+      }
+
+      if (matchesNameList(aggregateCall, LlmMicroUsdCostMetricNames)) {
+        return Promise.resolve(result([row(1500000)]));
+      }
+
+      if (matchesNameList(aggregateCall, LlmCostMetricNames)) {
+        return Promise.resolve(result([row(2)]));
+      }
+
+      return Promise.resolve(result([]));
+    });
+
+    await renderOverview();
+
+    /*
+     * $2.00 + $1.50. Summing the raw values first would read as $1,500,002 —
+     * which is exactly what a shared Sum over both name lists would produce.
+     */
+    expect(screen.getByText("$3.5000")).toBeInTheDocument();
+  });
+
+  test("queries BOTH cost name lists, not just the USD one", async () => {
+    aggregateMock.mockImplementation((call: unknown) => {
+      const aggregateCall: AggregateCall = call as AggregateCall;
+
+      const spanResult: AggregatedResult | null =
+        spanSumsWithZeroCost(aggregateCall);
+
+      return Promise.resolve(spanResult || result([]));
+    });
+
+    await renderOverview();
+
+    const metricCostCalls: Array<AggregateCall> = aggregateMock.mock.calls
+      .map((args: Array<unknown>): AggregateCall => {
+        return args[0] as AggregateCall;
+      })
+      .filter((candidate: AggregateCall): boolean => {
+        return modelNameOf(candidate) === "Metric";
+      });
+
+    expect(
+      metricCostCalls.some((candidate: AggregateCall): boolean => {
+        return matchesNameList(candidate, LlmCostMetricNames);
+      }),
+    ).toBe(true);
+
+    expect(
+      metricCostCalls.some((candidate: AggregateCall): boolean => {
+        return matchesNameList(candidate, LlmMicroUsdCostMetricNames);
+      }),
+    ).toBe(true);
+  });
+
+  test("never sums spans and metrics: a project with span cost reads span cost only", async () => {
+    aggregateMock.mockImplementation((call: unknown) => {
+      const aggregateCall: AggregateCall = call as AggregateCall;
+
+      if (modelNameOf(aggregateCall) === "Metric") {
+        /*
+         * $9 of Codex spend sitting in the metric stream. Summed with the
+         * $4 of span cost it would surface as $13 — every dollar twice.
+         */
+        return Promise.resolve(result([row(9000000)]));
+      }
+
+      if (columnOf(aggregateCall) === "llmCost") {
+        return Promise.resolve(result([row(4)]));
+      }
+
+      if (columnOf(aggregateCall) === "llmInputTokens") {
+        return Promise.resolve(result([row(120)]));
+      }
+
+      return Promise.resolve(result([row(30)]));
+    });
+
+    await renderOverview();
+
+    expect(screen.getByText("$4.0000")).toBeInTheDocument();
+    expect(screen.queryByText("$13.0000")).not.toBeInTheDocument();
+
+    // The metric stream is not even consulted while spans have something.
+    const metricCalls: Array<unknown> = aggregateMock.mock.calls.filter(
+      (args: Array<unknown>): boolean => {
+        return modelNameOf(args[0] as AggregateCall) === "Metric";
+      },
+    );
+
+    expect(metricCalls).toHaveLength(0);
+    expect(screen.queryAllByText("from GenAI metrics")).toHaveLength(0);
+  });
+});