@oneuptime/common 12.0.24 → 12.0.25

package/Tests/Utils/Telemetry/LlmMetricQuery.test.ts

@@ -6,6 +6,7 @@ import Query from "../../../Types/BaseDatabase/Query";
 import ObjectID from "../../../Types/ObjectID";
 import {
   LlmCostMetricNames,
+  LlmMicroUsdCostMetricNames,
   LlmTokenUsageMetricNames,
 } from "../../../Types/Telemetry/LlmMetricConventions";
 import LlmMetricQuery, {
@@ -13,6 +14,8 @@ import LlmMetricQuery, {
   LlmMetricTokenTotals,
   METRIC_MODEL_ATTRIBUTE_KEY,
   METRIC_SYSTEM_ATTRIBUTE_KEY,
+  METRIC_TEAM_ATTRIBUTE_KEY,
+  METRIC_USER_ATTRIBUTE_KEY,
 } from "../../../Utils/Telemetry/LlmMetricQuery";
 import { describe, expect, test } from "@jest/globals";
 
@@ -470,3 +473,298 @@ describe("LlmMetricQuery", () => {
     });
   });
 });
+
+/*
+ * Employee / team scoping of metric-sourced spend.
+ *
+ * The bug class: a scoping dimension that silently does nothing. If a filter
+ * is dropped, mis-keyed, or written to the wrong attribute name, the query
+ * quietly returns the WHOLE project's spend under one person's name — a
+ * number that renders perfectly and attributes an org-wide bill to one
+ * engineer. Nothing about that failure looks like a failure.
+ */
+describe("LlmMetricQuery — employee and team scoping", () => {
+  test("narrows by employee email using the primary identity key", () => {
+    const query: Record<string, unknown> = LlmMetricQuery.buildBaseQuery({
+      scope: scope({ llmUserEmail: "ada@example.com" }),
+      metricNames: ["a"],
+    }) as unknown as Record<string, unknown>;
+
+    expect(query["attributes"]).toEqual({
+      [METRIC_USER_ATTRIBUTE_KEY]: "ada@example.com",
+    });
+    expect(METRIC_USER_ATTRIBUTE_KEY).toBe("user.email");
+  });
+
+  test("narrows by employee id on the same primary identity key", () => {
+    /*
+     * The metric stream has ONE identity attribute, unlike the span side
+     * where id and email are separate columns — so an id-scoped query filters
+     * the same key.
+     */
+    const query: Record<string, unknown> = LlmMetricQuery.buildBaseQuery({
+      scope: scope({ llmUserId: "acct-9f2" }),
+      metricNames: ["a"],
+    }) as unknown as Record<string, unknown>;
+
+    expect(query["attributes"]).toEqual({
+      [METRIC_USER_ATTRIBUTE_KEY]: "acct-9f2",
+    });
+  });
+
+  test("email wins over id when a caller supplies both", () => {
+    /*
+     * Both map to one key, so one has to win deterministically rather than by
+     * object-literal order. Email is the preferred spelling of the list.
+     */
+    const query: Record<string, unknown> = LlmMetricQuery.buildBaseQuery({
+      scope: scope({ llmUserEmail: "ada@example.com", llmUserId: "acct-9f2" }),
+      metricNames: ["a"],
+    }) as unknown as Record<string, unknown>;
+
+    expect(query["attributes"]).toEqual({
+      [METRIC_USER_ATTRIBUTE_KEY]: "ada@example.com",
+    });
+  });
+
+  test("narrows by team using the primary team key", () => {
+    const query: Record<string, unknown> = LlmMetricQuery.buildBaseQuery({
+      scope: scope({ llmTeam: "platform" }),
+      metricNames: ["a"],
+    }) as unknown as Record<string, unknown>;
+
+    expect(query["attributes"]).toEqual({
+      [METRIC_TEAM_ATTRIBUTE_KEY]: "platform",
+    });
+    expect(METRIC_TEAM_ATTRIBUTE_KEY).toBe("team.id");
+  });
+
+  test("identity filters compose with provider, model and service", () => {
+    const query: Record<string, unknown> = LlmMetricQuery.buildBaseQuery({
+      scope: scope({
+        serviceId: SERVICE_ID,
+        llmSystem: "anthropic",
+        llmModel: "claude-opus-4-8",
+        llmUserEmail: "ada@example.com",
+        llmTeam: "platform",
+      }),
+      metricNames: ["a"],
+    }) as unknown as Record<string, unknown>;
+
+    expect(query["primaryEntityId"]).toBe(SERVICE_ID);
+    expect(query["attributes"]).toEqual({
+      [METRIC_SYSTEM_ATTRIBUTE_KEY]: "anthropic",
+      [METRIC_MODEL_ATTRIBUTE_KEY]: "claude-opus-4-8",
+      [METRIC_USER_ATTRIBUTE_KEY]: "ada@example.com",
+      [METRIC_TEAM_ATTRIBUTE_KEY]: "platform",
+    });
+  });
+
+  test("the user and team filters never collide on one key", () => {
+    // A shared key would make a team filter silently overwrite the person.
+    expect(METRIC_USER_ATTRIBUTE_KEY).not.toBe(METRIC_TEAM_ATTRIBUTE_KEY);
+  });
+
+  test("empty-string identity values leave the query unscoped", () => {
+    /*
+     * The dangerous direction is the opposite of under-counting: an empty
+     * filter that still emits an attribute predicate would match NOTHING and
+     * report $0 for a person who spent thousands.
+     */
+    const query: Record<string, unknown> = LlmMetricQuery.buildBaseQuery({
+      scope: scope({ llmUserEmail: "", llmUserId: "", llmTeam: "" }),
+      metricNames: ["a"],
+    }) as unknown as Record<string, unknown>;
+
+    expect(query).not.toHaveProperty("attributes");
+  });
+
+  test("every cost and token query carries the identity scoping", () => {
+    const scoped: LlmMetricScope = scope({
+      llmUserEmail: "ada@example.com",
+      llmTeam: "platform",
+    });
+
+    for (const query of [
+      LlmMetricQuery.buildCostQuery(scoped),
+      LlmMetricQuery.buildMicroUsdCostQuery(scoped),
+      LlmMetricQuery.buildTokenQuery(scoped),
+    ] as Array<Query<Metric>>) {
+      const record: Record<string, unknown> = query as unknown as Record<
+        string,
+        unknown
+      >;
+
+      expect(record["attributes"]).toEqual({
+        [METRIC_USER_ATTRIBUTE_KEY]: "ada@example.com",
+        [METRIC_TEAM_ATTRIBUTE_KEY]: "platform",
+      });
+    }
+  });
+});
+
+/*
+ * Micro-USD cost, and the single place its scale factor is applied.
+ *
+ * The bug class this block exists for is a MILLION-FOLD unit error. The
+ * OpenAI Codex CLI reports spend in millionths of a dollar. If its metric
+ * name reaches the USD query, or the 1e-6 factor is dropped or typo'd, a $3
+ * turn lands in a cost budget as $3,000,000 and instantly breaches every
+ * threshold a customer has configured. Nothing throws; the alerts just fire.
+ */
+describe("LlmMetricQuery — micro-USD cost", () => {
+  test("the micro-USD query selects exactly the micro-USD metric names", () => {
+    const query: Record<string, unknown> =
+      LlmMetricQuery.buildMicroUsdCostQuery(scope()) as unknown as Record<
+        string,
+        unknown
+      >;
+
+    expect((query["name"] as Includes).values).toEqual(
+      LlmMicroUsdCostMetricNames,
+    );
+  });
+
+  test("the USD and micro-USD queries never select the same metric name", () => {
+    const usd: Includes = (
+      LlmMetricQuery.buildCostQuery(scope()) as unknown as Record<
+        string,
+        unknown
+      >
+    )["name"] as Includes;
+
+    const micro: Includes = (
+      LlmMetricQuery.buildMicroUsdCostQuery(scope()) as unknown as Record<
+        string,
+        unknown
+      >
+    )["name"] as Includes;
+
+    const overlap: Array<string> = (usd.values as Array<string>).filter(
+      (name: string) => {
+        return (micro.values as Array<string>).includes(name);
+      },
+    );
+
+    expect(overlap).toEqual([]);
+  });
+
+  test("the micro-USD query bounds rows with the same time predicate", () => {
+    const query: Record<string, unknown> =
+      LlmMetricQuery.buildMicroUsdCostQuery(scope()) as unknown as Record<
+        string,
+        unknown
+      >;
+
+    const time: InBetween<Date> = query["time"] as InBetween<Date>;
+
+    expect(time).toBeInstanceOf(InBetween);
+    expect(time.startValue).toBe(START);
+    expect(time.endValue).toBe(END);
+  });
+
+  describe("combineCostTotals", () => {
+    test("a 1,500,000 micro-USD datapoint is $1.50", () => {
+      // The canonical Codex figure. This is the whole point of the function.
+      expect(
+        LlmMetricQuery.combineCostTotals({ usd: 0, microUsd: 1_500_000 }),
+      ).toBeCloseTo(1.5, 10);
+    });
+
+    test("one million micro-USD is exactly one dollar", () => {
+      expect(
+        LlmMetricQuery.combineCostTotals({ usd: 0, microUsd: 1_000_000 }),
+      ).toBe(1);
+    });
+
+    test("adds a USD total and a micro-USD total in the same unit", () => {
+      expect(
+        LlmMetricQuery.combineCostTotals({ usd: 4.25, microUsd: 1_500_000 }),
+      ).toBeCloseTo(5.75, 10);
+    });
+
+    test("a micro-USD total is NOT added raw", () => {
+      /*
+       * The regression guard, stated as the failure it prevents: without the
+       * scale factor this returns 3,000,000 instead of 3.
+       */
+      const combined: number = LlmMetricQuery.combineCostTotals({
+        usd: 0,
+        microUsd: 3_000_000,
+      });
+
+      expect(combined).toBeCloseTo(3, 10);
+      expect(combined).toBeLessThan(10);
+    });
+
+    test("sub-cent micro-USD figures survive as fractions", () => {
+      // A single cheap turn: 1250 micro-USD = $0.00125.
+      expect(
+        LlmMetricQuery.combineCostTotals({ usd: 0, microUsd: 1250 }),
+      ).toBeCloseTo(0.00125, 10);
+    });
+
+    test("returns 0 when both totals are 0", () => {
+      expect(LlmMetricQuery.combineCostTotals({ usd: 0, microUsd: 0 })).toBe(0);
+    });
+
+    test("preserves a negative micro-USD total, which a delta counter can emit", () => {
+      expect(
+        LlmMetricQuery.combineCostTotals({ usd: 2, microUsd: -1_000_000 }),
+      ).toBeCloseTo(1, 10);
+    });
+
+    /*
+     * Non-finite inputs contribute 0 rather than poisoning the result, exactly
+     * as sumAggregatedRows does. A NaN spend compares false against every
+     * threshold, which silently DISABLES a budget monitor instead of tripping
+     * it — the failure mode a cost alert must never have.
+     */
+    test.each([
+      ["NaN usd", NaN, 1_000_000, 1],
+      ["NaN microUsd", 2, NaN, 2],
+      ["Infinity usd", Infinity, 1_000_000, 1],
+      ["-Infinity usd", -Infinity, 2_000_000, 2],
+      ["Infinity microUsd", 3, Infinity, 3],
+      ["-Infinity microUsd", 3, -Infinity, 3],
+      ["both non-finite", NaN, Infinity, 0],
+    ])(
+      "%s contributes 0 instead of poisoning the total",
+      (
+        _label: string,
+        usd: number,
+        microUsd: number,
+        expected: number,
+      ): void => {
+        const combined: number = LlmMetricQuery.combineCostTotals({
+          usd: usd,
+          microUsd: microUsd,
+        });
+
+        expect(isFinite(combined)).toBe(true);
+        expect(combined).toBeCloseTo(expected, 10);
+      },
+    );
+
+    test("composes with sumAggregatedRows end to end", () => {
+      /*
+       * The realistic shape: a Claude Code cost metric in dollars and three
+       * Codex turns in micro-USD, folded the way getCostInUSD folds them.
+       */
+      const usd: number = LlmMetricQuery.sumAggregatedRows([
+        row(0.5),
+        row(0.25),
+      ] as Array<AggregatedModel>);
+
+      const microUsd: number = LlmMetricQuery.sumAggregatedRows([
+        row(1_500_000),
+        row(500_000),
+        row("not-a-number"),
+      ] as Array<AggregatedModel>);
+
+      expect(
+        LlmMetricQuery.combineCostTotals({ usd: usd, microUsd: microUsd }),
+      ).toBeCloseTo(2.75, 10);
+    });
+  });
+});

package/Types/Telemetry/LlmConventions.ts

@@ -16,6 +16,65 @@
  * lookup helpers return the first key that is present.
  */
 
+/**
+ * The prefix OTLP ingest stamps onto every RESOURCE attribute before the
+ * attributes map reaches any extractor.
+ *
+ * This is not cosmetic and it is the difference between a feature that works
+ * and one that silently reports nothing. All three ingest services flatten
+ * resource attributes through
+ * `TelemetryUtil.getAttributes({ items, prefixKeysWithString: "resource" })`
+ * — OtelTracesIngestService, OtelMetricsIngestService and
+ * OtelLogsIngestService alike. So an operator who sets
+ * `OTEL_RESOURCE_ATTRIBUTES=team.id=platform` does NOT produce an attribute
+ * called `team.id`; they produce one called `resource.team.id`.
+ *
+ * OtelTracesIngestService already learned this the hard way for session
+ * replay (see its `sessionIdAttributeKeys`, which lists both spellings and
+ * whose comment notes that reading only the bare key "would look correct in a
+ * unit test and fail in production"). Identity is the same trap, one tier
+ * worse: identity is precisely the kind of value a fleet stamps ONCE on the
+ * resource rather than on every span, and Cursor documents `cursor.user.id` /
+ * `cursor.team.id` as resource attributes outright.
+ */
+export const RESOURCE_ATTRIBUTE_KEY_PREFIX: string = "resource.";
+
+/**
+ * Widen a key list so it matches both the span-attribute spelling and the
+ * resource-attribute spelling of every key in it.
+ *
+ * The bare block comes FIRST in its entirety, then the resource-prefixed
+ * block, because a span attribute is strictly more specific than a resource
+ * attribute: the resource describes the whole process, the span describes one
+ * call. Preserving each block's internal order keeps the documented
+ * preference ordering intact within each tier.
+ *
+ * Pure and exported so the widening is one tested function rather than a
+ * hand-maintained second copy of every list that drifts the first time
+ * somebody adds a key.
+ */
+export function withResourcePrefixedKeys(keys: Array<string>): Array<string> {
+  return [
+    ...keys,
+    ...keys.map((key: string) => {
+      return `${RESOURCE_ATTRIBUTE_KEY_PREFIX}${key}`;
+    }),
+  ];
+}
+
+/*
+ * Note what is deliberately NOT widened: every list below that carries a
+ * per-CALL value — system, operation, models, tokens, cost, conversation id,
+ * tool name, request parameters, prompts and completions. Those describe one
+ * LLM request and have no business being set on a resource, which describes
+ * the whole process; a resource-scoped `gen_ai.request.model` would be a
+ * misconfiguration, not a convention to support. Widening them would double
+ * the number of map lookups on the hottest ingest path to chase a spelling
+ * nobody emits. Only the IDENTITY lists — user id, user email, team, and the
+ * end-user EXCLUSION list — are widened, because identity is exactly the
+ * value an operator sets once via OTEL_RESOURCE_ATTRIBUTES.
+ */
+
 // Provider / system, e.g. "openai", "anthropic", "aws.bedrock".
 export const LlmSystemAttributeKeys: Array<string> = [
   "gen_ai.system",
@@ -96,6 +155,202 @@ export const LlmAgentNameAttributeKeys: Array<string> = [
   "agent.name",
 ];
 
+/*
+ * ---------------------------------------------------------------------------
+ * WHO ran this call — the EMPLOYEE, never the employee's own customer.
+ * ---------------------------------------------------------------------------
+ *
+ * The LLM feature was born trace-only and had no concept of a human being, so
+ * "which of our engineers burned $4k on Opus last month" was unanswerable. The
+ * three lists below make the human actor a first-class span dimension, so
+ * spend can be grouped by person and by cost centre.
+ *
+ * THE CORRECTNESS RULE, and it is not a stylistic one: an LLM span can carry
+ * TWO different humans. The employee who made the call, and the DOWNSTREAM
+ * CUSTOMER on whose behalf the call was made (OpenAI's `user` request
+ * parameter, which instrumentations echo back as gen_ai.user / llm.user, and
+ * LiteLLM's end-user id). Those two are not interchangeable. Mapping a
+ * downstream customer into the employee columns produces internal chargeback
+ * that is not merely imprecise but WRONG — a support bot serving 40k
+ * customers would manufacture 40k phantom "employees", and the engineer who
+ * actually owns that spend would appear to have spent nothing.
+ *
+ * So the customer-identity keys are enumerated separately in
+ * LlmEndUserAttributeKeys below and are deliberately NOT read into any
+ * column. The exclusion is pinned by a test; if someone later wants
+ * downstream-customer analytics it must land in its OWN column, never by
+ * appending a key to the lists below.
+ */
+
+/*
+ * The employee / internal actor id.
+ *
+ * Provenance, preferred first:
+ *   - user.id      OTel general semantic conventions; THE canonical key for
+ *                  the human actor and the one to standardize on.
+ *   - enduser.id   Also still an ACTIVE semconv attribute — the 1.25
+ *                  deprecation removed enduser.role and enduser.scope only,
+ *                  so this remains a legitimate, widely-emitted spelling.
+ *   - litellm.metadata.user_api_key_user_id  /  metadata.user_api_key_user_id
+ *                  LiteLLM proxy: the internal user who OWNS the virtual key
+ *                  the request authenticated with. Note the sibling key
+ *                  user_api_key_end_user_id is the CUSTOMER and is excluded.
+ *
+ *                  BOTH spellings are required, and the bare one is the one
+ *                  that actually fires for most deployments. LiteLLM's
+ *                  default `otel` callback (v1) stamps its metadata with a
+ *                  bare "metadata." prefix — see METRIC_METADATA_KEYS and the
+ *                  `common_attrs[f"metadata.{key}"]` assignment in
+ *                  litellm/integrations/opentelemetry.py. The "litellm."
+ *                  namespace only appears under OpenTelemetry v2, which is
+ *                  OFF by default (LITELLM_OTEL_V2, `enabled: default=False`).
+ *                  Recognizing only the v2 spelling meant a stock LiteLLM
+ *                  proxy — the exact "one virtual key per employee" setup our
+ *                  docs recommend as the cleanest chargeback architecture —
+ *                  produced ZERO attribution, silently, with every gateway
+ *                  dollar landing in Unattributed and no error to explain it.
+ *   - traceloop.association.properties.user_id   OpenLLMetry.
+ *   - langfuse.user.id                           Langfuse.
+ *   - user.account_uuid / user.account_id        Claude Code (and Codex for
+ *                  the latter) stamp the signed-in account onto every span.
+ *   - cursor.user.id   Cursor's OTel export; an opaque team-scoped integer,
+ *                  which is why it sorts last — it needs the Cursor admin API
+ *                  to resolve to a person, unlike the keys above.
+ *
+ * The `Base` list is the SPAN-attribute spelling only. Consumers must import
+ * LlmUserIdAttributeKeys, which adds the resource-attribute tier — see
+ * withResourcePrefixedKeys. The base list is exported solely so tests and
+ * docs can cross-check the two tiers against each other.
+ */
+export const LlmUserIdBaseAttributeKeys: Array<string> = [
+  "user.id",
+  "enduser.id",
+  // LiteLLM OTel v2 spelling, then the v1 default. Both are needed — see above.
+  "litellm.metadata.user_api_key_user_id",
+  "metadata.user_api_key_user_id",
+  "traceloop.association.properties.user_id",
+  "langfuse.user.id",
+  "user.account_uuid",
+  "user.account_id",
+  "cursor.user.id",
+];
+
+export const LlmUserIdAttributeKeys: Array<string> = withResourcePrefixedKeys(
+  LlmUserIdBaseAttributeKeys,
+);
+
+/*
+ * The employee's email address — in practice the only identity value a
+ * manager can read without a lookup table, which is why it gets its own
+ * column rather than being folded into the id.
+ *
+ * user.email is the OTel general-semconv key and is emitted NATIVELY by
+ * Claude Code, Gemini CLI and OpenAI Codex, so for the coding-agent fleet
+ * this list is usually the one that hits.
+ *
+ * The two LiteLLM spellings matter more than their position suggests: the
+ * proxy carries the key owner's email in its metadata (user_api_key_user_email
+ * is in METRIC_METADATA_KEYS), so a gateway issuing one virtual key per
+ * employee yields a real, human-readable email with no lookup table and no
+ * per-application instrumentation. That is the cleanest chargeback setup
+ * available today, and it is what our AI-gateway guide recommends. As with
+ * the id list, the bare "metadata." spelling is the DEFAULT one and the
+ * "litellm." namespace only appears under the opt-in OpenTelemetry v2 mode.
+ *
+ * Because this column holds real PII, TraceScrubRuleService scrubs it under
+ * the Attributes scope exactly as it scrubs the attribute it was derived from
+ * — see the identity-column pass in scrubSpan. A denormalized column that
+ * skipped the scrub pass would make a customer's email-redaction rule
+ * silently ineffective on the very column most likely to hold an email.
+ */
+export const LlmUserEmailBaseAttributeKeys: Array<string> = [
+  "user.email",
+  // LiteLLM OTel v2 spelling, then the v1 default. Both are needed — see above.
+  "litellm.metadata.user_api_key_user_email",
+  "metadata.user_api_key_user_email",
+  "traceloop.association.properties.user_email",
+  "enduser.email",
+];
+
+export const LlmUserEmailAttributeKeys: Array<string> =
+  withResourcePrefixedKeys(LlmUserEmailBaseAttributeKeys);
+
+/*
+ * The team / cost centre the spend charges to.
+ *
+ * The first four are not emitted by any instrumentation on their own: they
+ * are what an organization sets via OTEL_RESOURCE_ATTRIBUTES on the agent
+ * process (OTEL_RESOURCE_ATTRIBUTES=team.id=platform,cost_center=RD-114), and
+ * are conventionally the ONLY way a coding-agent CLI learns which budget it
+ * belongs to. The remaining entries are the gateway/vendor spellings that
+ * arrive without any operator configuration.
+ *
+ * That first sentence is exactly why the resource tier is not optional here:
+ * OTEL_RESOURCE_ATTRIBUTES sets a RESOURCE attribute, which ingest delivers
+ * as `resource.team.id`. A list of bare keys alone would mean the one
+ * mechanism the docs present as THE way to attribute spend to a cost centre
+ * never matches a single row.
+ */
+export const LlmTeamBaseAttributeKeys: Array<string> = [
+  "team.id",
+  "team",
+  "cost_center",
+  "department",
+  // LiteLLM OTel v2 spellings, then the v1 default. See the id list above.
+  "litellm.metadata.user_api_key_team_id",
+  "litellm.team.id",
+  "metadata.user_api_key_team_id",
+  "cursor.team.id",
+];
+
+export const LlmTeamAttributeKeys: Array<string> = withResourcePrefixedKeys(
+  LlmTeamBaseAttributeKeys,
+);
+
+/*
+ * DELIBERATELY EXCLUDED — the DOWNSTREAM CUSTOMER, not the employee.
+ *
+ * Exported so the exclusion is a documented, testable decision rather than an
+ * omission somebody "fixes" later:
+ *
+ *   - gen_ai.user / llm.user
+ *       Both carry the OpenAI `user` REQUEST PARAMETER, which the API
+ *       documents as the caller's own end user, sent for abuse monitoring. On
+ *       a SaaS product's spans this is the SaaS product's customer.
+ *   - litellm.metadata.user_api_key_end_user_id  /
+ *     metadata.user_api_key_end_user_id  /  litellm.end_user.id
+ *       LiteLLM's explicit end-user id, distinct from the key-owner id above
+ *       (which IS the employee and IS recognized). All three spellings are
+ *       listed for the same reason the employee lists carry two: the bare
+ *       "metadata." form is what the DEFAULT v1 otel callback emits, while
+ *       "litellm.end_user.id" is the dedicated attribute the opt-in v2 mode
+ *       defines in litellm/integrations/otel/model/semconv.py. Excluding only
+ *       the v2 metadata spelling would leave the two spellings a real
+ *       deployment actually produces free to be mistaken for an employee.
+ *
+ * None of these is denormalized into a column by this change. They remain
+ * available in the raw attributes map for anyone who queries them directly.
+ *
+ * This list is resource-widened too, and for the OPPOSITE reason to the
+ * lists above. Here the widening does not make anything match — it makes the
+ * EXCLUSION hold. The exclusion is enforced as a set relation ("no end-user
+ * key appears in an employee list"), so if this list stopped at the bare
+ * spellings while the employee lists carried both tiers, a future
+ * `resource.gen_ai.user` could be appended to an employee list and every
+ * exclusion test would still pass. Both tiers here, both tiers there.
+ */
+export const LlmEndUserBaseAttributeKeys: Array<string> = [
+  "gen_ai.user",
+  "llm.user",
+  "litellm.metadata.user_api_key_end_user_id",
+  "metadata.user_api_key_end_user_id",
+  "litellm.end_user.id",
+];
+
+export const LlmEndUserAttributeKeys: Array<string> = withResourcePrefixedKeys(
+  LlmEndUserBaseAttributeKeys,
+);
+
 export const LlmToolNameAttributeKeys: Array<string> = [
   "gen_ai.tool.name",
   "tool.name",