@oneuptime/common 12.0.14 → 12.0.16

package/Tests/Server/Utils/Telemetry/LlmSpan.test.ts

@@ -3,6 +3,7 @@ import LlmSpanUtil, {
 } from "../../../../Server/Utils/Telemetry/LlmSpan";
 import { AttributeType } from "../../../../Server/Utils/Telemetry/Telemetry";
 import Dictionary from "../../../../Types/Dictionary";
+import { LlmModelPrice } from "../../../../Types/Telemetry/LlmCostCatalog";
 import { describe, expect, test } from "@jest/globals";
 
 type Attrs = Dictionary<AttributeType | Array<AttributeType>>;
@@ -162,3 +163,382 @@ describe("LlmSpanUtil.extract", () => {
     expect(fields.llmInputTokens).toBe(10);
   });
 });
+
+describe("LlmSpanUtil.extract — cost fallback from the pricing catalog", () => {
+  test("SDK-reported cost always wins over the catalog", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "gpt-4o",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 1_000_000,
+      // Catalog would say $12.50; the SDK (negotiated rate) says less.
+      "gen_ai.usage.cost": 9.99,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmCost).toBe(9.99);
+  });
+
+  test("an explicitly reported zero cost wins — no catalog estimate", () => {
+    // A gateway fronting a free/local model reports cost 0; that 0 is real.
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "gpt-4o",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 500_000,
+      "gen_ai.usage.cost": 0,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmCost).toBe(0);
+  });
+
+  test("a reported zero cost as a string also wins", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "meta",
+      "gen_ai.request.model": "llama-3.1-8b-instant",
+      "gen_ai.usage.input_tokens": 100_000,
+      "gen_ai.usage.cost": "0",
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmCost).toBe(0);
+  });
+
+  test("LiteLLM's v1 cost breakdown key is read as reported cost", () => {
+    /*
+     * LiteLLM's otel callback reports cost under gen_ai.cost.*, not
+     * gen_ai.usage.cost — its total must win over the catalog estimate.
+     */
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "gpt-4o",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 1_000_000,
+      "gen_ai.cost.total_cost": 7.25,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmCost).toBe(7.25);
+  });
+
+  test("LiteLLM's OTel v2 cost key is read as reported cost", () => {
+    const attrs: Attrs = {
+      "gen_ai.provider.name": "openai",
+      "gen_ai.request.model": "gpt-4o",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 1_000_000,
+      "litellm.cost.total": 6.5,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmCost).toBe(6.5);
+  });
+
+  test("a malformed reported cost falls back to the catalog", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "gpt-4o",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 0,
+      "gen_ai.usage.cost": "not-a-number",
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmCost).toBe(2.5);
+  });
+
+  test("cost is computed from tokens when the SDK reports none", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "gpt-4o",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 100_000,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+
+    // 1M * $2.50/M + 0.1M * $10/M = $3.50
+    expect(fields.llmCost).toBe(3.5);
+  });
+
+  test("response model is priced in preference to the request model", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      // Alias requested; cheaper mini snapshot actually served.
+      "gen_ai.request.model": "gpt-4o",
+      "gen_ai.response.model": "gpt-4o-mini-2024-07-18",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 0,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+
+    // Priced as gpt-4o-mini ($0.15/M input), not gpt-4o ($2.50/M).
+    expect(fields.llmCost).toBe(0.15);
+  });
+
+  test("request model prices the call when no response model exists", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "anthropic",
+      "gen_ai.request.model": "claude-3-5-sonnet-20241022",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 0,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmCost).toBe(3);
+  });
+
+  test("unknown model leaves cost at zero — no guessing", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "my-private-finetune",
+      "gen_ai.usage.input_tokens": 5000,
+      "gen_ai.usage.output_tokens": 5000,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmCost).toBe(0);
+  });
+
+  test("no tokens leaves cost at zero even for a known model", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "gpt-4o",
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmCost).toBe(0);
+  });
+
+  test("no model leaves cost at zero even with tokens", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.usage.input_tokens": 42,
+      "gen_ai.usage.output_tokens": 8,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmCost).toBe(0);
+  });
+
+  test("Bedrock-decorated model id is priced via normalization", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "aws.bedrock",
+      "gen_ai.request.model": "us.anthropic.claude-3-5-sonnet-20241022-v2:0",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 0,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmCost).toBe(3);
+  });
+
+  test("tool spans without tokens are not priced", () => {
+    const attrs: Attrs = {
+      "gen_ai.operation.name": "execute_tool",
+      "gen_ai.tool.name": "web_search",
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.isLlmSpan).toBe(true);
+    expect(fields.llmCost).toBe(0);
+  });
+});
+
+describe("LlmSpanUtil.extract — conversation id", () => {
+  test("gen_ai.conversation.id is extracted", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.conversation.id": "conv-1234",
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmConversationId).toBe("conv-1234");
+  });
+
+  test("session.id (OpenInference / Langfuse) is the fallback", () => {
+    const attrs: Attrs = {
+      "llm.model_name": "gpt-4o",
+      "session.id": "sess-abc",
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmConversationId).toBe("sess-abc");
+  });
+
+  test("OpenLLMetry association property is recognized", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "anthropic",
+      "traceloop.association.properties.session_id": "chat-42",
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmConversationId).toBe("chat-42");
+  });
+
+  test("the semconv key wins when multiple conventions are present", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.conversation.id": "conv-semconv",
+      "session.id": "sess-openinference",
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmConversationId).toBe("conv-semconv");
+  });
+
+  test("absent conversation id yields an empty string", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "gpt-4o",
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.llmConversationId).toBe("");
+  });
+
+  test("a non-LLM RUM span with session.id gets no conversation id", () => {
+    /*
+     * "session.id" is a generic OTel key browser/RUM spans carry — it is
+     * already denormalized into the sessionId column. It must only become a
+     * conversation id on LLM spans, never on the RUM fleet.
+     */
+    const attrs: Attrs = {
+      "http.method": "GET",
+      "http.route": "/checkout",
+      "session.id": "rum-session-123",
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+    expect(fields.isLlmSpan).toBe(false);
+    expect(fields.llmConversationId).toBe("");
+  });
+});
+
+describe("LlmSpanUtil.extract — project price overrides", () => {
+  test("an override prices a custom model the catalog does not know", () => {
+    const overrides: Array<LlmModelPrice> = [
+      {
+        modelPrefix: "my-custom-finetune",
+        inputPricePerMillionTokensInUSD: 1,
+        outputPricePerMillionTokensInUSD: 2,
+      },
+    ];
+
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "my-custom-finetune-v3",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 500_000,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs, overrides);
+
+    expect(fields.isLlmSpan).toBe(true);
+    // 1M * $1/M + 0.5M * $2/M = 2.
+    expect(fields.llmCost).toBe(2);
+  });
+
+  test("an override beats the built-in catalog on a prefix-length tie", () => {
+    const overrides: Array<LlmModelPrice> = [
+      {
+        modelPrefix: "gpt-4o",
+        inputPricePerMillionTokensInUSD: 1.25,
+        outputPricePerMillionTokensInUSD: 5,
+      },
+    ];
+
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "gpt-4o",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 1_000_000,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs, overrides);
+
+    // Built-in gpt-4o would price this at 2.5 + 10 = 12.5.
+    expect(fields.llmCost).toBe(6.25);
+  });
+
+  test("a longer built-in prefix still beats a shorter override", () => {
+    const overrides: Array<LlmModelPrice> = [
+      {
+        modelPrefix: "gpt-4o",
+        inputPricePerMillionTokensInUSD: 1.25,
+        outputPricePerMillionTokensInUSD: 5,
+      },
+    ];
+
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "gpt-4o-mini-2024-07-18",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 1_000_000,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs, overrides);
+
+    // Built-in gpt-4o-mini: 0.15 + 0.6 = 0.75.
+    expect(fields.llmCost).toBe(0.75);
+  });
+
+  test("an SDK-reported cost still wins over any override", () => {
+    const overrides: Array<LlmModelPrice> = [
+      {
+        modelPrefix: "gpt-4o",
+        inputPricePerMillionTokensInUSD: 1.25,
+        outputPricePerMillionTokensInUSD: 5,
+      },
+    ];
+
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "gpt-4o",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 1_000_000,
+      "gen_ai.usage.cost": 0.42,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs, overrides);
+
+    expect(fields.llmCost).toBe(0.42);
+  });
+
+  test("a zero-price override marks the span free rather than unpriced", () => {
+    const overrides: Array<LlmModelPrice> = [
+      {
+        modelPrefix: "llama-self-hosted",
+        inputPricePerMillionTokensInUSD: 0,
+        outputPricePerMillionTokensInUSD: 0,
+      },
+    ];
+
+    const attrs: Attrs = {
+      "gen_ai.system": "ollama",
+      "gen_ai.request.model": "llama-self-hosted-70b",
+      "gen_ai.usage.input_tokens": 5000,
+      "gen_ai.usage.output_tokens": 5000,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs, overrides);
+
+    expect(fields.isLlmSpan).toBe(true);
+    expect(fields.llmCost).toBe(0);
+  });
+
+  test("without overrides an unknown model still costs zero", () => {
+    const attrs: Attrs = {
+      "gen_ai.system": "openai",
+      "gen_ai.request.model": "my-custom-finetune-v3",
+      "gen_ai.usage.input_tokens": 1_000_000,
+      "gen_ai.usage.output_tokens": 500_000,
+    };
+
+    const fields: LlmSpanFields = LlmSpanUtil.extract(attrs);
+
+    expect(fields.llmCost).toBe(0);
+  });
+});

package/Tests/Server/Utils/Telemetry/LogErrorPatternSql.test.ts

@@ -0,0 +1,207 @@
+import {
+  LOG_ERROR_PATTERN_SOURCE_EXPRESSION,
+  buildLogErrorPatternExpression,
+  clampLogErrorPattern,
+  getLogErrorPatternParameterCount,
+} from "../../../../Server/Utils/Telemetry/LogErrorPatternSql";
+import { Statement } from "../../../../Server/Utils/AnalyticsDatabase/Statement";
+import {
+  LOG_ERROR_PATTERN_MAX_LENGTH,
+  LOG_ERROR_PATTERN_RULES,
+  LogErrorPatternRule,
+} from "../../../../Utils/Telemetry/LogErrorPattern";
+import { describe, expect, test } from "@jest/globals";
+
+/*
+ * The SQL compiler and the JavaScript normalizer must stay two views of one
+ * rule table. If they drift, the Insights page groups errors one way in the
+ * database and describes them another way in the browser — a discrepancy
+ * that is invisible until someone notices the counts do not add up.
+ */
+
+describe("buildLogErrorPatternExpression", () => {
+  test("nests one replaceRegexpAll per rule around the body expression", () => {
+    const statement: Statement = buildLogErrorPatternExpression();
+
+    const openCount: number = (
+      statement.query.match(/replaceRegexpAll\(/g) || []
+    ).length;
+
+    expect(openCount).toBe(LOG_ERROR_PATTERN_RULES.length);
+    expect(statement.query).toContain(LOG_ERROR_PATTERN_SOURCE_EXPRESSION);
+  });
+
+  test("coalesces the Nullable body column before rewriting it", () => {
+    /*
+     * `replaceRegexpAll(NULL, ...)` is NULL, which would collapse every
+     * body-less row into one meaningless NULL group at the top of the list.
+     */
+    expect(LOG_ERROR_PATTERN_SOURCE_EXPRESSION).toBe("ifNull(body, '')");
+    expect(buildLogErrorPatternExpression().query).toContain(
+      "replaceRegexpAll(ifNull(body, '')",
+    );
+  });
+
+  test("the whole expression is one balanced, fully-bound fragment", () => {
+    /*
+     * The expression is spliced into seven different statements; an
+     * unbalanced paren or an unbound placeholder would only surface as a
+     * ClickHouse syntax error at runtime, on a dashboard.
+     */
+    const statement: Statement = buildLogErrorPatternExpression();
+
+    let depth: number = 0;
+
+    for (const character of statement.query) {
+      if (character === "(") {
+        depth++;
+      }
+
+      if (character === ")") {
+        depth--;
+      }
+
+      expect(depth).toBeGreaterThanOrEqual(0);
+    }
+
+    expect(depth).toBe(0);
+
+    const referenced: Array<string> = Array.from(
+      statement.query.matchAll(/\{(p\d+):/g),
+    ).map((match: RegExpMatchArray): string => {
+      return match[1] as string;
+    });
+
+    expect(referenced.length).toBeGreaterThan(0);
+
+    for (const name of referenced) {
+      expect(statement.query_params).toHaveProperty(name);
+    }
+  });
+
+  test("binds every rule pattern and replacement as a parameter, never as SQL text", () => {
+    const statement: Statement = buildLogErrorPatternExpression();
+    const values: Array<unknown> = Object.values(statement.query_params);
+
+    for (const rule of LOG_ERROR_PATTERN_RULES) {
+      expect(values).toContain(rule.pattern);
+      expect(values).toContain(rule.replacement);
+      /*
+       * A regex spliced into the query text would be both an injection
+       * surface and a quoting nightmare (every rule contains backslashes).
+       */
+      expect(statement.query).not.toContain(rule.pattern);
+    }
+  });
+
+  test("applies rules innermost-first, matching the normalizer's iteration order", () => {
+    const statement: Statement = buildLogErrorPatternExpression();
+    const params: Record<string, unknown> = statement.query_params;
+
+    LOG_ERROR_PATTERN_RULES.forEach(
+      (rule: LogErrorPatternRule, index: number): void => {
+        expect(params[`p${index * 2}`]).toBe(rule.pattern);
+        expect(params[`p${index * 2 + 1}`]).toBe(rule.replacement);
+      },
+    );
+  });
+
+  test("truncates by CHARACTER, never by byte", () => {
+    const statement: Statement = buildLogErrorPatternExpression();
+
+    /*
+     * ClickHouse's plain `substring` counts bytes, so a 300-byte cut through
+     * a Japanese message or an emoji lands mid-sequence and the group key
+     * ends in a broken UTF-8 fragment. The absence is asserted too:
+     * "substringUTF8(" trivially contains "substring", so a loose check here
+     * would pass either way — which is how the byte-based version shipped in
+     * the first place.
+     */
+    expect(statement.query).toContain("substringUTF8(");
+    expect(statement.query).not.toMatch(/[^8]substring\(/);
+
+    expect(Object.values(statement.query_params)).toContain(
+      LOG_ERROR_PATTERN_MAX_LENGTH,
+    );
+  });
+
+  test("trims on both sides of the truncation, as the normalizer does", () => {
+    /*
+     * trim -> slice -> trim. Without the outer trim a cut landing mid-space
+     * would yield a pattern with a trailing space, which is a different
+     * GROUP BY key from the same pattern trimmed.
+     */
+    const query: string = buildLogErrorPatternExpression().query;
+
+    expect(query.startsWith("trimBoth(substringUTF8(trimBoth(")).toBe(true);
+    expect(query.endsWith("))")).toBe(true);
+  });
+
+  test("reports its own parameter count", () => {
+    const statement: Statement = buildLogErrorPatternExpression();
+
+    expect(getLogErrorPatternParameterCount()).toBe(
+      Object.keys(statement.query_params).length,
+    );
+  });
+
+  test("accepts an alternate trusted source expression", () => {
+    const statement: Statement = buildLogErrorPatternExpression(
+      "ifNull(attributes['message'], '')",
+    );
+
+    expect(statement.query).toContain(
+      "replaceRegexpAll(ifNull(attributes['message'], '')",
+    );
+  });
+});
+
+describe("clampLogErrorPattern", () => {
+  test("leaves a pattern within the limit untouched", () => {
+    expect(clampLogErrorPattern("connection refused to <ip>")).toBe(
+      "connection refused to <ip>",
+    );
+  });
+
+  test("clamps an oversized pattern to the expression's own maximum", () => {
+    const oversized: string = "x".repeat(LOG_ERROR_PATTERN_MAX_LENGTH * 3);
+
+    expect(clampLogErrorPattern(oversized).length).toBe(
+      LOG_ERROR_PATTERN_MAX_LENGTH,
+    );
+  });
+
+  test("an empty pattern clamps to itself", () => {
+    expect(clampLogErrorPattern("")).toBe("");
+  });
+
+  test("counts CODE POINTS, matching what substringUTF8 cuts by", () => {
+    /*
+     * 300 code points, but 301 UTF-16 units because of the emoji. Measured
+     * the JS way the clamp would fire on a key the database considers
+     * exactly at the limit, slice it, and leave a lone surrogate at the
+     * tail — after which the `=` bind matches zero rows and every
+     * drill-down panel comes back empty for a row showing a real count,
+     * with nothing thrown to notice.
+     */
+    const atLimit: string = `${"x".repeat(299)}\u{1F680}`;
+
+    expect(Array.from(atLimit).length).toBe(LOG_ERROR_PATTERN_MAX_LENGTH);
+    expect(atLimit.length).toBe(LOG_ERROR_PATTERN_MAX_LENGTH + 1);
+
+    expect(clampLogErrorPattern(atLimit)).toBe(atLimit);
+  });
+
+  test("never cuts an astral character in half", () => {
+    const oversized: string = "\u{1F680}".repeat(
+      LOG_ERROR_PATTERN_MAX_LENGTH + 50,
+    );
+
+    const clamped: string = clampLogErrorPattern(oversized);
+
+    expect(Array.from(clamped).length).toBe(LOG_ERROR_PATTERN_MAX_LENGTH);
+    // No lone surrogate at either edge.
+    expect(clamped).not.toMatch(/[\uD800-\uDBFF]$/);
+    expect(clamped).not.toMatch(/^[\uDC00-\uDFFF]/);
+  });
+});