@trazum/core 1.81.0 → 1.83.0

package/src/langsmith.ts

@@ -0,0 +1,315 @@
+/**
+ * LangSmith's run export, read as a usage log.
+ *
+ * The fifth converter in the pattern `from-claude-code` started: a pure
+ * function turns one tool's export into Trazum's usage-log records, and every
+ * door prices it from there. LangSmith is the tracing product most LangChain
+ * teams already run, so the export exists before Trazum does.
+ *
+ * **The format is derived, not guessed.** Every field below is `RunBase` and
+ * `Run` in `python/langsmith/schemas.py` in langchain-ai/langsmith-sdk. A
+ * converter written from memory of an API is a converter that silently
+ * mis-reads somebody's bill.
+ *
+ * ## A run is not a call, and that is the whole difficulty
+ *
+ * LangSmith records a **run**, and a trace is a tree of them: chains, tools and
+ * retrievers alongside the model calls. Only `run_type: "llm"` carries token
+ * counts, and summing the tree would count the same tokens once per level. So
+ * everything else is skipped and **counted out loud** — a converter that
+ * quietly dropped two thirds of a file would look like one that read it.
+ *
+ * ## The model, and why it is refused rather than inferred
+ *
+ * There is no model column. The name lives in `extra.metadata.ls_model_name`,
+ * or in the invocation parameters the SDK records beside it, and a run whose
+ * metadata carries neither cannot be priced. The obvious guess is the run's
+ * own `name` — LangChain names them after the class, `ChatOpenAI`, which is a
+ * client and not a model. Pricing a call by the class that made it would be a
+ * figure attributed to something it does not describe, which is the fault this
+ * repository keeps finding in itself. Such a run is counted in `unnamedModel`
+ * and dropped.
+ *
+ * ## What deliberately does not cross
+ *
+ * `inputs` and `outputs` are the prompt and the completion, on every single
+ * run. **Neither is read.** `extra.metadata` is read for two named keys and
+ * nothing else, because it is a free-form bag the operator fills and the next
+ * thing they put in it might be a credential. The converter names the fields
+ * it takes and takes nothing else, and a fixture plants a marker in each and
+ * greps the whole output.
+ *
+ * ## What it refuses to invent
+ *
+ * There is no cache-token split. `prompt_cost_details` looks like one and is
+ * not: it is LangSmith's own priced breakdown in dollars, computed from
+ * LangSmith's price table. It never enters a record, and neither does
+ * `total_cost` — `reportedCostUsd` returns it on its own, the way
+ * `from-litellm` keeps the gateway's arithmetic apart from Trazum's. Two price
+ * tables, two figures; merging them is how a report becomes quietly wrong.
+ */
+
+/** One converted record, shaped exactly as `parseUsageLine` reads it. */
+export interface LangsmithRecord {
+  model: string;
+  ts?: string;
+  label?: string;
+  session?: string;
+  usage: {
+    input_tokens: number;
+    output_tokens: number;
+  };
+}
+
+export interface LangsmithConversion {
+  records: LangsmithRecord[];
+  /** Runs that were model calls and converted. */
+  rows: number;
+  /**
+   * Runs that were not model calls: chains, tools, retrievers, prompts.
+   *
+   * Counted rather than ignored. They are most of a LangSmith export, and a
+   * reader who converted a thousand runs into three hundred records is owed
+   * the reason.
+   */
+  notModelCalls: number;
+  /** Model calls whose metadata named no model: counted, never guessed. */
+  unnamedModel: number;
+  /** Model calls carrying no token counts at all. */
+  noTokens: number;
+  /**
+   * LangSmith's own cost total, in dollars, or null when no run reported one.
+   *
+   * Kept apart from anything Trazum computes and never merged into a record.
+   * Null is an absence, not a total of nothing.
+   */
+  reportedCostUsd: number | null;
+  /** Lines or documents that did not parse as JSON at all. */
+  unparseable: number;
+}
+
+const asObject = (value: unknown): Record<string, unknown> | null =>
+  typeof value === 'object' && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+
+const tryParse = (text: string): unknown => {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+};
+
+const asCount = (value: unknown): number | undefined => {
+  const n = typeof value === 'string' ? Number(value) : value;
+  return typeof n === 'number' && Number.isFinite(n) && n >= 0 ? n : undefined;
+};
+
+const asName = (value: unknown): string | null =>
+  typeof value === 'string' && value.trim() !== '' ? value.trim() : null;
+
+/** `extra.metadata`, which is where LangSmith documents everything user-set. */
+function metadataOf(run: Record<string, unknown>): Record<string, unknown> | null {
+  const extra = asObject(run.extra);
+  return extra === null ? asObject(run.metadata) : (asObject(extra.metadata) ?? asObject(run.metadata));
+}
+
+/**
+ * The model, from the two named places it is recorded and nowhere else.
+ *
+ * `ls_model_name` is what the SDK writes; `model` and `model_name` are what a
+ * team writes by hand when they set metadata themselves. The run's `name` is
+ * deliberately not consulted: LangChain names a run after the client class,
+ * and `ChatOpenAI` is not a model anybody can price.
+ */
+function modelOf(run: Record<string, unknown>): string | null {
+  const metadata = metadataOf(run);
+  if (metadata !== null) {
+    for (const key of ['ls_model_name', 'model_name', 'model']) {
+      const found = asName(metadata[key]);
+      if (found !== null) return found;
+    }
+    // The invocation parameters, where the SDK also records it.
+    const params = asObject(metadata.invocation_params);
+    if (params !== null) {
+      for (const key of ['model', 'model_name']) {
+        const found = asName(params[key]);
+        if (found !== null) return found;
+      }
+    }
+  }
+  return null;
+}
+
+/**
+ * The label, from the tags a team already puts on its traces.
+ *
+ * `tags` is LangSmith's own list and the first entry wins, because a workload
+ * has one name in a bill. Then the two metadata keys a team would plausibly
+ * name a workload by. Never joined, never invented.
+ */
+function labelOf(run: Record<string, unknown>): string | undefined {
+  if (Array.isArray(run.tags)) {
+    for (const tag of run.tags) {
+      const found = asName(tag);
+      if (found !== null) return found;
+    }
+  }
+  const metadata = metadataOf(run);
+  if (metadata !== null) {
+    for (const key of ['label', 'workload']) {
+      const found = asName(metadata[key]);
+      if (found !== null) return found;
+    }
+  }
+  return undefined;
+}
+
+/** The run's start time as an ISO instant, or undefined. */
+function timeOf(run: Record<string, unknown>): string | undefined {
+  const raw = run.start_time ?? run.startTime;
+  if (typeof raw !== 'string' && typeof raw !== 'number') return undefined;
+  const ms = typeof raw === 'number' ? raw : Date.parse(raw);
+  if (!Number.isFinite(ms)) return undefined;
+  return new Date(ms).toISOString();
+}
+
+/**
+ * The token counts, from the two shapes LangSmith has written them in.
+ *
+ * `prompt_tokens` and `completion_tokens` sit on the run; newer exports also
+ * carry a nested `usage_metadata` with `input_tokens` and `output_tokens`.
+ * Both are read because both are in the wild, and an export from last year is
+ * still somebody's bill.
+ */
+function tokensOf(run: Record<string, unknown>): { input?: number; output?: number } {
+  const nested = asObject(run.usage_metadata);
+  return {
+    input: asCount(run.prompt_tokens) ?? asCount(nested?.input_tokens),
+    output: asCount(run.completion_tokens) ?? asCount(nested?.output_tokens),
+  };
+}
+
+/**
+ * Convert a LangSmith run export.
+ *
+ * Accepts a JSON array of runs, a single run, `{ runs: [...] }` as the list
+ * endpoint returns it, or newline-delimited runs. Pure over its input.
+ */
+export function langsmithRecords(text: string): LangsmithConversion {
+  const out: LangsmithConversion = {
+    records: [],
+    rows: 0,
+    notModelCalls: 0,
+    unnamedModel: 0,
+    noTokens: 0,
+    reportedCostUsd: null,
+    unparseable: 0,
+  };
+
+  const runs: unknown[] = [];
+  const whole = tryParse(text);
+  if (whole !== undefined) {
+    if (Array.isArray(whole)) runs.push(...whole);
+    else {
+      const doc = asObject(whole);
+      if (doc !== null && Array.isArray(doc.runs)) runs.push(...doc.runs);
+      else if (doc !== null) runs.push(doc);
+    }
+  } else {
+    for (const line of text.split('\n')) {
+      if (line.trim() === '') continue;
+      const parsed = tryParse(line);
+      if (parsed === undefined) out.unparseable += 1;
+      else if (Array.isArray(parsed)) runs.push(...parsed);
+      else runs.push(parsed);
+    }
+  }
+
+  let cost = 0;
+  let sawCost = false;
+
+  for (const raw of runs) {
+    const run = asObject(raw);
+    if (run === null) continue;
+
+    /*
+      A run is only a call when LangSmith says it is one. The tree carries
+      chains, tools, retrievers and prompts alongside the model calls, and the
+      parent chain of an llm run repeats its child's tokens: summing the tree
+      would bill the same tokens once per level. `run_type` is the field that
+      settles it, and a run with no run_type at all is not a run.
+    */
+    const type = asName(run.run_type);
+    if (type === null) continue;
+    if (type !== 'llm') {
+      out.notModelCalls += 1;
+      continue;
+    }
+
+    out.rows += 1;
+
+    const reported = asCount(run.total_cost);
+    if (reported !== undefined) {
+      cost += reported;
+      sawCost = true;
+    }
+
+    const model = modelOf(run);
+    if (model === null) {
+      out.unnamedModel += 1;
+      continue;
+    }
+
+    const { input, output } = tokensOf(run);
+    if (input === undefined && output === undefined) {
+      out.noTokens += 1;
+      continue;
+    }
+    if (input === 0 && output === 0) out.noTokens += 1;
+
+    const ts = timeOf(run);
+    const label = labelOf(run);
+    /*
+      The trace is the conversation and the run id is one call. `trace_id`
+      spans the calls a single request made, which is the identity Trazum's
+      conversation findings need; answering them from `id` would report every
+      call as a conversation of one.
+    */
+    const session = asName(run.trace_id);
+
+    out.records.push({
+      model,
+      ...(ts !== undefined ? { ts } : {}),
+      ...(label !== undefined ? { label } : {}),
+      ...(session !== null ? { session } : {}),
+      usage: { input_tokens: input ?? 0, output_tokens: output ?? 0 },
+    });
+  }
+
+  // Rounded because these are dollars summed in floating point, and a total
+  // printed to eight decimal places reads as a precision nobody has.
+  if (sawCost) out.reportedCostUsd = Math.round(cost * 1e6) / 1e6;
+
+  return out;
+}
+
+/**
+ * Whether a file looks like a LangSmith run export, by shape.
+ *
+ * Deliberately narrow: `run_type` beside one of the trace fields only
+ * LangSmith writes. A looser test would claim a Helicone export or a LiteLLM
+ * spend log — both carry `prompt_tokens` — and refuse it in this file's words
+ * instead of letting the code that can read it try.
+ */
+export function looksLikeLangsmith(text: string, prefixBytes = 8192): boolean {
+  const head = text.slice(0, prefixBytes);
+  if (!head.includes('"run_type"')) return false;
+  return (
+    head.includes('"trace_id"') ||
+    head.includes('"dotted_order"') ||
+    head.includes('"start_time"') ||
+    head.includes('"session_id"')
+  );
+}

package/src/levers.ts

@@ -1,4 +1,4 @@
-import { effectivePricing, multipliersFor } from './pricing.js';
+import { effectivePricing, isOffered, multipliersFor } from './pricing.js';
 import { UNLABELLED } from './usage.js';
 import type { PricingCatalogue } from './pricing.js';
 import type { UsageBreakdown, UsageProfileReport } from './usage.js';
@@ -21,7 +21,7 @@ import type { Capability, ModelPricing } from './types.js';
  *
  * | lever | what it moves |
  * |---|---|
- * | which model the call goes to | Opus 5 → Sonnet 5 is 40% off; → Haiku 4.5 is 80% |
+ * | which model the call goes to | Opus 5 → Sonnet 5 is 60% off; → Haiku 4.5 is 80% |
  * | the Batch API | 50% flat, on input and output |
  * | prompt caching | 3–4x the rules |
  * | shortening the prompt | ~1% |
@@ -204,7 +204,7 @@ function candidateFor(
       m.id !== model.id &&
       m.capability === target &&
       m.provider === model.provider &&
-      m.recommendable !== false &&
+      isOffered(m) &&
       m.contextWindow >= avgInput,
   );
   if (candidates.length === 0) return null;

package/src/litellm.ts

@@ -0,0 +1,243 @@
+/**
+ * LiteLLM's spend log, read as a usage log.
+ *
+ * `from-claude-code` proved the pattern and `from-otel` generalised it: a pure
+ * converter turns one tool's export into Trazum's usage-log records, and every
+ * door prices it from there. LiteLLM is the gateway a great many teams already
+ * put in front of every provider, so its `LiteLLM_SpendLogs` table is the one
+ * export most likely to already exist on somebody's disk.
+ *
+ * **The format is derived, not guessed.** Every field below is read off
+ * `litellm/proxy/schema.prisma` in BerriAI/litellm, model `LiteLLM_SpendLogs`.
+ * A converter written from memory of an API is a converter that silently
+ * mis-reads somebody's bill, and this project has one rule above all others
+ * about numbers it cannot justify.
+ *
+ * ## What maps
+ *
+ * | LiteLLM | Trazum |
+ * | --- | --- |
+ * | `model` | `model` |
+ * | `prompt_tokens` | `usage.input_tokens` |
+ * | `completion_tokens` | `usage.output_tokens` |
+ * | `startTime` | `ts` |
+ * | `session_id` | `session` |
+ * | `request_tags[0]`, else `metadata.tags[0]`, else `model_group` | `label` |
+ *
+ * ## What deliberately does not cross
+ *
+ * The row carries `messages` and `response` — the prompt itself and the
+ * completion — plus `api_key` (hashed, still credential-shaped),
+ * `requester_ip_address`, `user` and `end_user`. **None of it is read.** The
+ * converter names the fields it takes and takes nothing else, and a fixture
+ * plants a marker in each of them and greps the whole output.
+ *
+ * ## What it refuses to invent
+ *
+ * `cache_hit` is a flag and `cache_key` an identifier; neither is a token
+ * count. A converted record therefore carries no cache split at all, and the
+ * caching questions come back `cannot-tell` rather than answered from a
+ * guess — the same refusal as inventing a price. `cacheFlagged` counts the
+ * rows that said "hit" so the operator can see the gap is real rather than
+ * empty.
+ *
+ * `spend` is LiteLLM's own priced figure and is **never** merged into
+ * Trazum's catalogue-priced total. It is a second measurement of the same
+ * calls, kept apart the way the store's provider-billed standing is kept
+ * apart from the log's, and returned on its own so a caller who wants to
+ * compare the two can, deliberately.
+ */
+
+/** One converted record, shaped exactly as `parseUsageLine` reads it. */
+export interface LiteLlmRecord {
+  model: string;
+  ts?: string;
+  label?: string;
+  session?: string;
+  usage: {
+    input_tokens: number;
+    output_tokens: number;
+  };
+}
+
+export interface LiteLlmConversion {
+  records: LiteLlmRecord[];
+  /** Rows that were spend logs and converted. */
+  rows: number;
+  /** Rows naming no model: counted, never guessed at, never dropped silently. */
+  unnamedModel: number;
+  /** Rows carrying no token counts at all — a logged call nobody can price. */
+  noTokens: number;
+  /** Rows LiteLLM marked as a cache hit, with no token split to act on. */
+  cacheFlagged: number;
+  /**
+   * LiteLLM's own total for the converted rows, in USD, or null when no row
+   * carried one. **Never** added to anything Trazum computes: it is the
+   * gateway's arithmetic over the gateway's price table, and merging two
+   * price tables into one figure is how a report becomes quietly wrong.
+   */
+  reportedSpendUsd: number | null;
+  /** Lines or documents that did not parse as JSON at all. */
+  unparseable: number;
+}
+
+const asObject = (value: unknown): Record<string, unknown> | null =>
+  typeof value === 'object' && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+
+const tryParse = (text: string): unknown => {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+};
+
+const asCount = (value: unknown): number | undefined => {
+  const n = typeof value === 'string' ? Number(value) : value;
+  return typeof n === 'number' && Number.isFinite(n) && n >= 0 ? n : undefined;
+};
+
+/**
+ * The first tag LiteLLM carries for a row, from the two places it puts them.
+ *
+ * `request_tags` is the documented column and `metadata.tags` is where the
+ * proxy writes them when the caller passes them in the request body. First
+ * only: a workload has one name in a bill, and joining several tags into one
+ * label would invent a workload that does not exist.
+ */
+function labelOf(row: Record<string, unknown>): string | undefined {
+  const fromColumn = Array.isArray(row.request_tags) ? row.request_tags : null;
+  if (fromColumn !== null && typeof fromColumn[0] === 'string' && fromColumn[0] !== '') {
+    return fromColumn[0];
+  }
+  const metadata = asObject(row.metadata);
+  const fromMetadata = metadata !== null && Array.isArray(metadata.tags) ? metadata.tags : null;
+  if (fromMetadata !== null && typeof fromMetadata[0] === 'string' && fromMetadata[0] !== '') {
+    return fromMetadata[0];
+  }
+  // The public model name a proxy route is known by. A weaker label than a
+  // tag — it groups by route rather than by workload — and better than none.
+  if (typeof row.model_group === 'string' && row.model_group !== '') return row.model_group;
+  return undefined;
+}
+
+/** `startTime` as an ISO instant, or undefined when it is not a time. */
+function timeOf(row: Record<string, unknown>): string | undefined {
+  const raw = row.startTime ?? row.start_time;
+  if (typeof raw !== 'string' && typeof raw !== 'number') return undefined;
+  const ms = typeof raw === 'number' ? raw : Date.parse(raw);
+  if (!Number.isFinite(ms)) return undefined;
+  return new Date(ms).toISOString();
+}
+
+/**
+ * Convert a LiteLLM spend-log export.
+ *
+ * Accepts a JSON array of rows, a single row, `{ data: [...] }` as the proxy's
+ * own endpoints return it, or newline-delimited rows — so a `psql --json`
+ * dump, an API response and a streamed capture all work. Pure over its input.
+ */
+export function litellmRecords(text: string): LiteLlmConversion {
+  const out: LiteLlmConversion = {
+    records: [],
+    rows: 0,
+    unnamedModel: 0,
+    noTokens: 0,
+    cacheFlagged: 0,
+    reportedSpendUsd: null,
+    unparseable: 0,
+  };
+
+  const rows: unknown[] = [];
+  const whole = tryParse(text);
+  if (whole !== undefined) {
+    if (Array.isArray(whole)) rows.push(...whole);
+    else {
+      const doc = asObject(whole);
+      // `{ data: [...] }` is what the proxy's spend endpoints return.
+      if (doc !== null && Array.isArray(doc.data)) rows.push(...doc.data);
+      else if (doc !== null) rows.push(doc);
+    }
+  } else {
+    for (const line of text.split('\n')) {
+      if (line.trim() === '') continue;
+      const parsed = tryParse(line);
+      if (parsed === undefined) out.unparseable += 1;
+      else if (Array.isArray(parsed)) rows.push(...parsed);
+      else rows.push(parsed);
+    }
+  }
+
+  let spend = 0;
+  let sawSpend = false;
+
+  for (const raw of rows) {
+    const row = asObject(raw);
+    if (row === null) continue;
+    // A spend log row is one that carries at least one of the two token
+    // columns. Anything else in the same file is not a call and is skipped
+    // rather than converted into a record of zero.
+    const input = asCount(row.prompt_tokens);
+    const output = asCount(row.completion_tokens);
+    if (input === undefined && output === undefined) continue;
+
+    out.rows += 1;
+
+    if (input === 0 && output === 0) out.noTokens += 1;
+    if (row.cache_hit === true || row.cache_hit === 'true' || row.cache_hit === 'True') {
+      out.cacheFlagged += 1;
+    }
+
+    const reported = asCount(row.spend);
+    if (reported !== undefined) {
+      spend += reported;
+      sawSpend = true;
+    }
+
+    const model = typeof row.model === 'string' ? row.model.trim() : '';
+    if (model === '') {
+      // Counted, never guessed at. `model_group` is a route name, not a model,
+      // and pricing a call by the route it took would be a figure attributed
+      // to something it does not describe.
+      out.unnamedModel += 1;
+      continue;
+    }
+
+    const ts = timeOf(row);
+    const label = labelOf(row);
+    const session = typeof row.session_id === 'string' && row.session_id !== ''
+      ? row.session_id
+      : undefined;
+
+    out.records.push({
+      model,
+      ...(ts !== undefined ? { ts } : {}),
+      ...(label !== undefined ? { label } : {}),
+      ...(session !== undefined ? { session } : {}),
+      usage: { input_tokens: input ?? 0, output_tokens: output ?? 0 },
+    });
+  }
+
+  out.reportedSpendUsd = sawSpend ? spend : null;
+  return out;
+}
+
+/**
+ * Whether a file looks like a LiteLLM spend log, by shape.
+ *
+ * Deliberately narrow: `request_id` beside one of the two token columns, and
+ * a field no other export in this project carries. A looser test would claim
+ * an OpenAI usage response, whose rows also have `prompt_tokens`, and refuse
+ * it in this file's words instead of letting the code that can read it try.
+ */
+export function looksLikeLiteLlm(text: string, prefixBytes = 8192): boolean {
+  const head = text.slice(0, prefixBytes);
+  if (!head.includes('prompt_tokens') && !head.includes('completion_tokens')) return false;
+  return (
+    head.includes('"request_id"') ||
+    head.includes('"custom_llm_provider"') ||
+    head.includes('"model_group"')
+  );
+}

package/src/measured-profile.ts

@@ -70,6 +70,10 @@ export interface MeasuredUsage {
    * call is the same size. It is handed over as the better of two
    * approximations — a measured token share beats a typed call share — and
    * named honestly here so no rendering can call it something it is not.
+   *
+   * The denominator is every input token, **writes included**. Leaving writes
+   * out made a constantly-rewritten cache report a high hit rate, which is the
+   * one workload that most needs to be told otherwise.
    */
   cacheReadShare: number;
   /**
@@ -105,6 +109,7 @@ export function measuredUsage(
   const outputTokens = slices.reduce((sum, row) => sum + row.breakdown.outputTokens, 0);
   const inputTokens = slices.reduce((sum, row) => sum + row.breakdown.inputTokens, 0);
   const cacheReadTokens = slices.reduce((sum, row) => sum + row.breakdown.cacheReadTokens, 0);
+  const cacheWriteTokens = slices.reduce((sum, row) => sum + row.breakdown.cacheWriteTokens, 0);
 
   // The model carrying the most spend. Ties break on the larger call count, so
   // the answer is stable rather than dependent on map ordering.
@@ -127,8 +132,27 @@ export function measuredUsage(
       ? { fromDays: spanDays, factor: SCALE_TO_DAYS / spanDays }
       : null;
 
-  const readShare =
-    inputTokens + cacheReadTokens > 0 ? cacheReadTokens / (inputTokens + cacheReadTokens) : 0;
+  /**
+   * Every input token in the denominator, writes included.
+   *
+   * This was `reads / (input + reads)`, which leaves cache *writes* out of the
+   * total it is a share of. A workload that rewrites its prefix on every call
+   * and reads it back rarely then reports a **high** hit rate: with 100 read
+   * tokens, no plain input and 10,000 written, the old expression answers 100%
+   * where the truth is 1%.
+   *
+   * That is the worst direction for this number to be wrong in. It is handed
+   * to `optimize` as `cacheHitRate`, which decides whether caching is paying
+   * off — so the shape that is burning money on writes was the shape most
+   * likely to be told its cache was working perfectly.
+   *
+   * Writes are input tokens: they are billed at the input rate times the write
+   * multiplier, they arrive in the same `usage` block, and any accounting of
+   * "what share of the input came from cache" that omits them is answering a
+   * narrower question than the one it is named for.
+   */
+  const allInputTokens = inputTokens + cacheReadTokens + cacheWriteTokens;
+  const readShare = allInputTokens > 0 ? cacheReadTokens / allInputTokens : 0;
 
   return {
     profile: {