@trazum/core 1.80.1 → 1.82.0

package/src/litellm.ts

@@ -0,0 +1,243 @@
+/**
+ * LiteLLM's spend log, read as a usage log.
+ *
+ * `from-claude-code` proved the pattern and `from-otel` generalised it: a pure
+ * converter turns one tool's export into Trazum's usage-log records, and every
+ * door prices it from there. LiteLLM is the gateway a great many teams already
+ * put in front of every provider, so its `LiteLLM_SpendLogs` table is the one
+ * export most likely to already exist on somebody's disk.
+ *
+ * **The format is derived, not guessed.** Every field below is read off
+ * `litellm/proxy/schema.prisma` in BerriAI/litellm, model `LiteLLM_SpendLogs`.
+ * A converter written from memory of an API is a converter that silently
+ * mis-reads somebody's bill, and this project has one rule above all others
+ * about numbers it cannot justify.
+ *
+ * ## What maps
+ *
+ * | LiteLLM | Trazum |
+ * | --- | --- |
+ * | `model` | `model` |
+ * | `prompt_tokens` | `usage.input_tokens` |
+ * | `completion_tokens` | `usage.output_tokens` |
+ * | `startTime` | `ts` |
+ * | `session_id` | `session` |
+ * | `request_tags[0]`, else `metadata.tags[0]`, else `model_group` | `label` |
+ *
+ * ## What deliberately does not cross
+ *
+ * The row carries `messages` and `response` — the prompt itself and the
+ * completion — plus `api_key` (hashed, still credential-shaped),
+ * `requester_ip_address`, `user` and `end_user`. **None of it is read.** The
+ * converter names the fields it takes and takes nothing else, and a fixture
+ * plants a marker in each of them and greps the whole output.
+ *
+ * ## What it refuses to invent
+ *
+ * `cache_hit` is a flag and `cache_key` an identifier; neither is a token
+ * count. A converted record therefore carries no cache split at all, and the
+ * caching questions come back `cannot-tell` rather than answered from a
+ * guess — the same refusal as inventing a price. `cacheFlagged` counts the
+ * rows that said "hit" so the operator can see the gap is real rather than
+ * empty.
+ *
+ * `spend` is LiteLLM's own priced figure and is **never** merged into
+ * Trazum's catalogue-priced total. It is a second measurement of the same
+ * calls, kept apart the way the store's provider-billed standing is kept
+ * apart from the log's, and returned on its own so a caller who wants to
+ * compare the two can, deliberately.
+ */
+
+/** One converted record, shaped exactly as `parseUsageLine` reads it. */
+export interface LiteLlmRecord {
+  model: string;
+  ts?: string;
+  label?: string;
+  session?: string;
+  usage: {
+    input_tokens: number;
+    output_tokens: number;
+  };
+}
+
+export interface LiteLlmConversion {
+  records: LiteLlmRecord[];
+  /** Rows that were spend logs and converted. */
+  rows: number;
+  /** Rows naming no model: counted, never guessed at, never dropped silently. */
+  unnamedModel: number;
+  /** Rows carrying no token counts at all — a logged call nobody can price. */
+  noTokens: number;
+  /** Rows LiteLLM marked as a cache hit, with no token split to act on. */
+  cacheFlagged: number;
+  /**
+   * LiteLLM's own total for the converted rows, in USD, or null when no row
+   * carried one. **Never** added to anything Trazum computes: it is the
+   * gateway's arithmetic over the gateway's price table, and merging two
+   * price tables into one figure is how a report becomes quietly wrong.
+   */
+  reportedSpendUsd: number | null;
+  /** Lines or documents that did not parse as JSON at all. */
+  unparseable: number;
+}
+
+const asObject = (value: unknown): Record<string, unknown> | null =>
+  typeof value === 'object' && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+
+const tryParse = (text: string): unknown => {
+  try {
+    return JSON.parse(text);
+  } catch {
+    return undefined;
+  }
+};
+
+const asCount = (value: unknown): number | undefined => {
+  const n = typeof value === 'string' ? Number(value) : value;
+  return typeof n === 'number' && Number.isFinite(n) && n >= 0 ? n : undefined;
+};
+
+/**
+ * The first tag LiteLLM carries for a row, from the two places it puts them.
+ *
+ * `request_tags` is the documented column and `metadata.tags` is where the
+ * proxy writes them when the caller passes them in the request body. First
+ * only: a workload has one name in a bill, and joining several tags into one
+ * label would invent a workload that does not exist.
+ */
+function labelOf(row: Record<string, unknown>): string | undefined {
+  const fromColumn = Array.isArray(row.request_tags) ? row.request_tags : null;
+  if (fromColumn !== null && typeof fromColumn[0] === 'string' && fromColumn[0] !== '') {
+    return fromColumn[0];
+  }
+  const metadata = asObject(row.metadata);
+  const fromMetadata = metadata !== null && Array.isArray(metadata.tags) ? metadata.tags : null;
+  if (fromMetadata !== null && typeof fromMetadata[0] === 'string' && fromMetadata[0] !== '') {
+    return fromMetadata[0];
+  }
+  // The public model name a proxy route is known by. A weaker label than a
+  // tag — it groups by route rather than by workload — and better than none.
+  if (typeof row.model_group === 'string' && row.model_group !== '') return row.model_group;
+  return undefined;
+}
+
+/** `startTime` as an ISO instant, or undefined when it is not a time. */
+function timeOf(row: Record<string, unknown>): string | undefined {
+  const raw = row.startTime ?? row.start_time;
+  if (typeof raw !== 'string' && typeof raw !== 'number') return undefined;
+  const ms = typeof raw === 'number' ? raw : Date.parse(raw);
+  if (!Number.isFinite(ms)) return undefined;
+  return new Date(ms).toISOString();
+}
+
+/**
+ * Convert a LiteLLM spend-log export.
+ *
+ * Accepts a JSON array of rows, a single row, `{ data: [...] }` as the proxy's
+ * own endpoints return it, or newline-delimited rows — so a `psql --json`
+ * dump, an API response and a streamed capture all work. Pure over its input.
+ */
+export function litellmRecords(text: string): LiteLlmConversion {
+  const out: LiteLlmConversion = {
+    records: [],
+    rows: 0,
+    unnamedModel: 0,
+    noTokens: 0,
+    cacheFlagged: 0,
+    reportedSpendUsd: null,
+    unparseable: 0,
+  };
+
+  const rows: unknown[] = [];
+  const whole = tryParse(text);
+  if (whole !== undefined) {
+    if (Array.isArray(whole)) rows.push(...whole);
+    else {
+      const doc = asObject(whole);
+      // `{ data: [...] }` is what the proxy's spend endpoints return.
+      if (doc !== null && Array.isArray(doc.data)) rows.push(...doc.data);
+      else if (doc !== null) rows.push(doc);
+    }
+  } else {
+    for (const line of text.split('\n')) {
+      if (line.trim() === '') continue;
+      const parsed = tryParse(line);
+      if (parsed === undefined) out.unparseable += 1;
+      else if (Array.isArray(parsed)) rows.push(...parsed);
+      else rows.push(parsed);
+    }
+  }
+
+  let spend = 0;
+  let sawSpend = false;
+
+  for (const raw of rows) {
+    const row = asObject(raw);
+    if (row === null) continue;
+    // A spend log row is one that carries at least one of the two token
+    // columns. Anything else in the same file is not a call and is skipped
+    // rather than converted into a record of zero.
+    const input = asCount(row.prompt_tokens);
+    const output = asCount(row.completion_tokens);
+    if (input === undefined && output === undefined) continue;
+
+    out.rows += 1;
+
+    if (input === 0 && output === 0) out.noTokens += 1;
+    if (row.cache_hit === true || row.cache_hit === 'true' || row.cache_hit === 'True') {
+      out.cacheFlagged += 1;
+    }
+
+    const reported = asCount(row.spend);
+    if (reported !== undefined) {
+      spend += reported;
+      sawSpend = true;
+    }
+
+    const model = typeof row.model === 'string' ? row.model.trim() : '';
+    if (model === '') {
+      // Counted, never guessed at. `model_group` is a route name, not a model,
+      // and pricing a call by the route it took would be a figure attributed
+      // to something it does not describe.
+      out.unnamedModel += 1;
+      continue;
+    }
+
+    const ts = timeOf(row);
+    const label = labelOf(row);
+    const session = typeof row.session_id === 'string' && row.session_id !== ''
+      ? row.session_id
+      : undefined;
+
+    out.records.push({
+      model,
+      ...(ts !== undefined ? { ts } : {}),
+      ...(label !== undefined ? { label } : {}),
+      ...(session !== undefined ? { session } : {}),
+      usage: { input_tokens: input ?? 0, output_tokens: output ?? 0 },
+    });
+  }
+
+  out.reportedSpendUsd = sawSpend ? spend : null;
+  return out;
+}
+
+/**
+ * Whether a file looks like a LiteLLM spend log, by shape.
+ *
+ * Deliberately narrow: `request_id` beside one of the two token columns, and
+ * a field no other export in this project carries. A looser test would claim
+ * an OpenAI usage response, whose rows also have `prompt_tokens`, and refuse
+ * it in this file's words instead of letting the code that can read it try.
+ */
+export function looksLikeLiteLlm(text: string, prefixBytes = 8192): boolean {
+  const head = text.slice(0, prefixBytes);
+  if (!head.includes('prompt_tokens') && !head.includes('completion_tokens')) return false;
+  return (
+    head.includes('"request_id"') ||
+    head.includes('"custom_llm_provider"') ||
+    head.includes('"model_group"')
+  );
+}

package/src/measured-profile.ts

@@ -70,6 +70,10 @@ export interface MeasuredUsage {
    * call is the same size. It is handed over as the better of two
    * approximations — a measured token share beats a typed call share — and
    * named honestly here so no rendering can call it something it is not.
+   *
+   * The denominator is every input token, **writes included**. Leaving writes
+   * out made a constantly-rewritten cache report a high hit rate, which is the
+   * one workload that most needs to be told otherwise.
    */
   cacheReadShare: number;
   /**
@@ -105,6 +109,7 @@ export function measuredUsage(
   const outputTokens = slices.reduce((sum, row) => sum + row.breakdown.outputTokens, 0);
   const inputTokens = slices.reduce((sum, row) => sum + row.breakdown.inputTokens, 0);
   const cacheReadTokens = slices.reduce((sum, row) => sum + row.breakdown.cacheReadTokens, 0);
+  const cacheWriteTokens = slices.reduce((sum, row) => sum + row.breakdown.cacheWriteTokens, 0);
 
   // The model carrying the most spend. Ties break on the larger call count, so
   // the answer is stable rather than dependent on map ordering.
@@ -127,8 +132,27 @@ export function measuredUsage(
       ? { fromDays: spanDays, factor: SCALE_TO_DAYS / spanDays }
       : null;
 
-  const readShare =
-    inputTokens + cacheReadTokens > 0 ? cacheReadTokens / (inputTokens + cacheReadTokens) : 0;
+  /**
+   * Every input token in the denominator, writes included.
+   *
+   * This was `reads / (input + reads)`, which leaves cache *writes* out of the
+   * total it is a share of. A workload that rewrites its prefix on every call
+   * and reads it back rarely then reports a **high** hit rate: with 100 read
+   * tokens, no plain input and 10,000 written, the old expression answers 100%
+   * where the truth is 1%.
+   *
+   * That is the worst direction for this number to be wrong in. It is handed
+   * to `optimize` as `cacheHitRate`, which decides whether caching is paying
+   * off — so the shape that is burning money on writes was the shape most
+   * likely to be told its cache was working perfectly.
+   *
+   * Writes are input tokens: they are billed at the input rate times the write
+   * multiplier, they arrive in the same `usage` block, and any accounting of
+   * "what share of the input came from cache" that omits them is answering a
+   * narrower question than the one it is named for.
+   */
+  const allInputTokens = inputTokens + cacheReadTokens + cacheWriteTokens;
+  const readShare = allInputTokens > 0 ? cacheReadTokens / allInputTokens : 0;
 
   return {
     profile: {

package/src/schemas.ts

@@ -151,6 +151,42 @@ export const CONTRACT_SCHEMAS: Record<ContractName, Record<string, unknown>> = {
     cannotSay: array,
     unpricedRecords: number,
   }),
+  'routing-measurement': doc('routing-measurement', {
+    slice: object,
+    evaluation: object,
+  }),
+  'example-pruning': doc('example-pruning', {
+    provider: string,
+    model: string,
+    selfAgreement: number,
+    recoverableTokens: number,
+    callsMade: number,
+    contributions: array,
+  }),
+  'store-inventory': doc('store-inventory', {
+    providers: array,
+    totalRecords: number,
+    span: objectOrNull,
+    possiblyDouble: number,
+    unknownVersion: number,
+    totalUsd: number,
+    unreadable: array,
+  }),
+  conformance: doc('conformance', {
+    contract: stringOrNull,
+    because: stringOrNull,
+    problems: array,
+    unavailable: array,
+    records: { type: ['number', 'null'] },
+    conforms: boolean,
+  }),
+  'watch-cycle': doc('watch-cycle', {
+    firedAtMs: number,
+    crossings: array,
+    suppressed: array,
+    abstentions: array,
+    gap: objectOrNull,
+  }),
 };
 
 /** The schema for one named contract — the object `trazum schema` prints. */

package/src/tokenizer.ts

@@ -2,20 +2,22 @@
  * Dependency-free token estimator.
  *
  * This is NOT a real tokenizer: it is a heuristic calibrated per character
- * class. It is built to keep the typical error on ordinary text
- * (English/Spanish, markdown, code) inside ±10%, which is plenty for comparing
- * two versions of the same prompt — but do NOT bill anyone from it.
+ * class, built for comparing two versions of the same prompt — but do NOT bill
+ * anyone from it.
  *
- * **That band is a design target that has not been measured.** It is printed on
- * every report and every dollar figure descends from it, and until
- * `scripts/measure-token-band.mjs` has been run against the official counting
- * endpoint, nothing in this repository establishes that it holds. It is stated
- * as one number for all text, which is a further assumption: the branches below
- * treat CJK, digits and punctuation quite differently from words, and there is
- * no reason those should land on the same accuracy.
+ * **Its accuracy is not one number, and the years it was written as one were
+ * the mistake.** The branches below treat CJK, digits and punctuation quite
+ * differently from words, and there was never a reason those should land on the
+ * same accuracy. Measured over 47 samples against the official counting
+ * endpoint they do not: the worst error is 3.2% on CJK, 5.6% on Latin prose,
+ * 25.1% on code and quoting, and 32.5% on a digit-dominant table. `band.ts`
+ * holds the bands those measurements earn and `bandFor(text)` is what a caller
+ * should print; the constant below is only the widest of them, for callers that
+ * have no text in hand.
  *
- * `test/token-band.test.js` asserts the band per text type as soon as the ground
- * truth exists. Until then it says so rather than passing quietly.
+ * `test/token-band.test.js` holds every sample against the band `bandFor` gives
+ * it, and a sample edited since it was measured fails rather than passing
+ * quietly.
  *
  * For exact numbers use `countTokensAnthropic` (the official token-counting
  * endpoint, which is free) or pass your own `TokenCounter`.
@@ -25,53 +27,47 @@ import { detectTextLanguage } from './language.js';
 import { SAFE_FETCH_INIT, checkedEndpoint } from './net.js';
 
 /**
- * The error band this estimator is published under, as a percentage.
+ * The widest band this estimator is published under, as a percentage.
  *
- * **Measured, not chosen, and it has moved three times.** It was `15` for eight
- * releases as a design target nobody had checked; the first measurement found two
- * of eight samples outside it and it went to `25`; fixing the digit divisor and
- * calibrating per language brought it back to `15`. This is the fourth value and
- * the first one that is comfortably above what the corpus actually shows.
+ * **A history worth keeping, because each value was believed at the time.** It
+ * was `15` for eight releases as a design target nobody had checked; the first
+ * measurement found two of eight samples outside it and it went to `25`; fixing
+ * the digit divisor and calibrating per language brought it back to `15`, and
+ * splitting kana from han — Japanese had been +11.2% and Chinese −3.2% under one
+ * rule — brought it to `10`. Four values, each an improvement, and all four
+ * wrong in the same way.
  *
- * ```
- * worst measured error   6.4%   (code-heavy, which nothing is fitted to)
- * published band        10%
- * ```
+ * **The corpus they were measured on was the corpus they were fitted to.**
+ * Twenty-one samples held thirteen files of Latin prose and exactly one each of
+ * code, numeric and punctuation, and those single files were where the constants
+ * came from. The worst error it could show was 6.4%, and 10 looked like a
+ * comfortable margin over it. Neither figure was about the estimator.
  *
- * **The margin is deliberate and it is not slack.** 6.4 rounded up is 7, and
- * publishing 7 would be a tighter claim than twenty-one samples across six text
- * types can support: the corpus has no Korean, no Arabic, no Cyrillic prose, no
- * mixed-script document, and a seventh text type could easily land at eight. A
- * band that becomes false the first time somebody measures something new is the
- * exact fault this whole exercise was fixing. Overstating the uncertainty is the
- * safe direction for a tool that reports money.
+ * **It is no longer one number, and this one is the widest.** A corpus of
+ * thirteen prose files and one each of code, numeric and punctuation put the
+ * band at 10, and those single files were the set the constants had been fitted
+ * to. Measured on forty-seven, the same estimator is 6% out on prose and 33%
+ * out on a CSV ledger. `bandFor` in `band.ts` is what a caller that has the
+ * text should use.
  *
- * What earned the drop from 15 was **splitting kana from han**. Every CJK
- * character was charged one token, which put Japanese at +11.2% — the worst error
- * anywhere in the corpus — while Chinese sat at −3.2% under the same rule. Kana
- * measure 0.75 tokens per character and han 1.05, and that pair takes the two
- * samples to −1.5% and +1.3%. See `KANA_TOKENS_PER_CHAR`.
- *
- * **Which samples the band rests on matters more than the number.** Eight of the
- * twenty-one had a constant fitted to them — seven Latin divisors and the digit
- * divisor — so their residuals are optimistic by construction. The two worst
- * errors in the corpus, `code-heavy` at 6.4% and `punctuation-heavy` at 5.7%, are
- * fitted to nothing at all, and they are what sets this figure.
- *
- * Exported so every report, README and tool description reads the same number. It
- * was a literal in twenty-four files before this, with the only machine-readable
- * copy in a test, and `token-band.test.js` now fails any file that states a
- * different one.
+ * This stays for the callers that do not have it — a figure covering the whole
+ * catalogue of text, which can only be the worst of them. Widening it from 10
+ * to 33 makes every claim that reads it true where it had been false, at the
+ * cost of understating the estimator on prose. That trade is deliberate: a
+ * caller with no text cannot know which it is holding, and there is only one
+ * safe direction to guess in.
  */
-export const ESTIMATE_ERROR_BAND_PCT = 10;
+export const ESTIMATE_ERROR_BAND_PCT = 33;
 
 /**
  * Whose tokenizer that band was measured against.
  *
  * The number above is not a property of the estimator; it is a property of the
- * estimator **against Claude**. Twenty-one samples, one counting endpoint, one
- * family. Trazum prices seven, and until somebody runs the harness against the
- * others nobody knows what the error is on a GPT or a Gemini prompt.
+ * estimator **against Claude**. Forty-seven samples, one counting endpoint, one
+ * family. Trazum prices seven, and the two that have since been run are far
+ * outside anything published here: DeepSeek 94.5% at worst, Mistral 103.1%. On
+ * a GPT or a Gemini prompt nobody has measured it at all, and
+ * `measuredForeignError` answers null rather than the nearest number.
  *
  * Exported as a name rather than left implicit in a comment because the band
  * had already leaked out of its domain in three places: an advisory that told a
@@ -173,6 +169,34 @@ const DIVISOR_BY_LANGUAGE: Readonly<Record<string, number>> = {
 const KANA_TOKENS_PER_CHAR = 0.75;
 const HAN_TOKENS_PER_CHAR = 1.05;
 
+/**
+ * Hangul, measured at last.
+ *
+ * The comment above used to end *"Hangul keeps the old cost of 1, because
+ * nothing here measures Korean. Guessing it from Japanese would be inventing a
+ * figure."* That was right, and it stayed right for exactly as long as nobody
+ * measured Korean. `cjk-korean.txt` now does: at 1 token per character the
+ * estimate came out **20.0% under** — 232 against 290 — and 1.20 takes it to
+ * -10.0%.
+ *
+ * **1.20 was the one-sample answer and it was wrong by ten points.** A second
+ * Korean sample in a different register — a deployment review against support
+ * copy — put both at -10.6% and -10.0% under it, and solving the constant on
+ * the two together lands on 1.35, where both come out at 0.0%.
+ *
+ * Two independent texts agreeing to within four tenths of a point across the
+ * whole search is what makes this a property of Hangul rather than a fit to a
+ * file. Their residuals move in lockstep at every candidate value: -10.6/-10.0
+ * at 1.20, -3.5/-3.1 at 1.30, 3.2/3.4 at 1.40. One sample cannot show that and
+ * two can, which is the entire argument for measuring more of the thin classes
+ * rather than tuning harder on the thin ones.
+ *
+ * Still in-sample: the honest test is a third Korean prompt, and the corpus
+ * grows one at a time.
+ */
+const HANGUL_TOKENS_PER_CHAR = 1.35;
+const HANGUL = /[가-힯]/;
+
 /**
  * What a prompt gets when the language could not be told.
  *
@@ -248,7 +272,11 @@ export function estimateTokens(text: string): number {
        * where the loop happens to break, not of what the text costs.
        */
       while (i < chars.length && CJK.test(chars[i]!)) {
-        cjkTokens += KANA.test(chars[i]!) ? KANA_TOKENS_PER_CHAR : HAN_TOKENS_PER_CHAR;
+        cjkTokens += KANA.test(chars[i]!)
+          ? KANA_TOKENS_PER_CHAR
+          : HANGUL.test(chars[i]!)
+            ? HANGUL_TOKENS_PER_CHAR
+            : HAN_TOKENS_PER_CHAR;
         i++;
       }
       continue;
@@ -315,7 +343,29 @@ export function estimateTokens(text: string): number {
       total += 1;
       i++;
     } else {
-      total += Math.ceil(n / 2);
+      /**
+       * One token per symbol, not one per two.
+       *
+       * Two was the shipped value and every symbol-dense sample in the corpus
+       * undercounted under it, in the same direction, by amounts that grow with
+       * density: punctuation-heavy -5.7%, code-heavy -6.4%,
+       * punctuation-markup -11.4%, code-sql -25.6%, code-shell -29.8%. Five
+       * samples agreeing on a direction is a shape, not a fit — a merge table
+       * covers `, ` and `. ` and covers `${`, `"$`, `--` and `&#8209;` far less
+       * well, and shell and SQL are made of the second kind.
+       *
+       * Searched over the whole corpus rather than the class it helps, so a
+       * value that fixed code by wrecking prose could not look like a win: at
+       * 1.0 the samples outside the published band drop from five to three and
+       * the mean error from 6.6% to 5.1%, with no sample moved more than a
+       * point in the wrong direction.
+       *
+       * The search wanted to go below 1.0 and was not allowed to. Below one is
+       * more than a token per symbol, which may well be true of SQL — code-sql
+       * is still 24.9% under — but it is a claim about a class this corpus has
+       * two samples of, and two samples do not earn a constant.
+       */
+      total += Math.ceil(n / 1);
     }
   }
 

package/src/types.ts

@@ -61,6 +61,7 @@ export type AdvisoryId =
   | 'cache-prefix-reorder'
   | 'batch-api'
   | 'model-downgrade'
+  | 'tier-signals-conflict'
   | 'output-dominated'
   | 'promo-pricing'
   | 'contradictory-instructions'