@trazum/core 1.9.0 → 1.25.0

package/src/tokenizer.ts

@@ -3,7 +3,7 @@
  *
  * This is NOT a real tokenizer: it is a heuristic calibrated per character
  * class. It is built to keep the typical error on ordinary text
- * (English/Spanish, markdown, code) inside ±15%, which is plenty for comparing
+ * (English/Spanish, markdown, code) inside ±10%, which is plenty for comparing
  * two versions of the same prompt — but do NOT bill anyone from it.
  *
  * **That band is a design target that has not been measured.** It is printed on
@@ -27,38 +27,67 @@ import { SAFE_FETCH_INIT, checkedEndpoint } from './net.js';
 /**
  * The error band this estimator is published under, as a percentage.
  *
- * **Measured, not chosen.** It was `15` for eight releases — a design target
- * nobody had checked — and the first run of `scripts/measure-token-band.mjs`
- * against the official counting endpoint found two of eight samples outside it,
- * both underestimating. Underestimating tokens means under-reporting cost, which
- * is the flattering direction and the worst one for this tool.
+ * **Measured, not chosen, and it has moved three times.** It was `15` for eight
+ * releases as a design target nobody had checked; the first measurement found two
+ * of eight samples outside it and it went to `25`; fixing the digit divisor and
+ * calibrating per language brought it back to `15`. This is the fourth value and
+ * the first one that is comfortably above what the corpus actually shows.
  *
- * `15` is the measured worst case rounded up, by the same rule that briefly made
- * it 25: the corpus now tops out at 11.2%, on Japanese. It landing back on the
- * number that was a guess for eight releases is a coincidence and not a
- * restoration — that 15 bounded nothing, and this one bounds eleven measured
- * samples across four languages and six text types.
+ * ```
+ * worst measured error   6.4%   (code-heavy, which nothing is fitted to)
+ * published band        10%
+ * ```
  *
- * **Read one caveat before trusting it.** Four of those samples are the Latin
- * languages whose divisors in `DIVISOR_BY_LANGUAGE` were calibrated on one or two
- * samples each, so their residuals are in-sample and optimistic by construction.
- * The band is set by the seven samples nothing was fitted to — worst 11.2% — and
- * the honest test of it is the next held-out sample in Spanish, French or German.
- * The corpus grows one sample at a time now, so that test is cheap to run.
+ * **The margin is deliberate and it is not slack.** 6.4 rounded up is 7, and
+ * publishing 7 would be a tighter claim than twenty-one samples across six text
+ * types can support: the corpus has no Korean, no Arabic, no Cyrillic prose, no
+ * mixed-script document, and a seventh text type could easily land at eight. A
+ * band that becomes false the first time somebody measures something new is the
+ * exact fault this whole exercise was fixing. Overstating the uncertainty is the
+ * safe direction for a tool that reports money.
  *
- * **What got it there was not accents.** A Spanish sample with zero accented
- * characters measured -22.9% against -22.1% for accented Spanish, which killed
- * diacritics as a signal. Languages differ in how many tokens their words cost —
- * English 3.44 characters per token, German 2.02 — so the estimator detects the
- * language and divides accordingly. See `language.ts`.
+ * What earned the drop from 15 was **splitting kana from han**. Every CJK
+ * character was charged one token, which put Japanese at +11.2% — the worst error
+ * anywhere in the corpus — while Chinese sat at −3.2% under the same rule. Kana
+ * measure 0.75 tokens per character and han 1.05, and that pair takes the two
+ * samples to −1.5% and +1.3%. See `KANA_TOKENS_PER_CHAR`.
  *
- * Exported so every report, README and tool description reads the same number.
- * It was a literal in twenty-four files before this, with the only machine-
- * readable copy in a test.
+ * **Which samples the band rests on matters more than the number.** Eight of the
+ * twenty-one had a constant fitted to them — seven Latin divisors and the digit
+ * divisor — so their residuals are optimistic by construction. The two worst
+ * errors in the corpus, `code-heavy` at 6.4% and `punctuation-heavy` at 5.7%, are
+ * fitted to nothing at all, and they are what sets this figure.
+ *
+ * Exported so every report, README and tool description reads the same number. It
+ * was a literal in twenty-four files before this, with the only machine-readable
+ * copy in a test, and `token-band.test.js` now fails any file that states a
+ * different one.
  */
-export const ESTIMATE_ERROR_BAND_PCT = 15;
+export const ESTIMATE_ERROR_BAND_PCT = 10;
 
 const CJK = /[぀-ヿ㐀-䶿一-鿿가-힯]/;
+
+/**
+ * Kana, separated from the rest of CJK because they do not cost the same.
+ *
+ * **This was the largest error left in the corpus.** Every CJK character was
+ * charged one token, and measured against the counting endpoint that put Japanese
+ * at **+11.2%** — the worst figure anywhere in twenty-one samples — while Chinese
+ * came out at −3.2% under the identical rule. One constant cannot be right for
+ * both, and the reason is visible in the samples: the Japanese one is 58% kana and
+ * the Chinese one is 0%.
+ *
+ * Kana are a small syllabary that appears in every sentence, so the merge table
+ * covers runs of them and several characters share a token. Han are tens of
+ * thousands of rare characters; a merge table cannot cover them and they cost
+ * about one each, sometimes more.
+ *
+ * The signal needs no detector. A character is kana or it is not, and the two
+ * samples separate perfectly — 58.3% against 0.00% — so this is a property of the
+ * character rather than a guess about the document. That is the difference between
+ * this and `language.ts`, which has to decide and is allowed to refuse.
+ */
+const KANA = /[぀-ヿ]/;
 const LETTER = /[A-Za-zÀ-ɏͰ-ϿЀ-ӿ]/;
 const DIGIT = /[0-9]/;
 
@@ -95,6 +124,27 @@ const DIVISOR_BY_LANGUAGE: Readonly<Record<string, number>> = {
   nl: 2.65,
 };
 
+/**
+ * Tokens per character for the two halves of CJK, measured.
+ *
+ * `0.75` and `1.05` come from a search over the two CJK samples in
+ * `test/fixtures/token-ground-truth.json`, and they take that pair from
+ * +11.2% / −3.2% to −1.5% / +1.3%.
+ *
+ * **Two samples fitted two constants, so those residuals are in-sample and
+ * optimistic by construction** — the same caveat the Latin divisors carry, stated
+ * for the same reason. What makes them worth having anyway is the size of the
+ * error they replace and the fact that they move in opposite directions: a single
+ * constant could not have been within four points of both, whatever it was set to.
+ * The honest test is a third CJK sample, and the corpus grows one at a time.
+ *
+ * Hangul keeps the old cost of 1, because nothing here measures Korean. Guessing
+ * it from Japanese would be inventing a figure — the two scripts have nothing in
+ * common that would make one predict the other.
+ */
+const KANA_TOKENS_PER_CHAR = 0.75;
+const HAN_TOKENS_PER_CHAR = 1.05;
+
 /**
  * What a prompt gets when the language could not be told.
  *
@@ -133,6 +183,11 @@ export function estimateTokens(text: string): number {
   const divisor = (language === null ? undefined : DIVISOR_BY_LANGUAGE[language]) ?? DEFAULT_DIVISOR;
 
   let total = 0;
+  /**
+   * CJK is summed separately because it is the one class counted in fractions of
+   * a token. Everything else is whole tokens by the character class it belongs to.
+   */
+  let cjkTokens = 0;
   let i = 0;
   const chars = Array.from(text);
 
@@ -155,12 +210,19 @@ export function estimateTokens(text: string): number {
     }
 
     if (CJK.test(ch)) {
-      let n = 0;
+      /**
+       * Accumulated as a fraction and rounded once, at the end.
+       *
+       * Rounding up per run was the first attempt and it was wrong by five
+       * points. Ordinary Japanese alternates kana and han inside every sentence,
+       * so the runs are short and there are many of them — and a `Math.ceil` per
+       * run charges most of a token for each boundary. That is an artefact of
+       * where the loop happens to break, not of what the text costs.
+       */
       while (i < chars.length && CJK.test(chars[i]!)) {
-        n++;
+        cjkTokens += KANA.test(chars[i]!) ? KANA_TOKENS_PER_CHAR : HAN_TOKENS_PER_CHAR;
         i++;
       }
-      total += n;
       continue;
     }
 
@@ -229,7 +291,8 @@ export function estimateTokens(text: string): number {
     }
   }
 
-  return total;
+  // Rounded once, over the whole document, rather than per run — see the CJK branch.
+  return total + Math.ceil(cjkTokens);
 }
 
 /** Asynchronous token counter, for remote sources. */

package/src/ttl-fit.ts

@@ -0,0 +1,251 @@
+import { effectivePricing, multipliersFor } from './pricing.js';
+import { UNLABELLED } from './usage.js';
+import type { PricingCatalogue } from './pricing.js';
+import type { UsageRecord } from './usage.js';
+
+/**
+ * Does the cache TTL fit how fast the turns actually arrive?
+ *
+ * ## The mechanism nothing else can see
+ *
+ * A cache entry lives 5 minutes, or an hour at twice the write price. Whether
+ * either is the right choice depends on one number the bill never shows: **how
+ * long the workload waits between turns.** A support agent whose users answer in
+ * nine minutes writes a 5-minute entry on every turn and reads it back on none of
+ * them — every write expires unread, which from the bill is indistinguishable
+ * from any other losing cache. `cacheEconomics` can say *that* money was lost;
+ * only the clock can say *why*, and the why decides the fix: the 1-hour TTL, or
+ * caching switched off.
+ *
+ * The opposite mistake is quieter and this is the only place it appears at all:
+ * turns arriving seconds apart, written at the 1-hour rate. Those writes work —
+ * the verdict above reads `paid-off` — and every one of them pays 2x input for
+ * endurance the workload never uses. **Switching them to the 5-minute TTL is the
+ * one exact saving in this module**: the same tokens at 1.25x instead of 2x,
+ * which is the same-tokens-different-rate arithmetic `cacheEconomics` already
+ * draws the counterfactual line at.
+ *
+ * ## What it measures, and how it stays honest
+ *
+ * The gap between consecutive turns **of the same conversation**, from the
+ * recorded clock — sessions are what a cache entry actually serves, and gaps
+ * between unrelated calls of a label say nothing about whether *this*
+ * conversation's next turn found the entry alive. Timestamps are sorted within
+ * each session before differencing, so the measurement is independent of the
+ * order of the log — the property the conversation tracker had to learn the
+ * hard way.
+ *
+ * The reported number is the **median** gap, named as such: a median survives
+ * the overnight gap between a user's Tuesday and Wednesday in a way a mean does
+ * not, and a verdict hung on a mean would flip on one lunch break.
+ *
+ * When the log did not record which TTL the writes used, the gap can sit where
+ * the verdict depends on the answer — over 5 minutes and under an hour survives
+ * one TTL and not the other. That is reported as `unsettled`, the same refusal
+ * `cacheEconomics` makes for the same missing field, and never resolved in the
+ * flattering direction.
+ *
+ * The session key is used to group turns and never leaves this module, like
+ * everywhere else the field is touched.
+ */
+
+/** Cache-entry lifetimes, in milliseconds. Anthropic's two published TTLs. */
+export const TTL_5M_MS = 5 * 60 * 1000;
+export const TTL_1H_MS = 60 * 60 * 1000;
+
+export type TtlFitVerdict =
+  /** The median gap outlives the entry: writes expire before the next turn. */
+  | 'expires-before-reuse'
+  /** 1-hour writes on gaps inside the 5-minute window: paying 2x for nothing. */
+  | 'overlong-ttl'
+  /** The TTL the log did not record decides the verdict, so nothing does. */
+  | 'unsettled'
+  /** The entry outlives the gap at the TTL the writes actually used. */
+  | 'fits';
+
+export interface CacheTtlFit {
+  label: string;
+  model: string;
+  modelName: string;
+  /** Conversations with at least two timestamped turns. Never which ones. */
+  sessions: number;
+  /** Gaps measured across them. */
+  gaps: number;
+  medianGapMs: number;
+  /** Write tokens the log said were 5-minute entries. */
+  write5mTokens: number;
+  /** Write tokens the log said were 1-hour entries. */
+  write1hTokens: number;
+  /** Write tokens whose TTL the log did not record. */
+  assumedTtlTokens: number;
+  verdict: TtlFitVerdict;
+  /**
+   * What the 1-hour writes would save at the 5-minute rate, when the gaps show
+   * the hour is never needed. Exact — the same tokens at 1.25x instead of 2x,
+   * at the model's own input rate — and zero for every other verdict.
+   */
+  overpayUsd: number;
+}
+
+export interface TtlFitOptions {
+  catalogue: PricingCatalogue;
+  /** Date the prices are read at, so a promotional rate resolves the same way. */
+  on?: Date;
+}
+
+export interface TtlFitTracker {
+  /** Feed one parsed record. */
+  add(record: UsageRecord): void;
+  /** The finished measurement. */
+  finish(): CacheTtlFit[];
+}
+
+interface Slice {
+  write5mTokens: number;
+  write1hTokens: number;
+  assumedTtlTokens: number;
+  /** Timestamps per session, sorted only at the end. */
+  sessions: Map<string, number[]>;
+}
+
+const median = (sorted: number[]): number => {
+  const mid = Math.floor(sorted.length / 2);
+  return sorted.length % 2 === 1 ? sorted[mid]! : (sorted[mid - 1]! + sorted[mid]!) / 2;
+};
+
+/**
+ * An accumulator, like the conversation tracker and for the same reason: a
+ * usage log is measured in megabytes and `profileUsage` makes one pass. What
+ * this holds is one number per timestamped call that belongs to a session,
+ * which is the minimum the gaps can be computed from at all.
+ */
+export function createTtlFitTracker(options: TtlFitOptions): TtlFitTracker {
+  const { catalogue, on = new Date() } = options;
+  const slices = new Map<string, Slice>();
+
+  const add = (record: UsageRecord): void => {
+    // An unpriced model has no rates to judge a TTL against, and contributes no
+    // dollars anywhere else either.
+    if (!catalogue.byId.has(record.model)) return;
+
+    const writes =
+      record.cacheWrite5mTokens + record.cacheWrite1hTokens;
+    const hasClockedTurn = record.session !== null && record.ts !== null;
+    // A slice exists once it writes to the cache or can contribute a gap;
+    // everything else has nothing to say here.
+    if (writes === 0 && !hasClockedTurn) return;
+
+    const key = `${record.label ?? UNLABELLED}\n${record.model}`;
+    let slice = slices.get(key);
+    if (!slice) {
+      slice = { write5mTokens: 0, write1hTokens: 0, assumedTtlTokens: 0, sessions: new Map() };
+      slices.set(key, slice);
+    }
+
+    if (record.writeTtlKnown) {
+      slice.write5mTokens += record.cacheWrite5mTokens;
+      slice.write1hTokens += record.cacheWrite1hTokens;
+    } else {
+      // The flat count sits in the 5m bucket by pricing convention; here it is
+      // kept apart, because the whole question is which TTL it really was.
+      slice.assumedTtlTokens += record.cacheWrite5mTokens;
+    }
+
+    if (hasClockedTurn) {
+      const turns = slice.sessions.get(record.session!);
+      if (turns) turns.push(record.ts!);
+      else slice.sessions.set(record.session!, [record.ts!]);
+    }
+  };
+
+  const finish = (): CacheTtlFit[] => {
+    const out: CacheTtlFit[] = [];
+
+    for (const [key, slice] of slices) {
+      const writes = slice.write5mTokens + slice.write1hTokens + slice.assumedTtlTokens;
+      // No writes means no TTL to judge. Gaps alone are the conversation
+      // tracker's business, not this module's.
+      if (writes === 0) continue;
+
+      const gaps: number[] = [];
+      let sessions = 0;
+      for (const turns of slice.sessions.values()) {
+        if (turns.length < 2) continue;
+        sessions += 1;
+        turns.sort((a, b) => a - b);
+        for (let i = 1; i < turns.length; i += 1) gaps.push(turns[i]! - turns[i - 1]!);
+      }
+      // Writes with no measurable gap: the caller reports "could not be
+      // measured" from the absence of a row, the same way missing sessions
+      // read everywhere else.
+      if (gaps.length === 0) continue;
+
+      const split = key.indexOf('\n');
+      const modelId = key.slice(split + 1);
+      const model = catalogue.byId.get(modelId)!;
+
+      gaps.sort((a, b) => a - b);
+      const gap = median(gaps);
+
+      /**
+       * First match wins, ordered so that a certain failure outranks an
+       * uncertain one and money comes last: writes that expire are broken at
+       * any price, and only writes that demonstrably work can be overpaying.
+       */
+      let verdict: TtlFitVerdict;
+      if (slice.write5mTokens > 0 && gap > TTL_5M_MS) verdict = 'expires-before-reuse';
+      else if (slice.write1hTokens > 0 && gap > TTL_1H_MS) verdict = 'expires-before-reuse';
+      else if (slice.assumedTtlTokens > 0 && gap > TTL_1H_MS) verdict = 'expires-before-reuse';
+      else if (slice.write1hTokens > 0 && gap <= TTL_5M_MS) verdict = 'overlong-ttl';
+      else if (slice.assumedTtlTokens > 0 && gap > TTL_5M_MS) verdict = 'unsettled';
+      else verdict = 'fits';
+
+      const { inputPerMTok } = effectivePricing(model, on);
+      const rates = multipliersFor(model);
+      const overpayUsd =
+        verdict === 'overlong-ttl'
+          ? (slice.write1hTokens / 1_000_000) *
+            inputPerMTok *
+            (rates.cacheWrite1h - rates.cacheWrite5m)
+          : 0;
+
+      out.push({
+        label: key.slice(0, split),
+        model: modelId,
+        modelName: model.displayName,
+        sessions,
+        gaps: gaps.length,
+        medianGapMs: gap,
+        write5mTokens: slice.write5mTokens,
+        write1hTokens: slice.write1hTokens,
+        assumedTtlTokens: slice.assumedTtlTokens,
+        verdict,
+        overpayUsd,
+      });
+    }
+
+    // Broken first, then overpaying by money, then the rest — the order
+    // somebody would act in.
+    const rank: Record<TtlFitVerdict, number> = {
+      'expires-before-reuse': 0,
+      'overlong-ttl': 1,
+      unsettled: 2,
+      fits: 3,
+    };
+    return out.sort(
+      (a, b) => rank[a.verdict] - rank[b.verdict] || b.overpayUsd - a.overpayUsd,
+    );
+  };
+
+  return { add, finish };
+}
+
+/** The same measurement over a list, for a caller holding one already. */
+export function cacheTtlFit(
+  records: readonly UsageRecord[],
+  options: TtlFitOptions,
+): CacheTtlFit[] {
+  const tracker = createTtlFitTracker(options);
+  for (const record of records) tracker.add(record);
+  return tracker.finish();
+}

package/src/types.ts

@@ -44,6 +44,7 @@ export type AdvisorySeverity = 'info' | 'opportunity' | 'warning';
 /** Every advisory the core can emit. */
 export type AdvisoryId =
   | 'context-overflow'
+  | 'context-near-limit'
   | 'prompt-caching'
   | 'prompt-caching-not-worth-it'
   | 'below-cache-minimum'