@trazum/core 1.8.0 → 1.10.0

package/src/advisories.ts

@@ -7,7 +7,7 @@ import type { PricingCatalogue } from './pricing.js';
 import { formatUsd } from './savings.js';
 import { analyzeExamples, findContradictions, findMovableSchema,
   findRestatedFormat } from './structure.js';
-import { estimateTokens } from './tokenizer.js';
+import { ESTIMATE_ERROR_BAND_PCT, estimateTokens } from './tokenizer.js';
 import type { Advisory, ModelPricing, TokenCounter, UsageProfile } from './types.js';
 
 function countSignals(haystack: string, signals: readonly string[]): number {
@@ -125,7 +125,25 @@ export function buildAdvisories(
   const monthlyOutputUsd =
     (usage.avgOutputTokens / 1_000_000) * outputPerMTok * usage.callsPerMonth * batchFactor;
 
-  // --- Context window ---
+  /**
+   * --- Context window ---
+   *
+   * The third place an estimate was compared against a hard threshold and the
+   * answer stated as fact, after `cache-prefix-reorder` and `prompt-caching`. This
+   * one has no dollar figure and is the most absolute of the three: **"The call
+   * will fail."**
+   *
+   * With a ±10% band it fails in both directions. An estimated 1,050,000 tokens
+   * against a 1,000,000 window can truly be 945,000 — the call succeeds and the
+   * reader has been sent to split a prompt that fitted. And an estimated 990,000
+   * can truly be 1,089,000, which does not fit, and nothing said anything at all.
+   *
+   * The silent direction is the worse one. A prompt over the window fails
+   * outright rather than degrading, so there is no partial result to notice.
+   */
+  const estimated = count === estimateTokens;
+  const band = ESTIMATE_ERROR_BAND_PCT / 100;
+
   if (tokensAfter > model.contextWindow) {
     advisories.push({
       id: 'context-overflow',
@@ -134,6 +152,20 @@ export function buildAdvisories(
         tokens: tokensAfter,
         modelName: model.displayName,
         contextWindow: model.contextWindow,
+        // Only an estimate can be uncertain. A caller who counted exactly is told
+        // the call fails, because it does.
+        uncertain: estimated && tokensAfter * (1 - band) <= model.contextWindow,
+      }),
+      estimatedMonthlyUsd: null,
+    });
+  } else if (estimated && tokensAfter * (1 + band) > model.contextWindow) {
+    advisories.push({
+      id: 'context-near-limit',
+      severity: 'warning',
+      ...t.advisories.contextNearLimit({
+        tokens: tokensAfter,
+        modelName: model.displayName,
+        contextWindow: model.contextWindow,
       }),
       estimatedMonthlyUsd: null,
     });
@@ -185,6 +217,22 @@ export function buildAdvisories(
             readPct: Math.round(rates.cacheRead * 100),
             writePct: Math.round(rates.cacheWrite5m * 100),
             explicit: (model.caching ?? 'explicit') === 'explicit',
+            /**
+             * The mirror of `couldReachMinimum` on `below-cache-minimum`, and the
+             * asymmetry between them was a real gap: that one hedged an estimate
+             * landing just *under* the threshold, while this one promised money on
+             * an estimate landing just *over* it. With a ±10% band an estimated
+             * 528-token prefix can truly be 475, and then nothing caches at all.
+             *
+             * The cautionary direction is the one that needed it, because this is
+             * the side with a dollar figure attached. Only when the number is an
+             * estimate: a caller who supplied their own counter has an
+             * authoritative prefix and hedging it would push them toward a check
+             * they have already done.
+             */
+            nearMinimum:
+              count === estimateTokens &&
+              cache.stablePrefixTokens * (1 - ESTIMATE_ERROR_BAND_PCT / 100) < minTokens,
           }),
           estimatedMonthlyUsd: saving,
         });
@@ -207,17 +255,65 @@ export function buildAdvisories(
           prefixTokens: cache.stablePrefixTokens,
           totalTokens: tokensAfter,
           mentionLowerMinimum: minTokens > 512,
+          /**
+           * Only when the number is an estimate and near the line.
+           *
+           * `count` defaults to `estimateTokens`; a caller who supplied their own
+           * counter — `--exact-tokens`, or the official endpoint — gets an
+           * authoritative number and no hedge, because hedging a measured figure
+           * is its own kind of dishonesty.
+           */
+          couldReachMinimum:
+            count === estimateTokens &&
+            cache.stablePrefixTokens * (1 + ESTIMATE_ERROR_BAND_PCT / 100) >= minTokens,
         }),
         estimatedMonthlyUsd: null,
       });
     }
 
-    // Stable content placed AFTER the first placeholder: never cached today,
-    // but moving it in front would make it cacheable.
+    /**
+     * Stable content placed AFTER the first placeholder: never cached today, and
+     * cacheable if it moves in front.
+     *
+     * **The prefix it would produce has to clear the minimum, and it did not used
+     * to be checked.** That was a money figure in the flattering direction, which
+     * is the one fault this file exists to avoid. On a 306-token support prompt
+     * against Claude Opus 5's 512-token minimum, the best prefix a rearrangement
+     * can build is 302 — so nothing caches, and the advisory offered $48.67 a
+     * month that cannot be collected.
+     *
+     * Worse, it said so in the same report as `below-cache-minimum`, which was
+     * telling the reader caching would not work here at all. Two advisories
+     * contradicting each other, and the one with a dollar sign winning the
+     * argument.
+     *
+     * `reorderForCache` already refused these prompts for exactly this reason, so
+     * the tool's advice and its action disagreed: follow the advice, run
+     * `--reorder`, and watch nothing happen.
+     */
+    /**
+     * The best prefix any rearrangement could build, compared strictly.
+     *
+     * **No band hedge here, and that was tried first.** Widening the comparison by
+     * ±10% — on the same reasoning that makes `below-cache-minimum` hedge near the
+     * line — opened a window between 466 and 512 tokens where this advisory
+     * offered a saving and `reorderForCache` refused to perform it. That is the
+     * fault being fixed, reintroduced one layer up, and a test caught it.
+     *
+     * The near-the-line case is already handled and in the right place:
+     * `below-cache-minimum` says the estimate is close to the threshold and names
+     * `--exact-tokens`. Settle the number and both this advisory and the command
+     * work from the same certainty. Two components disagreeing is worse than one
+     * of them being briefly quiet.
+     */
+    const reorderedPrefix = cache.stablePrefixTokens + cache.staticTokensAfter;
+    const reachableAfterReorder = reorderedPrefix >= minTokens;
+
     if (
       cache.firstPlaceholder &&
       cache.staticTokensAfter >= 200 &&
-      cache.staticTokensAfter >= tokensAfter * 0.3
+      cache.staticTokensAfter >= tokensAfter * 0.3 &&
+      reachableAfterReorder
     ) {
       const movableShare = tokensAfter > 0 ? cache.staticTokensAfter / tokensAfter : 0;
       const saving = monthlyInputUsd * movableShare * Math.max(0, 1 - factor);
@@ -228,6 +324,13 @@ export function buildAdvisories(
           staticTokensAfter: cache.staticTokensAfter,
           sharePct: Math.round(movableShare * 100),
           placeholder: cache.firstPlaceholder,
+          /**
+           * Trazum can do this, and until now it told you to do it by hand.
+           * `reorderForCache` moves whole blocks, refuses any block carrying a
+           * backward reference, and refuses everything after one — so the command
+           * is the safe way to attempt what the prose was describing.
+           */
+          command: 'trazum optimize <file> --reorder',
         }),
         estimatedMonthlyUsd: saving > 0 ? saving : null,
       });

package/src/i18n/en.ts

@@ -91,9 +91,18 @@ export const en: CoreMessages = {
   },
 
   advisories: {
-    contextOverflow: ({ tokens, modelName, contextWindow }) => ({
-      title: 'The prompt does not fit in the context window',
-      detail: `The optimised prompt is ~${n(tokens)} tokens and ${modelName} accepts ${n(contextWindow)}. The call will fail: split the content or move to a model with a larger window.`,
+    contextOverflow: ({ tokens, modelName, contextWindow, uncertain }) => ({
+      title: uncertain
+        ? 'The prompt probably does not fit in the context window'
+        : 'The prompt does not fit in the context window',
+      detail: uncertain
+        ? `The optimised prompt is ~${n(tokens)} tokens against ${modelName}'s ${n(contextWindow)}. That count is an estimate and it is close to the line, so the call will probably fail but might not — settle it with --exact-tokens before rewriting anything. The counting endpoint is free. If it does exceed the window, split the content or move to a model with a larger one.`
+        : `The optimised prompt is ~${n(tokens)} tokens and ${modelName} accepts ${n(contextWindow)}. The call will fail: split the content or move to a model with a larger window.`,
+    }),
+
+    contextNearLimit: ({ tokens, modelName, contextWindow }) => ({
+      title: 'The prompt may not fit in the context window',
+      detail: `The optimised prompt is ~${n(tokens)} tokens against ${modelName}'s ${n(contextWindow)}, which fits — but that count is an estimate and its error range reaches past the window, so the real prompt may not. A call that exceeds the window fails outright rather than degrading, and nothing else here warns about it. Confirm with --exact-tokens; the counting endpoint is free.`,
     }),
 
     promptCaching: ({
@@ -106,6 +115,7 @@ export const en: CoreMessages = {
       readPct,
       writePct,
       explicit,
+      nearMinimum,
     }) => {
       const scope = placeholder
         ? `The stable prefix — everything before the first placeholder ${placeholder} — is ~${n(prefixTokens)} of the prompt's ${n(totalTokens)} tokens, and clears ${modelName}'s ${n(minTokens)}-token cacheable minimum.`
@@ -113,9 +123,12 @@ export const en: CoreMessages = {
       const how = explicit
         ? 'Put the cache marker at the end of the stable prefix: any byte that changes before the cut invalidates everything after it.'
         : `${modelName} caches automatically above its minimum, so there is nothing to set — but the same rule applies: any byte that changes before the cut invalidates everything after it.`;
+      const hedge = nearMinimum
+        ? ` One caveat on the figure: that prefix count is an estimate and it is close to the line, so the real one may be below the ${n(minTokens)}-token minimum — in which case nothing caches and this saving is not there. Settle it with --exact-tokens before budgeting from it. The counting endpoint is free.`
+        : '';
       return {
         title: 'Turn on prompt caching for the stable prefix',
-        detail: `${scope} At a ${hitRatePct}% hit rate, a cache read costs ${readPct}% of the input price and a write costs ${writePct}%. ${how}`,
+        detail: `${scope} At a ${hitRatePct}% hit rate, a cache read costs ${readPct}% of the input price and a write costs ${writePct}%. ${how}${hedge}`,
       };
     },
 
@@ -132,6 +145,7 @@ export const en: CoreMessages = {
       prefixTokens,
       totalTokens,
       mentionLowerMinimum,
+      couldReachMinimum,
     }) => {
       const reason = placeholder
         ? `here the first variable placeholder (${placeholder}) shows up at ~${n(prefixTokens)} tokens and only what precedes it can be cached`
@@ -142,13 +156,16 @@ export const en: CoreMessages = {
           `${modelName} needs at least ${n(minTokens)} prefix tokens to cache; ${reason}. Setting cache_control will not error, it simply will not cache.` +
           (mentionLowerMinimum
             ? ' Claude Opus 5 lowers that minimum to 512 tokens, so short prompts that miss here would cache there.'
+            : '') +
+          (couldReachMinimum
+            ? ' That prefix count is an estimate and it is close to the line, so the real one may already be above it — check with --exact-tokens before deciding this is not available to you. The counting endpoint is free.'
             : ''),
       };
     },
 
-    cachePrefixReorder: ({ staticTokensAfter, sharePct, placeholder }) => ({
+    cachePrefixReorder: ({ staticTokensAfter, sharePct, placeholder, command }) => ({
       title: 'Move the stable instructions ahead of the first placeholder',
-      detail: `About ~${n(staticTokensAfter)} tokens of stable content (${sharePct}% of the prompt) sit after the first variable placeholder ${placeholder}, so today they never get cached. Reorder the template — fixed instructions and context first, placeholders last — and that content starts being read from cache at 10% of the price. Check that reordering does not change what the prompt asks for.`,
+      detail: `About ~${n(staticTokensAfter)} tokens of stable content (${sharePct}% of the prompt) sit after the first variable placeholder ${placeholder}, so today they never get cached. Fixed instructions and context first, placeholders last, and that content starts being read from cache at 10% of the price. Run \`${command}\` to attempt it: whole blocks only, and it refuses to move anything that refers back to earlier text. Read the diff — order carries meaning, and "summarise the text above" is nonsense in front of the text it points at.`,
     }),
 
     batchApi: () => ({

package/src/i18n/es.ts

@@ -91,9 +91,18 @@ export const es: CoreMessages = {
   },
 
   advisories: {
-    contextOverflow: ({ tokens, modelName, contextWindow }) => ({
-      title: 'El prompt no cabe en la ventana de contexto',
-      detail: `El prompt optimizado ocupa ~${n(tokens)} tokens y ${modelName} admite ${n(contextWindow)}. La llamada fallará: divide el contenido o cambia a un modelo con ventana mayor.`,
+    contextOverflow: ({ tokens, modelName, contextWindow, uncertain }) => ({
+      title: uncertain
+        ? 'El prompt probablemente no cabe en la ventana de contexto'
+        : 'El prompt no cabe en la ventana de contexto',
+      detail: uncertain
+        ? `El prompt optimizado ocupa ~${n(tokens)} tokens frente a los ${n(contextWindow)} de ${modelName}. Ese recuento es una estimación y está cerca del límite, así que la llamada fallará probablemente, pero puede que no —confírmalo con --exact-tokens antes de reescribir nada. El endpoint de conteo es gratis. Si de verdad se pasa, divide el contenido o cambia a un modelo con ventana mayor.`
+        : `El prompt optimizado ocupa ~${n(tokens)} tokens y ${modelName} admite ${n(contextWindow)}. La llamada fallará: divide el contenido o cambia a un modelo con ventana mayor.`,
+    }),
+
+    contextNearLimit: ({ tokens, modelName, contextWindow }) => ({
+      title: 'El prompt puede no caber en la ventana de contexto',
+      detail: `El prompt optimizado ocupa ~${n(tokens)} tokens frente a los ${n(contextWindow)} de ${modelName}, así que cabe —pero ese recuento es una estimación y su margen de error se pasa de la ventana, así que el prompt real puede no caber. Una llamada que excede la ventana falla del todo en lugar de degradarse, y nada más aquí avisa de eso. Confírmalo con --exact-tokens; el endpoint de conteo es gratis.`,
     }),
 
     promptCaching: ({
@@ -106,6 +115,7 @@ export const es: CoreMessages = {
       readPct,
       writePct,
       explicit,
+      nearMinimum,
     }) => {
       const scope = placeholder
         ? `El prefijo estable —lo anterior al primer marcador ${placeholder}— son ~${n(prefixTokens)} de los ${n(totalTokens)} tokens del prompt, y supera el mínimo cacheable de ${n(minTokens)} de ${modelName}.`
@@ -113,9 +123,12 @@ export const es: CoreMessages = {
       const how = explicit
         ? 'Coloca el marcador de caché al final del prefijo estable: cualquier byte que cambie antes del corte invalida todo lo que va detrás.'
         : `${modelName} cachea automáticamente por encima de su mínimo, así que no hay nada que activar; pero la regla es la misma: cualquier byte que cambie antes del corte invalida todo lo que va detrás.`;
+      const hedge = nearMinimum
+        ? ` Un aviso sobre la cifra: ese recuento del prefijo es una estimación y está cerca del límite, así que el real puede quedar por debajo del mínimo de ${n(minTokens)} tokens —y entonces no se cachea nada y este ahorro no existe. Confírmalo con --exact-tokens antes de presupuestar sobre él. El endpoint de conteo es gratis.`
+        : '';
       return {
         title: 'Activa prompt caching en el prefijo estable',
-        detail: `${scope} Con una tasa de acierto del ${hitRatePct}%, la lectura de caché cuesta un ${readPct}% del precio de entrada y la escritura un ${writePct}%. ${how}`,
+        detail: `${scope} Con una tasa de acierto del ${hitRatePct}%, la lectura de caché cuesta un ${readPct}% del precio de entrada y la escritura un ${writePct}%. ${how}${hedge}`,
       };
     },
 
@@ -132,6 +145,7 @@ export const es: CoreMessages = {
       prefixTokens,
       totalTokens,
       mentionLowerMinimum,
+      couldReachMinimum,
     }) => {
       const reason = placeholder
         ? `aquí el primer marcador variable (${placeholder}) aparece a los ~${n(prefixTokens)} tokens y solo lo anterior puede cachearse`
@@ -142,13 +156,16 @@ export const es: CoreMessages = {
           `${modelName} necesita al menos ${n(minTokens)} tokens de prefijo para cachear; ${reason}. Marcar cache_control no dará error, simplemente no cacheará.` +
           (mentionLowerMinimum
             ? ' Claude Opus 5 baja ese mínimo a 512 tokens, así que prompts cortos que aquí no cachean, allí sí.'
+            : '') +
+          (couldReachMinimum
+            ? ' Ese recuento del prefijo es una estimación y está cerca del límite, así que el real puede estar ya por encima: compruébalo con --exact-tokens antes de dar esto por perdido. El endpoint de conteo es gratis.'
             : ''),
       };
     },
 
-    cachePrefixReorder: ({ staticTokensAfter, sharePct, placeholder }) => ({
+    cachePrefixReorder: ({ staticTokensAfter, sharePct, placeholder, command }) => ({
       title: 'Mueve las instrucciones estables antes del primer marcador',
-      detail: `Unos ~${n(staticTokensAfter)} tokens de contenido estable (el ${sharePct}% del prompt) están después del primer marcador variable ${placeholder}, así que hoy no se cachean nunca. Reordena la plantilla —instrucciones y contexto fijos primero, marcadores al final— y ese contenido pasa a leerse de caché al 10% del precio. Revisa que la reordenación no cambie el sentido del prompt.`,
+      detail: `Unos ~${n(staticTokensAfter)} tokens de contenido estable (el ${sharePct}% del prompt) están después del primer marcador variable ${placeholder}, así que hoy no se cachean nunca. Instrucciones y contexto fijos primero, marcadores al final, y ese contenido empieza a leerse de caché al 10% del precio. Ejecuta \`${command}\` para intentarlo: solo mueve bloques completos y se niega a mover cualquiera que se refiera a texto anterior. Lee el diff —el orden significa algo, y «resume el texto de arriba» no tiene sentido delante del texto al que apunta.`,
     }),
 
     batchApi: () => ({

package/src/i18n/types.ts

@@ -51,6 +51,17 @@ export interface RuleCopy {
 // --------------------------------------------------------------------------
 
 export interface ContextOverflowParams {
+  /**
+   * The count is an estimate and its band reaches back under the window, so
+   * "the call will fail" is a prediction rather than a fact.
+   */
+  uncertain: boolean;
+  tokens: number;
+  modelName: string;
+  contextWindow: number;
+}
+
+export interface ContextNearLimitParams {
   tokens: number;
   modelName: string;
   contextWindow: number;
@@ -60,6 +71,20 @@ export interface PromptCachingParams {
   /** First template placeholder, or `null` when the prompt has none. */
   placeholder: string | null;
   prefixTokens: number;
+  /**
+   * The prefix is an estimate and the band reaches below the minimum, so the
+   * saving may not be collectable at all.
+   *
+   * The mirror of `BelowCacheMinimumParams.couldReachMinimum`, and the asymmetry
+   * was a real gap: that one hedged an estimate landing just *under* a hard
+   * threshold, while this one promised money on an estimate landing just *over*
+   * it. With a ±10% band an estimated 528-token prefix can truly be 475, in which
+   * case nothing caches and the figure beside this advisory is uncollectable.
+   *
+   * The cautionary direction matters more than the encouraging one, because this
+   * is the side with a dollar sign attached.
+   */
+  nearMinimum: boolean;
   totalTokens: number;
   minTokens: number;
   modelName: string;
@@ -88,12 +113,24 @@ export interface BelowCacheMinimumParams {
   totalTokens: number;
   /** Whether to point at Claude Opus 5's lower 512-token minimum. */
   mentionLowerMinimum: boolean;
+  /**
+   * The prefix is *estimated* and close enough to the minimum that the real
+   * count could be above it.
+   *
+   * Without this the advisory asserts "caching will not work here" from a number
+   * measured to ±10%, and on a prefix near the threshold that is not an imprecise
+   * figure — it is wrong advice, and it costs the reader the largest saving
+   * Trazum offers.
+   */
+  couldReachMinimum: boolean;
 }
 
 export interface CachePrefixReorderParams {
   staticTokensAfter: number;
   sharePct: number;
   placeholder: string;
+  /** The command that attempts it, because Trazum can do this itself. */
+  command: string;
 }
 
 export interface ModelDowngradeParams {
@@ -183,6 +220,7 @@ export interface CoreMessages {
   suggest: SuggestMessages;
   advisories: {
     contextOverflow(p: ContextOverflowParams): LocalizedMessage;
+    contextNearLimit(p: ContextNearLimitParams): LocalizedMessage;
     promptCaching(p: PromptCachingParams): LocalizedMessage;
     promptCachingNotWorthIt(): LocalizedMessage;
     belowCacheMinimum(p: BelowCacheMinimumParams): LocalizedMessage;

package/src/index.ts

@@ -1,5 +1,20 @@
 export * from './types.js';
-export { estimateTokens, countTokensAnthropic } from './tokenizer.js';
+export { ESTIMATE_ERROR_BAND_PCT, estimateTokens, countTokensAnthropic } from './tokenizer.js';
+export {
+  UNLABELLED,
+  cacheHitRate,
+  parseUsageLine,
+  profileUsage,
+  sharesOf,
+} from './usage.js';
+export type {
+  UsageProfileOptions,
+  UsageBreakdown,
+  UsageProfileReport,
+  UsageRecord,
+  UsageShares,
+} from './usage.js';
+export { DETECTABLE_LANGUAGES, detectTextLanguage } from './language.js';
 export { countSentences, profilePrompt } from './profile.js';
 export { PHRASE_LANGUAGES } from './phrases.js';
 export { toPromptfoo } from './promptfoo.js';

package/src/language.ts

@@ -0,0 +1,112 @@
+/**
+ * Which language a prompt is written in, to the extent it can be told cheaply.
+ *
+ * **Why the estimator needs this at all.** Measured against the counting
+ * endpoint, `estimateTokens` is accurate on English (+1.0%) and badly low on every
+ * other Latin language: German -37.3%, Spanish -22.9%, French -15.1%. Characters
+ * per token says why — English 3.44, French 2.66, Spanish 2.53, German 2.02 —
+ * while the estimator applied one divisor to all of them. Non-English text is
+ * thinner in the merge table, so the same number of characters costs more tokens.
+ *
+ * **Accents are not the signal.** That was the first hypothesis and it was tested
+ * and killed: a Spanish sample with zero accented characters comes out at -22.9%,
+ * against -22.1% for accented Spanish. Diacritics correlate with non-English text
+ * in a corpus and not in a prompt, and weighting them moved the figure by three
+ * points. What separates these languages is which words they are made of.
+ *
+ * So this counts function words — the shortest, commonest, most language-specific
+ * tokens there are. `the of and to` against `der die und ist` against
+ * `que de la en`.
+ *
+ * **It answers `null` when unsure, and that is the important part.** A prompt is
+ * not an essay: it can be three lines, or English instructions wrapped around a
+ * Spanish example, or a JSON schema with no prose at all. Guessing on those would
+ * apply a language's divisor to text that is not in that language, which is how a
+ * fix for one case becomes a regression for four. `null` means "use the default",
+ * and the default is the English-calibrated behaviour this has always had.
+ */
+
+/**
+ * Function words per language, lower-cased, matched whole.
+ *
+ * Chosen for frequency and for *not overlapping*, which matters more: `de` is
+ * frequent in Spanish, French and Portuguese, so it earns nothing and is a
+ * liability. Each list is words that are common in its own language and rare in
+ * the others, which is what lets a small count decide.
+ *
+ * `en` is included even though it is the default, because the margin rule needs to
+ * know when English *won*: a Spanish prompt quoting an English sentence should not
+ * become Spanish on two hits.
+ */
+const FUNCTION_WORDS: Readonly<Record<string, readonly string[]>> = {
+  en: ['the', 'and', 'of', 'to', 'is', 'that', 'with', 'for', 'you', 'this', 'are', 'not'],
+  es: ['que', 'los', 'las', 'del', 'una', 'para', 'con', 'como', 'pero', 'sus', 'este', 'siempre'],
+  fr: ['les', 'des', 'une', 'dans', 'pour', 'avec', 'vous', 'est', 'sur', 'aux', 'cette', 'toujours'],
+  de: ['der', 'die', 'das', 'und', 'nicht', 'sie', 'ist', 'mit', 'auf', 'einen', 'einer', 'immer'],
+  pt: ['que', 'dos', 'das', 'uma', 'para', 'com', 'como', 'não', 'seu', 'este', 'pelo', 'sempre'],
+  /**
+   * Rebuilt once, because half of it was Spanish. `per con del una sempre` are as
+   * common in Spanish as in Italian, so they earned nothing and cost the margin
+   * rule its answer: an Italian code-review prompt scored a tie and came back
+   * `null`, fell through to the English divisor, and measured -21.9%. These are
+   * words Italian has and Spanish does not.
+   */
+  it: ['il', 'della', 'nella', 'nel', 'che', 'quando', 'senza', 'gli', 'delle', 'degli', 'più', 'anche'],
+  nl: ['het', 'een', 'van', 'niet', 'zijn', 'met', 'voor', 'dat', 'aan', 'deze', 'wordt', 'altijd'],
+};
+
+/** Languages this can name. Exported so callers can key a table off it. */
+export const DETECTABLE_LANGUAGES: readonly string[] = Object.keys(FUNCTION_WORDS);
+
+/**
+ * How many hits the winner needs, absolutely and relative to the runner-up.
+ *
+ * Both bars exist for the same reason and neither is enough alone. The absolute
+ * one stops a three-line prompt being classified on a single word. The ratio stops
+ * a document that is genuinely mixed — English instructions around a Spanish
+ * example — from being called whichever language happened to appear once more.
+ *
+ * Deliberately cautious in the direction of `null`. A wrong language costs
+ * accuracy on text that was fine before; `null` costs only the improvement.
+ */
+const MIN_HITS = 4;
+const MIN_RATIO = 1.6;
+
+/**
+ * The language of `text`, or `null` when no answer is safe.
+ *
+ * Case-insensitive, whole words only, and it stops reading after a bounded
+ * prefix: a prompt can be a megabyte, the answer does not get better after a few
+ * thousand words, and this runs inside `estimateTokens` on every call.
+ */
+export function detectTextLanguage(text: string): string | null {
+  if (!text) return null;
+
+  // Bounded so this stays cheap on a large prompt. Function words are frequent
+  // enough that a prefix this size decides anything a whole document would.
+  const sample = text.length > 20_000 ? text.slice(0, 20_000) : text;
+
+  const words = sample.toLowerCase().match(/[\p{L}]+/gu);
+  if (!words || words.length < MIN_HITS * 2) return null;
+
+  const seen = new Set(words);
+  const scores: Array<[string, number]> = [];
+  for (const [language, list] of Object.entries(FUNCTION_WORDS)) {
+    let hits = 0;
+    // Distinct words rather than occurrences: a prompt that repeats "the" forty
+    // times is not more English than one that uses forty different English words,
+    // and counting occurrences lets one hammered word decide.
+    for (const word of list) if (seen.has(word)) hits++;
+    scores.push([language, hits]);
+  }
+
+  scores.sort((a, b) => b[1] - a[1]);
+  const [best, bestHits] = scores[0]!;
+  const runnerUp = scores[1]?.[1] ?? 0;
+
+  if (bestHits < MIN_HITS) return null;
+  // A runner-up of zero is a clear win; guard the division rather than special-case.
+  if (runnerUp > 0 && bestHits / runnerUp < MIN_RATIO) return null;
+
+  return best;
+}