@trazum/core 1.8.0

package/dist/suggest.js

@@ -0,0 +1,151 @@
+import { getMessages } from './i18n/index.js';
+import { segment } from './segment.js';
+import { estimateTokens } from './tokenizer.js';
+export const SUGGEST_SYSTEM_PROMPT = `You find phrases in a prompt that say something in more words than they need.
+
+Return ONLY a JSON array. Each element is {"before": "...", "after": "..."}.
+
+Rules:
+- "before" MUST be copied character for character from the prompt. Do not paraphrase it, do not fix its punctuation, do not change its capitalisation. If you cannot copy it exactly, leave it out.
+- "after" says the same thing in fewer words. Use "" to delete the phrase entirely.
+- Preserve meaning exactly. Never change what the prompt asks for, its constraints, its output format or its success criteria.
+- Never touch code, URLs, template placeholders ({{x}}, \${x}, {x}) or XML/HTML tags — do not include them in "before" at all.
+- Keep the original language of the prompt.
+- Prefer a few high-value rewrites to many trivial ones. Return [] if there is nothing worth changing.
+
+No explanation, no code fences, no commentary. The array alone.`;
+/** Strips a code fence if the model wrapped its JSON despite being told not to. */
+function unwrap(text) {
+    const trimmed = text.trim();
+    const fenced = /^(?:```|~~~)[a-zA-Z]*\n([\s\S]*?)\n?(?:```|~~~)$/.exec(trimmed);
+    return (fenced?.[1] ?? trimmed).trim();
+}
+/**
+ * Character ranges that must not be edited.
+ *
+ * `segment` returns the pieces in order without offsets, so they are
+ * accumulated here rather than recomputed per suggestion.
+ */
+function protectedRanges(prompt) {
+    const ranges = [];
+    let at = 0;
+    for (const piece of segment(prompt)) {
+        if (piece.kind === 'protected')
+            ranges.push([at, at + piece.text.length]);
+        at += piece.text.length;
+    }
+    return ranges;
+}
+const overlaps = (a, b) => a[0] < b[1] && b[0] < a[1];
+/** Every offset at which `needle` occurs in `haystack`. */
+function occurrences(haystack, needle) {
+    if (needle === '')
+        return [];
+    const found = [];
+    let at = haystack.indexOf(needle);
+    while (at !== -1) {
+        found.push(at);
+        at = haystack.indexOf(needle, at + needle.length);
+    }
+    return found;
+}
+/**
+ * Asks the provider for rewrites and returns the ones that survive checking.
+ *
+ * A provider that returns something unparseable yields an empty result rather
+ * than throwing: the deterministic rules have already run, and a malformed
+ * answer from an optional pass should cost the caller nothing.
+ */
+export async function suggestRewrites(prompt, provider, options = {}) {
+    const count = options.tokenCounter ?? estimateTokens;
+    const max = options.max ?? 20;
+    const raw = await provider.complete({ system: SUGGEST_SYSTEM_PROMPT, user: prompt });
+    let proposed;
+    try {
+        const parsed = JSON.parse(unwrap(raw));
+        proposed = Array.isArray(parsed) ? parsed : [];
+    }
+    catch {
+        return { suggestions: [], rejected: [], provider: provider.name, model: provider.model };
+    }
+    const guarded = protectedRanges(prompt);
+    const suggestions = [];
+    const rejected = [];
+    const taken = [];
+    for (const item of proposed) {
+        if (typeof item?.before !== 'string' || typeof item?.after !== 'string')
+            continue;
+        const before = item.before;
+        const after = item.after;
+        if (before.trim() === '')
+            continue;
+        const reject = (reason) => rejected.push({ before, after, reason });
+        const at = occurrences(prompt, before);
+        if (at.length === 0) {
+            reject('not-found');
+            continue;
+        }
+        // Occurrences that sit clear of protected content. A phrase that appears
+        // three times, once inside a code block, is still worth rewriting in the
+        // other two — refusing the whole suggestion would be the easier answer and
+        // the wrong one.
+        const usable = at.filter((start) => !guarded.some((range) => overlaps([start, start + before.length], range)));
+        if (usable.length === 0) {
+            reject('touches-protected');
+            continue;
+        }
+        // The replacement must not bring protected content in with it.
+        if (after !== '' && segment(after).some((piece) => piece.kind === 'protected')) {
+            reject('introduces-protected');
+            continue;
+        }
+        const saved = (count(before) - count(after)) * usable.length;
+        if (saved <= 0) {
+            reject('no-saving');
+            continue;
+        }
+        const free = usable.filter((start) => !taken.some((range) => overlaps([start, start + before.length], range)));
+        if (free.length === 0) {
+            reject('overlaps');
+            continue;
+        }
+        for (const start of free)
+            taken.push([start, start + before.length]);
+        suggestions.push({
+            before,
+            after,
+            offsets: free,
+            tokensSaved: (count(before) - count(after)) * free.length,
+        });
+    }
+    suggestions.sort((a, b) => b.tokensSaved - a.tokensSaved);
+    return {
+        suggestions: suggestions.slice(0, max),
+        rejected,
+        provider: provider.name,
+        model: provider.model,
+    };
+}
+/**
+ * Applies suggestions to the prompt.
+ *
+ * Right to left, so an earlier edit cannot move the offsets of a later one —
+ * the bug that makes every naive implementation of this corrupt long prompts,
+ * and one that shows up only when two suggestions are far enough apart that a
+ * short test never notices.
+ */
+export function applyRewrites(prompt, suggestions) {
+    const edits = suggestions
+        .flatMap((s) => s.offsets.map((start) => ({ start, end: start + s.before.length, after: s.after })))
+        .sort((a, b) => b.start - a.start);
+    let text = prompt;
+    for (const edit of edits) {
+        text = text.slice(0, edit.start) + edit.after + text.slice(edit.end);
+    }
+    return text;
+}
+/** A reason code turned into a sentence, for a report. */
+export function rejectionText(reason, locale) {
+    return getMessages(locale).suggest[reason]();
+}
+//# sourceMappingURL=suggest.js.map

package/dist/suggest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"suggest.js","sourceRoot":"","sources":["../src/suggest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AA0EhD,MAAM,CAAC,MAAM,qBAAqB,GAAG;;;;;;;;;;;;gEAY2B,CAAC;AASjE,mFAAmF;AACnF,SAAS,MAAM,CAAC,IAAY;IAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;IAC5B,MAAM,MAAM,GAAG,kDAAkD,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAChF,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AACzC,CAAC;AAED;;;;;GAKG;AACH,SAAS,eAAe,CAAC,MAAc;IACrC,MAAM,MAAM,GAA4B,EAAE,CAAC;IAC3C,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACpC,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW;YAAE,MAAM,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;QAC1E,EAAE,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,QAAQ,GAAG,CAAC,CAAmB,EAAE,CAAmB,EAAW,EAAE,CACrE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AAE7B,2DAA2D;AAC3D,SAAS,WAAW,CAAC,QAAgB,EAAE,MAAc;IACnD,IAAI,MAAM,KAAK,EAAE;QAAE,OAAO,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAClC,OAAO,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC;QACjB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,MAAc,EACd,QAAqB,EACrB,OAAO,GAAmB,EAAE;IAE5B,MAAM,KAAK,GAAG,OAAO,CAAC,YAAY,IAAI,cAAc,CAAC;IACrD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,EAAE,CAAC;IAE9B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,EAAE,qBAAqB,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;IAErF,IAAI,QAAsD,CAAC;IAC3D,IAAI,CAAC;QACH,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAChD,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IACjD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,WAAW,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC;IAC3F,CAAC;IAED,MAAM,OAAO,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;IACxC,MAAM,WAAW,GAAwB,EAAE,CAAC;IAC5C,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,MAAM,KAAK,GAA4B,EAAE,CAAC;IAE1C,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,IAAI,OAAO,IAAI,EAAE,MAAM,KAAK,QAAQ,IAAI,OAAO,IAAI,EAAE,KAAK,KAAK,QAAQ;YAAE,SAAS;QAClF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACzB,IAAI,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE;YAAE,SAAS;QAEnC,MAAM,MAAM,GAAG,CAAC,MAAsB,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;QAEpF,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACvC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACpB,MAAM,CAAC,WAAW,CAAC,CAAC;YACpB,SAAS;QACX,CAAC;QAED,yEAAyE;QACzE,yEAAyE;QACzE,2EAA2E;QAC3E,iBAAiB;QACjB,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,CACtB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,CACrF,CAAC;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,mBAAmB,CAAC,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,+DAA+D;QAC/D,IAAI,KAAK,KAAK,EAAE,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,WAAW,CAAC,EAAE,CAAC;YAC/E,MAAM,CAAC,sBAAsB,CAAC,CAAC;YAC/B,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAC7D,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,MAAM,CAAC,WAAW,CAAC,CAAC;YACpB,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CACxB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAC,CACnF,CAAC;QACF,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACtB,MAAM,CAAC,UAAU,CAAC,CAAC;YACnB,SAAS;QACX,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QACrE,WAAW,CAAC,IAAI,CAAC;YACf,MAAM;YACN,KAAK;YACL,OAAO,EAAE,IAAI;YACb,WAAW,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM;SAC1D,CAAC,CAAC;IACL,CAAC;IAED,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC;IAC1D,OAAO;QACL,WAAW,EAAE,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QACtC,QAAQ;QACR,QAAQ,EAAE,QAAQ,CAAC,IAAI;QACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;KACtB,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAAC,MAAc,EAAE,WAAyC;IACrF,MAAM,KAAK,GAAG,WAAW;SACtB,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;SACnG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAErC,IAAI,IAAI,GAAG,MAAM,CAAC;IAClB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACvE,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,aAAa,CAAC,MAAsB,EAAE,MAAc;IAClE,OAAO,WAAW,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;AAC/C,CAAC"}

package/dist/tokenizer.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Dependency-free token estimator.
+ *
+ * This is NOT a real tokenizer: it is a heuristic calibrated per character
+ * class. It is built to keep the typical error on ordinary text
+ * (English/Spanish, markdown, code) inside ±15%, which is plenty for comparing
+ * two versions of the same prompt — but do NOT bill anyone from it.
+ *
+ * **That band is a design target that has not been measured.** It is printed on
+ * every report and every dollar figure descends from it, and until
+ * `scripts/measure-token-band.mjs` has been run against the official counting
+ * endpoint, nothing in this repository establishes that it holds. It is stated
+ * as one number for all text, which is a further assumption: the branches below
+ * treat CJK, digits and punctuation quite differently from words, and there is
+ * no reason those should land on the same accuracy.
+ *
+ * `test/token-band.test.js` asserts the band per text type as soon as the ground
+ * truth exists. Until then it says so rather than passing quietly.
+ *
+ * For exact numbers use `countTokensAnthropic` (the official token-counting
+ * endpoint, which is free) or pass your own `TokenCounter`.
+ */
+/**
+ * Estimates how many tokens `text` occupies.
+ *
+ * Rules per character class:
+ * - words: ~4 effective characters per token (minimum 1)
+ * - numbers: ~3 digits per token
+ * - punctuation: ~2 marks per token
+ * - newlines: ~1 token per 2 consecutive newlines
+ * - CJK: 1 token per character
+ * - emoji and symbols outside the BMP: 2 tokens
+ * - spaces: 0 (absorbed into the following token)
+ */
+export declare function estimateTokens(text: string): number;
+/** Asynchronous token counter, for remote sources. */
+export type AsyncTokenCounter = (text: string) => Promise<number>;
+export interface AnthropicCounterOptions {
+    apiKey: string;
+    /** Model to count against. Token counts are model-specific. */
+    model?: string;
+    baseUrl?: string;
+    fetchImpl?: typeof fetch;
+    /** See `OpenAiCompatibleOptions.allowInsecure`: only when you chose the URL. */
+    allowInsecure?: boolean;
+}
+/**
+ * Exact counter using the official `/v1/messages/count_tokens` endpoint.
+ * The endpoint does not bill tokens, so you can use it freely to measure.
+ *
+ * The third door, and the one nobody had looked at: this takes a `baseUrl` and
+ * sends an `x-api-key` to it. Both providers were hardened at the boundary and
+ * this was left with no check at all, because it is called a counter rather
+ * than a provider. It goes through the same gate now.
+ */
+export declare function countTokensAnthropic(options: AnthropicCounterOptions): AsyncTokenCounter;
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAeH;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAwFnD;AAED,sDAAsD;AACtD,MAAM,MAAM,iBAAiB,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAElE,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,gFAAgF;IAChF,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,uBAAuB,GAAG,iBAAiB,CA6BxF"}

package/dist/tokenizer.js

@@ -0,0 +1,157 @@
+/**
+ * Dependency-free token estimator.
+ *
+ * This is NOT a real tokenizer: it is a heuristic calibrated per character
+ * class. It is built to keep the typical error on ordinary text
+ * (English/Spanish, markdown, code) inside ±15%, which is plenty for comparing
+ * two versions of the same prompt — but do NOT bill anyone from it.
+ *
+ * **That band is a design target that has not been measured.** It is printed on
+ * every report and every dollar figure descends from it, and until
+ * `scripts/measure-token-band.mjs` has been run against the official counting
+ * endpoint, nothing in this repository establishes that it holds. It is stated
+ * as one number for all text, which is a further assumption: the branches below
+ * treat CJK, digits and punctuation quite differently from words, and there is
+ * no reason those should land on the same accuracy.
+ *
+ * `test/token-band.test.js` asserts the band per text type as soon as the ground
+ * truth exists. Until then it says so rather than passing quietly.
+ *
+ * For exact numbers use `countTokensAnthropic` (the official token-counting
+ * endpoint, which is free) or pass your own `TokenCounter`.
+ */
+import { SAFE_FETCH_INIT, checkedEndpoint } from './net.js';
+const CJK = /[぀-ヿ㐀-䶿一-鿿가-힯]/;
+const LETTER = /[A-Za-zÀ-ɏͰ-ϿЀ-ӿ]/;
+const DIGIT = /[0-9]/;
+/** Effective word length: non-ASCII characters split into more tokens. */
+function effectiveLength(word) {
+    let len = 0;
+    for (const ch of word)
+        len += ch.charCodeAt(0) > 127 ? 2 : 1;
+    return len;
+}
+/**
+ * Estimates how many tokens `text` occupies.
+ *
+ * Rules per character class:
+ * - words: ~4 effective characters per token (minimum 1)
+ * - numbers: ~3 digits per token
+ * - punctuation: ~2 marks per token
+ * - newlines: ~1 token per 2 consecutive newlines
+ * - CJK: 1 token per character
+ * - emoji and symbols outside the BMP: 2 tokens
+ * - spaces: 0 (absorbed into the following token)
+ */
+export function estimateTokens(text) {
+    if (!text)
+        return 0;
+    let total = 0;
+    let i = 0;
+    const chars = Array.from(text);
+    while (i < chars.length) {
+        const ch = chars[i];
+        if (ch === ' ' || ch === '\t' || ch === '\r') {
+            i++;
+            continue;
+        }
+        if (ch === '\n') {
+            let n = 0;
+            while (i < chars.length && chars[i] === '\n') {
+                n++;
+                i++;
+            }
+            total += Math.ceil(n / 2);
+            continue;
+        }
+        if (CJK.test(ch)) {
+            let n = 0;
+            while (i < chars.length && CJK.test(chars[i])) {
+                n++;
+                i++;
+            }
+            total += n;
+            continue;
+        }
+        if (LETTER.test(ch)) {
+            let word = '';
+            while (i < chars.length && (LETTER.test(chars[i]) || DIGIT.test(chars[i]))) {
+                word += chars[i];
+                i++;
+            }
+            total += Math.max(1, Math.ceil(effectiveLength(word) / 4));
+            continue;
+        }
+        if (DIGIT.test(ch)) {
+            let n = 0;
+            while (i < chars.length && DIGIT.test(chars[i])) {
+                n++;
+                i++;
+            }
+            total += Math.ceil(n / 3);
+            continue;
+        }
+        // Outside the BMP (emoji, unusual symbols): usually 2+ tokens.
+        if (ch.codePointAt(0) > 0xffff) {
+            total += 2;
+            i++;
+            continue;
+        }
+        // ASCII punctuation and symbols.
+        let n = 0;
+        while (i < chars.length &&
+            !LETTER.test(chars[i]) &&
+            !DIGIT.test(chars[i]) &&
+            !CJK.test(chars[i]) &&
+            chars[i] !== ' ' &&
+            chars[i] !== '\n' &&
+            chars[i] !== '\t' &&
+            chars[i] !== '\r' &&
+            chars[i].codePointAt(0) <= 0xffff) {
+            n++;
+            i++;
+        }
+        if (n === 0) {
+            // Unclassified character: count it as 1 and advance so the loop cannot stall.
+            total += 1;
+            i++;
+        }
+        else {
+            total += Math.ceil(n / 2);
+        }
+    }
+    return total;
+}
+/**
+ * Exact counter using the official `/v1/messages/count_tokens` endpoint.
+ * The endpoint does not bill tokens, so you can use it freely to measure.
+ *
+ * The third door, and the one nobody had looked at: this takes a `baseUrl` and
+ * sends an `x-api-key` to it. Both providers were hardened at the boundary and
+ * this was left with no check at all, because it is called a counter rather
+ * than a provider. It goes through the same gate now.
+ */
+export function countTokensAnthropic(options) {
+    const { apiKey, model = 'claude-opus-5', baseUrl = 'https://api.anthropic.com', fetchImpl = fetch, allowInsecure = false, } = options;
+    const endpoint = checkedEndpoint(baseUrl, { allowInsecure, name: 'anthropic-count-tokens' });
+    return async (text) => {
+        if (!text.trim())
+            return 0;
+        const res = await fetchImpl(`${endpoint}/v1/messages/count_tokens`, {
+            ...SAFE_FETCH_INIT,
+            method: 'POST',
+            headers: {
+                'content-type': 'application/json',
+                'x-api-key': apiKey,
+                'anthropic-version': '2023-06-01',
+            },
+            body: JSON.stringify({ model, messages: [{ role: 'user', content: text }] }),
+        });
+        if (!res.ok) {
+            throw new Error(`count_tokens failed (${res.status}): ${await res.text()}`);
+        }
+        const data = (await res.json());
+        return data.input_tokens;
+    };
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE5D,MAAM,GAAG,GAAG,gBAAgB,CAAC;AAC7B,MAAM,MAAM,GAAG,mBAAmB,CAAC;AACnC,MAAM,KAAK,GAAG,OAAO,CAAC;AAEtB,0EAA0E;AAC1E,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,EAAE,IAAI,IAAI;QAAE,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE/B,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QAErB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAC7C,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC7C,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACjB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBAC/C,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACpB,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,EAAE,CAAC;gBAC7E,IAAI,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC;gBAClB,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC3D,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBACjD,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,+DAA+D;QAC/D,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAE,GAAG,MAAM,EAAE,CAAC;YAChC,KAAK,IAAI,CAAC,CAAC;YACX,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,iCAAiC;QACjC,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OACE,CAAC,GAAG,KAAK,CAAC,MAAM;YAChB,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACvB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACtB,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACpB,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG;YAChB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAE,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,MAAM,EACnC,CAAC;YACD,CAAC,EAAE,CAAC;YACJ,CAAC,EAAE,CAAC;QACN,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,8EAA8E;YAC9E,KAAK,IAAI,CAAC,CAAC;YACX,CAAC,EAAE,CAAC;QACN,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAeD;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAgC;IACnE,MAAM,EACJ,MAAM,EACN,KAAK,GAAG,eAAe,EACvB,OAAO,GAAG,2BAA2B,EACrC,SAAS,GAAG,KAAK,EACjB,aAAa,GAAG,KAAK,GACtB,GAAG,OAAO,CAAC;IAEZ,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAE7F,OAAO,KAAK,EAAE,IAAY,EAAmB,EAAE;QAC7C,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,CAAC,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,QAAQ,2BAA2B,EAAE;YAClE,GAAG,eAAe;YAClB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,WAAW,EAAE,MAAM;gBACnB,mBAAmB,EAAE,YAAY;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;SAC7E,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA6B,CAAC;QAC5D,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC,CAAC;AACJ,CAAC"}