@trazum/core 1.9.0 → 1.25.0

package/dist/session-cost.js

@@ -0,0 +1,90 @@
+import { effectivePricing, multipliersFor } from './pricing.js';
+import { UNLABELLED } from './usage.js';
+/** Every billed dollar of one call, at its own model's rates. */
+function costOf(record, catalogue, on) {
+    const model = catalogue.byId.get(record.model);
+    if (!model)
+        return null;
+    const { inputPerMTok, outputPerMTok } = effectivePricing(model, on);
+    const rates = multipliersFor(model);
+    const per = (tokens, rate) => (tokens / 1_000_000) * rate;
+    return (per(record.inputTokens, inputPerMTok) +
+        per(record.cacheReadTokens, inputPerMTok * rates.cacheRead) +
+        per(record.cacheWrite5mTokens, inputPerMTok * rates.cacheWrite5m) +
+        per(record.cacheWrite1hTokens, inputPerMTok * rates.cacheWrite1h) +
+        per(record.outputTokens, outputPerMTok));
+}
+const median = (sorted) => {
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+};
+/**
+ * The 95th percentile by **nearest rank**: the smallest value at or above which
+ * 95% of the conversations sit. Interpolating between two conversations would
+ * report a cost no conversation had, and this figure exists to be compared
+ * against a real quota.
+ */
+const p95 = (sorted) => sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)];
+export function createSessionCostTracker(options) {
+    const { catalogue, on = new Date(), minSessions = 5 } = options;
+    const slices = new Map();
+    const add = (record) => {
+        if (record.session === null)
+            return;
+        const cost = costOf(record, catalogue, on);
+        // An unpriced model contributes no dollars anywhere else either.
+        if (cost === null)
+            return;
+        const sliceKey = `${record.label ?? UNLABELLED}\n${record.model}`;
+        let sessions = slices.get(sliceKey);
+        if (!sessions) {
+            sessions = new Map();
+            slices.set(sliceKey, sessions);
+        }
+        const existing = sessions.get(record.session);
+        if (existing) {
+            existing.usd += cost;
+            existing.turns += 1;
+        }
+        else {
+            sessions.set(record.session, { usd: cost, turns: 1 });
+        }
+    };
+    const finish = () => {
+        const out = [];
+        for (const [sliceKey, sessions] of slices) {
+            if (sessions.size < minSessions)
+                continue;
+            const split = sliceKey.indexOf('\n');
+            const modelId = sliceKey.slice(split + 1);
+            const model = catalogue.byId.get(modelId);
+            if (!model)
+                continue;
+            const costs = [...sessions.values()].map((s) => s.usd).sort((a, b) => a - b);
+            const turns = [...sessions.values()].map((s) => s.turns).sort((a, b) => a - b);
+            out.push({
+                label: sliceKey.slice(0, split),
+                model: modelId,
+                modelName: model.displayName,
+                sessions: sessions.size,
+                calls: turns.reduce((sum, t) => sum + t, 0),
+                totalUsd: costs.reduce((sum, c) => sum + c, 0),
+                medianUsd: median(costs),
+                p95Usd: p95(costs),
+                maxUsd: costs[costs.length - 1],
+                medianTurns: median(turns),
+            });
+        }
+        // The most money first — the order somebody would act in.
+        return out.sort((a, b) => b.totalUsd - a.totalUsd || a.label.localeCompare(b.label));
+    };
+    return { add, finish };
+}
+/** The same measurement over a list, for a caller holding one already. */
+export function sessionCostShapes(records, options) {
+    const tracker = createSessionCostTracker(options);
+    for (const record of records)
+        tracker.add(record);
+    return tracker.finish();
+}
+//# sourceMappingURL=session-cost.js.map

package/dist/session-cost.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"session-cost.js","sourceRoot":"","sources":["../src/session-cost.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAwExC,iEAAiE;AACjE,SAAS,MAAM,CAAC,MAAmB,EAAE,SAA2B,EAAE,EAAQ;IACxE,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC/C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,EAAE,YAAY,EAAE,aAAa,EAAE,GAAG,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACpE,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,CAAC,MAAc,EAAE,IAAY,EAAU,EAAE,CAAC,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC;IAClF,OAAO,CACL,GAAG,CAAC,MAAM,CAAC,WAAW,EAAE,YAAY,CAAC;QACrC,GAAG,CAAC,MAAM,CAAC,eAAe,EAAE,YAAY,GAAG,KAAK,CAAC,SAAS,CAAC;QAC3D,GAAG,CAAC,MAAM,CAAC,kBAAkB,EAAE,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACjE,GAAG,CAAC,MAAM,CAAC,kBAAkB,EAAE,YAAY,GAAG,KAAK,CAAC,YAAY,CAAC;QACjE,GAAG,CAAC,MAAM,CAAC,YAAY,EAAE,aAAa,CAAC,CACxC,CAAC;AACJ,CAAC;AAED,MAAM,MAAM,GAAG,CAAC,MAAgB,EAAU,EAAE;IAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAE,GAAG,MAAM,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,CAAC;AACxF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,GAAG,GAAG,CAAC,MAAgB,EAAU,EAAE,CACvC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC;AAE5E,MAAM,UAAU,wBAAwB,CAAC,OAA2B;IAClE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,IAAI,EAAE,EAAE,WAAW,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;IAChE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuD,CAAC;IAE9E,MAAM,GAAG,GAAG,CAAC,MAAmB,EAAQ,EAAE;QACxC,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI;YAAE,OAAO;QACpC,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;QAC3C,iEAAiE;QACjE,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO;QAE1B,MAAM,QAAQ,GAAG,GAAG,MAAM,CAAC,KAAK,IAAI,UAAU,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC;QAClE,IAAI,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACpC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACjC,CAAC;QACD,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QAC9C,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,GAAG,IAAI,IAAI,CAAC;YACrB,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;QACtB,CAAC;aAAM,CAAC;YACN,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC;QACxD,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,GAAuB,EAAE;QACtC,MAAM,GAAG,GAAuB,EAAE,CAAC;QAEnC,KAAK,MAAM,CAAC,QAAQ,EAAE,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC;YAC1C,IAAI,QAAQ,CAAC,IAAI,GAAG,WAAW;gBAAE,SAAS;YAC1C,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,KAAK;gBAAE,SAAS;YAErB,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC7E,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC/E,GAAG,CAAC,IAAI,CAAC;gBACP,KAAK,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;gBAC/B,KAAK,EAAE,OAAO;gBACd,SAAS,EAAE,KAAK,CAAC,WAAW;gBAC5B,QAAQ,EAAE,QAAQ,CAAC,IAAI;gBACvB,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC3C,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC;gBAC9C,SAAS,EAAE,MAAM,CAAC,KAAK,CAAC;gBACxB,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC;gBAClB,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAE;gBAChC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC;aAC3B,CAAC,CAAC;QACL,CAAC;QAED,0DAA0D;QAC1D,OAAO,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IACvF,CAAC,CAAC;IAEF,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACzB,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,iBAAiB,CAC/B,OAA+B,EAC/B,OAA2B;IAE3B,MAAM,OAAO,GAAG,wBAAwB,CAAC,OAAO,CAAC,CAAC;IAClD,KAAK,MAAM,MAAM,IAAI,OAAO;QAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC;AAC1B,CAAC"}

package/dist/session-ledger.d.ts

@@ -0,0 +1,77 @@
+import type { PricingCatalogue } from './pricing.js';
+import type { UsageRecord } from './usage.js';
+/**
+ * Cache writes made by conversations that never came back.
+ *
+ * ## The waste the aggregate hides
+ *
+ * A cache write is a bet: pay 1.25x input now (2x at the 1-hour TTL) so the
+ * *next* call reads the prefix at 0.1x. A conversation that ends after its
+ * first turn never places that next call — its write bought reuse that its own
+ * conversation never made. On a workload with many short sessions this is a
+ * steady leak, and it hides inside healthy-looking totals: the long sessions'
+ * reads pay for the cache overall, so `cacheEconomics` reports `paid-off`
+ * while every one-turn drive-by pays the premium for nothing.
+ *
+ * ## The caveat that keeps the figure honest
+ *
+ * The provider's cache is keyed by prefix content, not by conversation. A
+ * one-turn session's write **can** be read back by a different session that
+ * sends the same prefix within the TTL — a shared system prompt does exactly
+ * that — and a usage log cannot see whose write a read hit. So the figure
+ * reported here is a **ceiling, named as one**: these writes paid off only if
+ * another conversation shared the prefix in time, and the log cannot say
+ * whether one did.
+ *
+ * There is one case where the ceiling collapses into a fact, and the caller
+ * can detect it from the slice it already has: when the slice recorded **zero
+ * cache reads**, nothing read those writes — within the session, across
+ * sessions, at all. The row deliberately does not decide this itself; the
+ * breakdown holding the slice's reads belongs to the caller, and deriving it
+ * twice is how two figures drift.
+ *
+ * Session keys group turns and never leave this module, like everywhere else
+ * the field is touched.
+ */
+export interface SingleTurnCacheWrites {
+    label: string;
+    model: string;
+    modelName: string;
+    /** Conversations seen in this slice — with a session key, priced model. */
+    sessions: number;
+    /** Median turns per conversation, for scale: 1-turn sessions in a sea of 40-turn ones read differently than in a sea of 2s. */
+    medianTurns: number;
+    /** Conversations that ended after exactly one recorded turn. */
+    singleTurnSessions: number;
+    /** Cache-write tokens those one-turn conversations paid for. */
+    singleTurnWriteTokens: number;
+    /**
+     * What those writes cost, at the same rates the bill used — the 5-minute
+     * rate for writes whose TTL the log did not state, so like the bill it is
+     * a floor when `assumedTtlTokens` is non-zero. A **ceiling on the waste**
+     * (another conversation may have read the prefix; the log cannot see it)
+     * built on a **floor of a price** — both directions named, neither guessed.
+     */
+    singleTurnWriteUsd: number;
+    /** The part of `singleTurnWriteTokens` whose TTL the log did not record. */
+    assumedTtlTokens: number;
+}
+export interface SessionLedgerOptions {
+    catalogue: PricingCatalogue;
+    /** Date the prices are read at, so a promotional rate resolves the same way. */
+    on?: Date;
+}
+export interface SessionLedgerTracker {
+    add(record: UsageRecord): void;
+    finish(): SingleTurnCacheWrites[];
+}
+/**
+ * An accumulator, like the TTL-fit tracker and for the same reason: one pass
+ * over a log measured in megabytes, holding one small tally per conversation.
+ * No timestamp needed — "came back" is a fact about turn count, not the clock,
+ * so this measures logs the TTL-fit cannot.
+ */
+export declare function createSessionLedgerTracker(options: SessionLedgerOptions): SessionLedgerTracker;
+/** The same measurement over a list, for a caller holding one already. */
+export declare function singleTurnCacheWrites(records: readonly UsageRecord[], options: SessionLedgerOptions): SingleTurnCacheWrites[];
+//# sourceMappingURL=session-ledger.d.ts.map

package/dist/session-ledger.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"session-ledger.d.ts","sourceRoot":"","sources":["../src/session-ledger.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,2EAA2E;IAC3E,QAAQ,EAAE,MAAM,CAAC;IACjB,+HAA+H;IAC/H,WAAW,EAAE,MAAM,CAAC;IACpB,gEAAgE;IAChE,kBAAkB,EAAE,MAAM,CAAC;IAC3B,gEAAgE;IAChE,qBAAqB,EAAE,MAAM,CAAC;IAC9B;;;;;;OAMG;IACH,kBAAkB,EAAE,MAAM,CAAC;IAC3B,4EAA4E;IAC5E,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,EAAE,gBAAgB,CAAC;IAC5B,gFAAgF;IAChF,EAAE,CAAC,EAAE,IAAI,CAAC;CACX;AAED,MAAM,WAAW,oBAAoB;IACnC,GAAG,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IAC/B,MAAM,IAAI,qBAAqB,EAAE,CAAC;CACnC;AAcD;;;;;GAKG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,oBAAoB,GAAG,oBAAoB,CAsF9F;AAED,0EAA0E;AAC1E,wBAAgB,qBAAqB,CACnC,OAAO,EAAE,SAAS,WAAW,EAAE,EAC/B,OAAO,EAAE,oBAAoB,GAC5B,qBAAqB,EAAE,CAIzB"}

package/dist/session-ledger.js

@@ -0,0 +1,99 @@
+import { effectivePricing, multipliersFor } from './pricing.js';
+import { UNLABELLED } from './usage.js';
+const median = (sorted) => {
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 1 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2;
+};
+/**
+ * An accumulator, like the TTL-fit tracker and for the same reason: one pass
+ * over a log measured in megabytes, holding one small tally per conversation.
+ * No timestamp needed — "came back" is a fact about turn count, not the clock,
+ * so this measures logs the TTL-fit cannot.
+ */
+export function createSessionLedgerTracker(options) {
+    const { catalogue, on = new Date() } = options;
+    const slices = new Map();
+    const add = (record) => {
+        // An unpriced model has no rate to price the waste at, and contributes no
+        // dollars anywhere else either.
+        if (record.session === null || !catalogue.byId.has(record.model))
+            return;
+        const key = `${record.label ?? UNLABELLED}\n${record.model}`;
+        let sessions = slices.get(key);
+        if (!sessions) {
+            sessions = new Map();
+            slices.set(key, sessions);
+        }
+        let tally = sessions.get(record.session);
+        if (!tally) {
+            tally = { turns: 0, write5mTokens: 0, write1hTokens: 0, assumedTokens: 0 };
+            sessions.set(record.session, tally);
+        }
+        tally.turns += 1;
+        if (record.writeTtlKnown) {
+            tally.write5mTokens += record.cacheWrite5mTokens;
+            tally.write1hTokens += record.cacheWrite1hTokens;
+        }
+        else {
+            // The flat count sits in the 5m bucket by pricing convention; kept
+            // apart here so the row can say how much of its price is a floor.
+            tally.assumedTokens += record.cacheWrite5mTokens;
+        }
+    };
+    const finish = () => {
+        const out = [];
+        for (const [key, sessions] of slices) {
+            let singleTurnSessions = 0;
+            let write5m = 0;
+            let write1h = 0;
+            let assumed = 0;
+            const turnCounts = [];
+            for (const tally of sessions.values()) {
+                turnCounts.push(tally.turns);
+                if (tally.turns !== 1)
+                    continue;
+                singleTurnSessions += 1;
+                write5m += tally.write5mTokens;
+                write1h += tally.write1hTokens;
+                assumed += tally.assumedTokens;
+            }
+            const singleTurnWriteTokens = write5m + write1h + assumed;
+            // One-turn conversations that wrote nothing wasted nothing; a row about
+            // them would be a finding about the absence of a finding.
+            if (singleTurnWriteTokens === 0)
+                continue;
+            const split = key.indexOf('\n');
+            const modelId = key.slice(split + 1);
+            const model = catalogue.byId.get(modelId);
+            const { inputPerMTok } = effectivePricing(model, on);
+            const rates = multipliersFor(model);
+            const per = (tokens, rate) => (tokens / 1_000_000) * inputPerMTok * rate;
+            // The bill's own convention: unstated TTLs at the cheaper rate, so this
+            // is the same floor the totals already stand on — never a new guess.
+            const singleTurnWriteUsd = per(write5m + assumed, rates.cacheWrite5m) + per(write1h, rates.cacheWrite1h);
+            turnCounts.sort((a, b) => a - b);
+            out.push({
+                label: key.slice(0, split),
+                model: modelId,
+                modelName: model.displayName,
+                sessions: sessions.size,
+                medianTurns: median(turnCounts),
+                singleTurnSessions,
+                singleTurnWriteTokens,
+                singleTurnWriteUsd,
+                assumedTtlTokens: assumed,
+            });
+        }
+        // The most money first — the order somebody would act in.
+        return out.sort((a, b) => b.singleTurnWriteUsd - a.singleTurnWriteUsd || a.label.localeCompare(b.label));
+    };
+    return { add, finish };
+}
+/** The same measurement over a list, for a caller holding one already. */
+export function singleTurnCacheWrites(records, options) {
+    const tracker = createSessionLedgerTracker(options);
+    for (const record of records)
+        tracker.add(record);
+    return tracker.finish();
+}
+//# sourceMappingURL=session-ledger.js.map

package/dist/session-ledger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"session-ledger.js","sourceRoot":"","sources":["../src/session-ledger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAChE,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAgFxC,MAAM,MAAM,GAAG,CAAC,MAAgB,EAAU,EAAE;IAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAE,GAAG,MAAM,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,CAAC;AACxF,CAAC,CAAC;AAEF;;;;;GAKG;AACH,MAAM,UAAU,0BAA0B,CAAC,OAA6B;IACtE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,IAAI,EAAE,EAAE,GAAG,OAAO,CAAC;IAC/C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqC,CAAC;IAE5D,MAAM,GAAG,GAAG,CAAC,MAAmB,EAAQ,EAAE;QACxC,0EAA0E;QAC1E,gCAAgC;QAChC,IAAI,MAAM,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC;YAAE,OAAO;QAEzE,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,KAAK,IAAI,UAAU,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC;QAC7D,IAAI,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC/B,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,QAAQ,GAAG,IAAI,GAAG,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC5B,CAAC;QACD,IAAI,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACzC,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,KAAK,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;YAC3E,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACtC,CAAC;QACD,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;QACjB,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;YACzB,KAAK,CAAC,aAAa,IAAI,MAAM,CAAC,kBAAkB,CAAC;YACjD,KAAK,CAAC,aAAa,IAAI,MAAM,CAAC,kBAAkB,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,mEAAmE;YACnE,kEAAkE;YAClE,KAAK,CAAC,aAAa,IAAI,MAAM,CAAC,kBAAkB,CAAC;QACnD,CAAC;IACH,CAAC,CAAC;IAEF,MAAM,MAAM,GAAG,GAA4B,EAAE;QAC3C,MAAM,GAAG,GAA4B,EAAE,CAAC;QAExC,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAI,MAAM,EAAE,CAAC;YACrC,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAC3B,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,IAAI,OAAO,GAAG,CAAC,CAAC;YAChB,MAAM,UAAU,GAAa,EAAE,CAAC;YAChC,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;gBACtC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;gBAC7B,IAAI,KAAK,CAAC,KAAK,KAAK,CAAC;oBAAE,SAAS;gBAChC,kBAAkB,IAAI,CAAC,CAAC;gBACxB,OAAO,IAAI,KAAK,CAAC,aAAa,CAAC;gBAC/B,OAAO,IAAI,KAAK,CAAC,aAAa,CAAC;gBAC/B,OAAO,IAAI,KAAK,CAAC,aAAa,CAAC;YACjC,CAAC;YACD,MAAM,qBAAqB,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,CAAC;YAC1D,wEAAwE;YACxE,0DAA0D;YAC1D,IAAI,qBAAqB,KAAK,CAAC;gBAAE,SAAS;YAE1C,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;YACrC,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC;YAC3C,MAAM,EAAE,YAAY,EAAE,GAAG,gBAAgB,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,cAAc,CAAC,KAAK,CAAC,CAAC;YACpC,MAAM,GAAG,GAAG,CAAC,MAAc,EAAE,IAAY,EAAU,EAAE,CACnD,CAAC,MAAM,GAAG,SAAS,CAAC,GAAG,YAAY,GAAG,IAAI,CAAC;YAC7C,wEAAwE;YACxE,qEAAqE;YACrE,MAAM,kBAAkB,GACtB,GAAG,CAAC,OAAO,GAAG,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,GAAG,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,YAAY,CAAC,CAAC;YAEhF,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACjC,GAAG,CAAC,IAAI,CAAC;gBACP,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;gBAC1B,KAAK,EAAE,OAAO;gBACd,SAAS,EAAE,KAAK,CAAC,WAAW;gBAC5B,QAAQ,EAAE,QAAQ,CAAC,IAAI;gBACvB,WAAW,EAAE,MAAM,CAAC,UAAU,CAAC;gBAC/B,kBAAkB;gBAClB,qBAAqB;gBACrB,kBAAkB;gBAClB,gBAAgB,EAAE,OAAO;aAC1B,CAAC,CAAC;QACL,CAAC;QAED,0DAA0D;QAC1D,OAAO,GAAG,CAAC,IAAI,CACb,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,kBAAkB,IAAI,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CACxF,CAAC;IACJ,CAAC,CAAC;IAEF,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACzB,CAAC;AAED,0EAA0E;AAC1E,MAAM,UAAU,qBAAqB,CACnC,OAA+B,EAC/B,OAA6B;IAE7B,MAAM,OAAO,GAAG,0BAA0B,CAAC,OAAO,CAAC,CAAC;IACpD,KAAK,MAAM,MAAM,IAAI,OAAO;QAAE,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC,MAAM,EAAE,CAAC;AAC1B,CAAC"}

package/dist/tokenizer.d.ts

@@ -3,7 +3,7 @@
  *
  * This is NOT a real tokenizer: it is a heuristic calibrated per character
  * class. It is built to keep the typical error on ordinary text
- * (English/Spanish, markdown, code) inside ±15%, which is plenty for comparing
+ * (English/Spanish, markdown, code) inside ±10%, which is plenty for comparing
  * two versions of the same prompt — but do NOT bill anyone from it.
  *
  * **That band is a design target that has not been measured.** It is printed on
@@ -23,36 +23,43 @@
 /**
  * The error band this estimator is published under, as a percentage.
  *
- * **Measured, not chosen.** It was `15` for eight releases — a design target
- * nobody had checked — and the first run of `scripts/measure-token-band.mjs`
- * against the official counting endpoint found two of eight samples outside it,
- * both underestimating. Underestimating tokens means under-reporting cost, which
- * is the flattering direction and the worst one for this tool.
+ * **Measured, not chosen, and it has moved three times.** It was `15` for eight
+ * releases as a design target nobody had checked; the first measurement found two
+ * of eight samples outside it and it went to `25`; fixing the digit divisor and
+ * calibrating per language brought it back to `15`. This is the fourth value and
+ * the first one that is comfortably above what the corpus actually shows.
  *
- * `15` is the measured worst case rounded up, by the same rule that briefly made
- * it 25: the corpus now tops out at 11.2%, on Japanese. It landing back on the
- * number that was a guess for eight releases is a coincidence and not a
- * restoration — that 15 bounded nothing, and this one bounds eleven measured
- * samples across four languages and six text types.
+ * ```
+ * worst measured error   6.4%   (code-heavy, which nothing is fitted to)
+ * published band        10%
+ * ```
  *
- * **Read one caveat before trusting it.** Four of those samples are the Latin
- * languages whose divisors in `DIVISOR_BY_LANGUAGE` were calibrated on one or two
- * samples each, so their residuals are in-sample and optimistic by construction.
- * The band is set by the seven samples nothing was fitted to — worst 11.2% — and
- * the honest test of it is the next held-out sample in Spanish, French or German.
- * The corpus grows one sample at a time now, so that test is cheap to run.
+ * **The margin is deliberate and it is not slack.** 6.4 rounded up is 7, and
+ * publishing 7 would be a tighter claim than twenty-one samples across six text
+ * types can support: the corpus has no Korean, no Arabic, no Cyrillic prose, no
+ * mixed-script document, and a seventh text type could easily land at eight. A
+ * band that becomes false the first time somebody measures something new is the
+ * exact fault this whole exercise was fixing. Overstating the uncertainty is the
+ * safe direction for a tool that reports money.
  *
- * **What got it there was not accents.** A Spanish sample with zero accented
- * characters measured -22.9% against -22.1% for accented Spanish, which killed
- * diacritics as a signal. Languages differ in how many tokens their words cost —
- * English 3.44 characters per token, German 2.02 — so the estimator detects the
- * language and divides accordingly. See `language.ts`.
+ * What earned the drop from 15 was **splitting kana from han**. Every CJK
+ * character was charged one token, which put Japanese at +11.2% — the worst error
+ * anywhere in the corpus — while Chinese sat at −3.2% under the same rule. Kana
+ * measure 0.75 tokens per character and han 1.05, and that pair takes the two
+ * samples to −1.5% and +1.3%. See `KANA_TOKENS_PER_CHAR`.
  *
- * Exported so every report, README and tool description reads the same number.
- * It was a literal in twenty-four files before this, with the only machine-
- * readable copy in a test.
+ * **Which samples the band rests on matters more than the number.** Eight of the
+ * twenty-one had a constant fitted to them — seven Latin divisors and the digit
+ * divisor — so their residuals are optimistic by construction. The two worst
+ * errors in the corpus, `code-heavy` at 6.4% and `punctuation-heavy` at 5.7%, are
+ * fitted to nothing at all, and they are what sets this figure.
+ *
+ * Exported so every report, README and tool description reads the same number. It
+ * was a literal in twenty-four files before this, with the only machine-readable
+ * copy in a test, and `token-band.test.js` now fails any file that states a
+ * different one.
  */
-export declare const ESTIMATE_ERROR_BAND_PCT = 15;
+export declare const ESTIMATE_ERROR_BAND_PCT = 10;
 /**
  * Estimates how many tokens `text` occupies.
  *

package/dist/tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAKH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAwD1C;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CA0GnD;AAED,sDAAsD;AACtD,MAAM,MAAM,iBAAiB,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAElE,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,gFAAgF;IAChF,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,uBAAuB,GAAG,iBAAiB,CA6BxF"}
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAKH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAmG1C;;;;;;;;;;;GAWG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAuHnD;AAED,sDAAsD;AACtD,MAAM,MAAM,iBAAiB,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAElE,MAAM,WAAW,uBAAuB;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,+DAA+D;IAC/D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,gFAAgF;IAChF,aAAa,CAAC,EAAE,OAAO,CAAC;CACzB;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,uBAAuB,GAAG,iBAAiB,CA6BxF"}

package/dist/tokenizer.js

@@ -3,7 +3,7 @@
  *
  * This is NOT a real tokenizer: it is a heuristic calibrated per character
  * class. It is built to keep the typical error on ordinary text
- * (English/Spanish, markdown, code) inside ±15%, which is plenty for comparing
+ * (English/Spanish, markdown, code) inside ±10%, which is plenty for comparing
  * two versions of the same prompt — but do NOT bill anyone from it.
  *
  * **That band is a design target that has not been measured.** It is printed on
@@ -25,37 +25,65 @@ import { SAFE_FETCH_INIT, checkedEndpoint } from './net.js';
 /**
  * The error band this estimator is published under, as a percentage.
  *
- * **Measured, not chosen.** It was `15` for eight releases — a design target
- * nobody had checked — and the first run of `scripts/measure-token-band.mjs`
- * against the official counting endpoint found two of eight samples outside it,
- * both underestimating. Underestimating tokens means under-reporting cost, which
- * is the flattering direction and the worst one for this tool.
- *
- * `15` is the measured worst case rounded up, by the same rule that briefly made
- * it 25: the corpus now tops out at 11.2%, on Japanese. It landing back on the
- * number that was a guess for eight releases is a coincidence and not a
- * restoration — that 15 bounded nothing, and this one bounds eleven measured
- * samples across four languages and six text types.
- *
- * **Read one caveat before trusting it.** Four of those samples are the Latin
- * languages whose divisors in `DIVISOR_BY_LANGUAGE` were calibrated on one or two
- * samples each, so their residuals are in-sample and optimistic by construction.
- * The band is set by the seven samples nothing was fitted to — worst 11.2% — and
- * the honest test of it is the next held-out sample in Spanish, French or German.
- * The corpus grows one sample at a time now, so that test is cheap to run.
- *
- * **What got it there was not accents.** A Spanish sample with zero accented
- * characters measured -22.9% against -22.1% for accented Spanish, which killed
- * diacritics as a signal. Languages differ in how many tokens their words cost —
- * English 3.44 characters per token, German 2.02 — so the estimator detects the
- * language and divides accordingly. See `language.ts`.
- *
- * Exported so every report, README and tool description reads the same number.
- * It was a literal in twenty-four files before this, with the only machine-
- * readable copy in a test.
+ * **Measured, not chosen, and it has moved three times.** It was `15` for eight
+ * releases as a design target nobody had checked; the first measurement found two
+ * of eight samples outside it and it went to `25`; fixing the digit divisor and
+ * calibrating per language brought it back to `15`. This is the fourth value and
+ * the first one that is comfortably above what the corpus actually shows.
+ *
+ * ```
+ * worst measured error   6.4%   (code-heavy, which nothing is fitted to)
+ * published band        10%
+ * ```
+ *
+ * **The margin is deliberate and it is not slack.** 6.4 rounded up is 7, and
+ * publishing 7 would be a tighter claim than twenty-one samples across six text
+ * types can support: the corpus has no Korean, no Arabic, no Cyrillic prose, no
+ * mixed-script document, and a seventh text type could easily land at eight. A
+ * band that becomes false the first time somebody measures something new is the
+ * exact fault this whole exercise was fixing. Overstating the uncertainty is the
+ * safe direction for a tool that reports money.
+ *
+ * What earned the drop from 15 was **splitting kana from han**. Every CJK
+ * character was charged one token, which put Japanese at +11.2% — the worst error
+ * anywhere in the corpus — while Chinese sat at −3.2% under the same rule. Kana
+ * measure 0.75 tokens per character and han 1.05, and that pair takes the two
+ * samples to −1.5% and +1.3%. See `KANA_TOKENS_PER_CHAR`.
+ *
+ * **Which samples the band rests on matters more than the number.** Eight of the
+ * twenty-one had a constant fitted to them — seven Latin divisors and the digit
+ * divisor — so their residuals are optimistic by construction. The two worst
+ * errors in the corpus, `code-heavy` at 6.4% and `punctuation-heavy` at 5.7%, are
+ * fitted to nothing at all, and they are what sets this figure.
+ *
+ * Exported so every report, README and tool description reads the same number. It
+ * was a literal in twenty-four files before this, with the only machine-readable
+ * copy in a test, and `token-band.test.js` now fails any file that states a
+ * different one.
  */
-export const ESTIMATE_ERROR_BAND_PCT = 15;
+export const ESTIMATE_ERROR_BAND_PCT = 10;
 const CJK = /[぀-ヿ㐀-䶿一-鿿가-힯]/;
+/**
+ * Kana, separated from the rest of CJK because they do not cost the same.
+ *
+ * **This was the largest error left in the corpus.** Every CJK character was
+ * charged one token, and measured against the counting endpoint that put Japanese
+ * at **+11.2%** — the worst figure anywhere in twenty-one samples — while Chinese
+ * came out at −3.2% under the identical rule. One constant cannot be right for
+ * both, and the reason is visible in the samples: the Japanese one is 58% kana and
+ * the Chinese one is 0%.
+ *
+ * Kana are a small syllabary that appears in every sentence, so the merge table
+ * covers runs of them and several characters share a token. Han are tens of
+ * thousands of rare characters; a merge table cannot cover them and they cost
+ * about one each, sometimes more.
+ *
+ * The signal needs no detector. A character is kana or it is not, and the two
+ * samples separate perfectly — 58.3% against 0.00% — so this is a property of the
+ * character rather than a guess about the document. That is the difference between
+ * this and `language.ts`, which has to decide and is allowed to refuse.
+ */
+const KANA = /[぀-ヿ]/;
 const LETTER = /[A-Za-zÀ-ɏͰ-ϿЀ-ӿ]/;
 const DIGIT = /[0-9]/;
 /**
@@ -90,6 +118,26 @@ const DIVISOR_BY_LANGUAGE = {
     pt: 3.05,
     nl: 2.65,
 };
+/**
+ * Tokens per character for the two halves of CJK, measured.
+ *
+ * `0.75` and `1.05` come from a search over the two CJK samples in
+ * `test/fixtures/token-ground-truth.json`, and they take that pair from
+ * +11.2% / −3.2% to −1.5% / +1.3%.
+ *
+ * **Two samples fitted two constants, so those residuals are in-sample and
+ * optimistic by construction** — the same caveat the Latin divisors carry, stated
+ * for the same reason. What makes them worth having anyway is the size of the
+ * error they replace and the fact that they move in opposite directions: a single
+ * constant could not have been within four points of both, whatever it was set to.
+ * The honest test is a third CJK sample, and the corpus grows one at a time.
+ *
+ * Hangul keeps the old cost of 1, because nothing here measures Korean. Guessing
+ * it from Japanese would be inventing a figure — the two scripts have nothing in
+ * common that would make one predict the other.
+ */
+const KANA_TOKENS_PER_CHAR = 0.75;
+const HAN_TOKENS_PER_CHAR = 1.05;
 /**
  * What a prompt gets when the language could not be told.
  *
@@ -126,6 +174,11 @@ export function estimateTokens(text) {
     const language = detectTextLanguage(text);
     const divisor = (language === null ? undefined : DIVISOR_BY_LANGUAGE[language]) ?? DEFAULT_DIVISOR;
     let total = 0;
+    /**
+     * CJK is summed separately because it is the one class counted in fractions of
+     * a token. Everything else is whole tokens by the character class it belongs to.
+     */
+    let cjkTokens = 0;
     let i = 0;
     const chars = Array.from(text);
     while (i < chars.length) {
@@ -144,12 +197,19 @@ export function estimateTokens(text) {
             continue;
         }
         if (CJK.test(ch)) {
-            let n = 0;
+            /**
+             * Accumulated as a fraction and rounded once, at the end.
+             *
+             * Rounding up per run was the first attempt and it was wrong by five
+             * points. Ordinary Japanese alternates kana and han inside every sentence,
+             * so the runs are short and there are many of them — and a `Math.ceil` per
+             * run charges most of a token for each boundary. That is an artefact of
+             * where the loop happens to break, not of what the text costs.
+             */
             while (i < chars.length && CJK.test(chars[i])) {
-                n++;
+                cjkTokens += KANA.test(chars[i]) ? KANA_TOKENS_PER_CHAR : HAN_TOKENS_PER_CHAR;
                 i++;
             }
-            total += n;
             continue;
         }
         if (LETTER.test(ch)) {
@@ -212,7 +272,8 @@ export function estimateTokens(text) {
             total += Math.ceil(n / 2);
         }
     }
-    return total;
+    // Rounded once, over the whole document, rather than per run — see the CJK branch.
+    return total + Math.ceil(cjkTokens);
 }
 /**
  * Exact counter using the official `/v1/messages/count_tokens` endpoint.

package/dist/tokenizer.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAE1C,MAAM,GAAG,GAAG,gBAAgB,CAAC;AAC7B,MAAM,MAAM,GAAG,mBAAmB,CAAC;AACnC,MAAM,KAAK,GAAG,OAAO,CAAC;AAEtB;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,mBAAmB,GAAqC;IAC5D,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;CACT,CAAC;AAEF;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,0EAA0E;AAC1E,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,EAAE,IAAI,IAAI;QAAE,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,8EAA8E;IAC9E,8DAA8D;IAC9D,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,IAAI,eAAe,CAAC;IAEnG,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE/B,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QAErB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAC7C,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC7C,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACjB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBAC/C,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,CAAC,CAAC;YACX,SAAS;QACX,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACpB,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,EAAE,CAAC;gBAC7E,IAAI,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC;gBAClB,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YACjE,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBACjD,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD;;;;;;;;;;;;eAYG;YACH,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,+DAA+D;QAC/D,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAE,GAAG,MAAM,EAAE,CAAC;YAChC,KAAK,IAAI,CAAC,CAAC;YACX,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,iCAAiC;QACjC,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OACE,CAAC,GAAG,KAAK,CAAC,MAAM;YAChB,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACvB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACtB,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACpB,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG;YAChB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAE,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,MAAM,EACnC,CAAC;YACD,CAAC,EAAE,CAAC;YACJ,CAAC,EAAE,CAAC;QACN,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,8EAA8E;YAC9E,KAAK,IAAI,CAAC,CAAC;YACX,CAAC,EAAE,CAAC;QACN,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAeD;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAgC;IACnE,MAAM,EACJ,MAAM,EACN,KAAK,GAAG,eAAe,EACvB,OAAO,GAAG,2BAA2B,EACrC,SAAS,GAAG,KAAK,EACjB,aAAa,GAAG,KAAK,GACtB,GAAG,OAAO,CAAC;IAEZ,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAE7F,OAAO,KAAK,EAAE,IAAY,EAAmB,EAAE;QAC7C,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,CAAC,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,QAAQ,2BAA2B,EAAE;YAClE,GAAG,eAAe;YAClB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,WAAW,EAAE,MAAM;gBACnB,mBAAmB,EAAE,YAAY;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;SAC7E,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA6B,CAAC;QAC5D,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC,CAAC;AACJ,CAAC"}
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../src/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AAE5D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAE1C,MAAM,GAAG,GAAG,gBAAgB,CAAC;AAE7B;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,IAAI,GAAG,OAAO,CAAC;AACrB,MAAM,MAAM,GAAG,mBAAmB,CAAC;AACnC,MAAM,KAAK,GAAG,OAAO,CAAC;AAEtB;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,mBAAmB,GAAqC;IAC5D,EAAE,EAAE,CAAC;IACL,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,GAAG;IACP,EAAE,EAAE,IAAI;IACR,EAAE,EAAE,IAAI;CACT,CAAC;AAEF;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,oBAAoB,GAAG,IAAI,CAAC;AAClC,MAAM,mBAAmB,GAAG,IAAI,CAAC;AAEjC;;;;;;;GAOG;AACH,MAAM,eAAe,GAAG,CAAC,CAAC;AAE1B,0EAA0E;AAC1E,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,MAAM,EAAE,IAAI,IAAI;QAAE,GAAG,IAAI,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,8EAA8E;IAC9E,8DAA8D;IAC9D,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,OAAO,GAAG,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,mBAAmB,CAAC,QAAQ,CAAC,CAAC,IAAI,eAAe,CAAC;IAEnG,IAAI,KAAK,GAAG,CAAC,CAAC;IACd;;;OAGG;IACH,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE/B,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;QACxB,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;QAErB,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAC7C,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBAC7C,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACjB;;;;;;;;eAQG;YACH,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBAC/C,SAAS,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,mBAAmB,CAAC;gBAC/E,CAAC,EAAE,CAAC;YACN,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACpB,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,EAAE,CAAC;gBAC7E,IAAI,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC;gBAClB,CAAC,EAAE,CAAC;YACN,CAAC;YACD,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC;YACjE,SAAS;QACX,CAAC;QAED,IAAI,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,GAAG,CAAC,CAAC;YACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;gBACjD,CAAC,EAAE,CAAC;gBACJ,CAAC,EAAE,CAAC;YACN,CAAC;YACD;;;;;;;;;;;;eAYG;YACH,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC;YAC5B,SAAS;QACX,CAAC;QAED,+DAA+D;QAC/D,IAAI,EAAE,CAAC,WAAW,CAAC,CAAC,CAAE,GAAG,MAAM,EAAE,CAAC;YAChC,KAAK,IAAI,CAAC,CAAC;YACX,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,iCAAiC;QACjC,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,OACE,CAAC,GAAG,KAAK,CAAC,MAAM;YAChB,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACvB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACtB,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC;YACpB,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG;YAChB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAC,KAAK,IAAI;YACjB,KAAK,CAAC,CAAC,CAAE,CAAC,WAAW,CAAC,CAAC,CAAE,IAAI,MAAM,EACnC,CAAC;YACD,CAAC,EAAE,CAAC;YACJ,CAAC,EAAE,CAAC;QACN,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YACZ,8EAA8E;YAC9E,KAAK,IAAI,CAAC,CAAC;YACX,CAAC,EAAE,CAAC;QACN,CAAC;aAAM,CAAC;YACN,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IAED,mFAAmF;IACnF,OAAO,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACtC,CAAC;AAeD;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAgC;IACnE,MAAM,EACJ,MAAM,EACN,KAAK,GAAG,eAAe,EACvB,OAAO,GAAG,2BAA2B,EACrC,SAAS,GAAG,KAAK,EACjB,aAAa,GAAG,KAAK,GACtB,GAAG,OAAO,CAAC;IAEZ,MAAM,QAAQ,GAAG,eAAe,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,wBAAwB,EAAE,CAAC,CAAC;IAE7F,OAAO,KAAK,EAAE,IAAY,EAAmB,EAAE;QAC7C,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAAE,OAAO,CAAC,CAAC;QAC3B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,GAAG,QAAQ,2BAA2B,EAAE;YAClE,GAAG,eAAe;YAClB,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,WAAW,EAAE,MAAM;gBACnB,mBAAmB,EAAE,YAAY;aAClC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;SAC7E,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,MAAM,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA6B,CAAC;QAC5D,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC,CAAC;AACJ,CAAC"}