open-multi-agent-kit 0.80.7 → 0.80.8

package/dist/core/context-budget-token-counter.js

@@ -0,0 +1,315 @@
+import { createRequire } from "node:module";
+const requireModule = createRequire(import.meta.url);
+export function createNodeOptionalModuleLoader() {
+    return {
+        resolve(specifier) {
+            try {
+                return requireModule.resolve(specifier);
+            }
+            catch {
+                return undefined;
+            }
+        },
+        load(specifier) {
+            return requireModule(specifier);
+        },
+    };
+}
+export function createFallbackTokenCounter() {
+    return {
+        id: "fallback-estimator",
+        priority: 0,
+        isAvailable: () => true,
+        supports: () => true,
+        countText(input, modelId) {
+            return estimateTextTokens(input, modelId);
+        },
+    };
+}
+export function estimateTextTokens(input, modelId = "unknown") {
+    if (input.length === 0) {
+        return createTokenResult(0, "estimated", "medium", "fallback-estimator", modelId, ["empty-input"]);
+    }
+    let asciiWord = 0;
+    let whitespace = 0;
+    let cjk = 0;
+    let hangul = 0;
+    let kana = 0;
+    let emojiOrWide = 0;
+    let punctuation = 0;
+    let other = 0;
+    for (const char of input) {
+        const codePoint = char.codePointAt(0) ?? 0;
+        if (/\s/u.test(char)) {
+            whitespace += 1;
+        }
+        else if (isHangul(codePoint)) {
+            hangul += 1;
+        }
+        else if (isHiraganaOrKatakana(codePoint)) {
+            kana += 1;
+        }
+        else if (isCjkIdeograph(codePoint)) {
+            cjk += 1;
+        }
+        else if (isAsciiAlphaNumeric(codePoint) || char === "_") {
+            asciiWord += 1;
+        }
+        else if (isEmojiOrWideSymbol(codePoint)) {
+            emojiOrWide += 1;
+        }
+        else if (isAsciiPunctuation(codePoint)) {
+            punctuation += 1;
+        }
+        else {
+            other += 1;
+        }
+    }
+    const codeLike = looksCodeLike(input, punctuation, whitespace);
+    const jsonLike = looksJsonLike(input);
+    const nonAsciiRatio = (hangul + cjk + kana + emojiOrWide + other) / Math.max(1, input.length);
+    const base = asciiWord / (codeLike ? 3.2 : 4) +
+        whitespace / 12 +
+        punctuation / 2.1 +
+        hangul / 0.95 +
+        kana / 1.0 +
+        cjk / 1.2 +
+        emojiOrWide * 1.8 +
+        other / 2;
+    const adjusted = base * (jsonLike ? 1.12 : 1) * (codeLike ? 1.08 : 1);
+    const tokens = Math.max(1, Math.ceil(adjusted));
+    const compositionParts = [];
+    if (hangul > 0)
+        compositionParts.push(`hangul:${hangul}`);
+    if (kana > 0)
+        compositionParts.push(`kana:${kana}`);
+    if (cjk > 0)
+        compositionParts.push(`cjk:${cjk}`);
+    if (asciiWord > 0)
+        compositionParts.push(`ascii:${asciiWord}`);
+    if (emojiOrWide > 0)
+        compositionParts.push(`emoji:${emojiOrWide}`);
+    const notes = [
+        codeLike ? "code-like" : "prose-like",
+        jsonLike ? "json-like" : "not-json-like",
+        nonAsciiRatio > 0 ? `non-ascii:${(nonAsciiRatio * 100).toFixed(0)}%` : "latin-only",
+        `composition(${compositionParts.join(",")})`,
+    ];
+    const confidence = nonAsciiRatio > 0.4 ? "low" : nonAsciiRatio > 0.15 ? "medium" : "high";
+    return createTokenResult(tokens, "estimated", confidence, "fallback-estimator", modelId, notes);
+}
+export function createOpenAiJsTokenCounter(loader = createNodeOptionalModuleLoader()) {
+    const packageNames = ["js-tiktoken", "gpt-tokenizer", "tiktoken"];
+    return {
+        id: "openai-bpe-js",
+        priority: 80,
+        isAvailable() {
+            return packageNames.some((specifier) => loader.resolve(specifier) !== undefined);
+        },
+        supports(modelId) {
+            return isOpenAiStyleModel(modelId);
+        },
+        countText(input, modelId) {
+            for (const specifier of packageNames) {
+                if (loader.resolve(specifier) === undefined) {
+                    continue;
+                }
+                const result = countWithOptionalTokenizerModule(loader.load(specifier), specifier, input, modelId);
+                if (result !== undefined) {
+                    return result;
+                }
+            }
+            throw new Error("no supported OpenAI JS tokenizer module shape found");
+        },
+    };
+}
+export function createOpenAiWasmTokenCounter(loader = createNodeOptionalModuleLoader()) {
+    const packageNames = ["@dqbd/tiktoken", "tiktoken"];
+    return {
+        id: "openai-bpe-wasm",
+        priority: 90,
+        isAvailable() {
+            return packageNames.some((specifier) => loader.resolve(specifier) !== undefined);
+        },
+        supports(modelId) {
+            return isOpenAiStyleModel(modelId);
+        },
+        countText(input, modelId) {
+            for (const specifier of packageNames) {
+                if (loader.resolve(specifier) === undefined) {
+                    continue;
+                }
+                const result = countWithOptionalTokenizerModule(loader.load(specifier), specifier, input, modelId);
+                if (result !== undefined) {
+                    return result;
+                }
+            }
+            throw new Error("no supported OpenAI WASM tokenizer module shape found");
+        },
+    };
+}
+export function createTokenCounterForMode(mode, loader = createNodeOptionalModuleLoader()) {
+    const fallback = createFallbackTokenCounter();
+    if (mode === "fallback") {
+        return fallback;
+    }
+    const adapters = mode === "openai-wasm" ? [createOpenAiWasmTokenCounter(loader)] : [createOpenAiJsTokenCounter(loader)];
+    if (mode === "auto") {
+        adapters.push(createOpenAiWasmTokenCounter(loader));
+    }
+    return createTokenCounterRegistry({ adapters, fallback });
+}
+export function createTokenCounterRegistry(options = {}) {
+    const fallback = options.fallback ?? createFallbackTokenCounter();
+    const adapters = [...(options.adapters ?? [])].sort((a, b) => b.priority - a.priority || a.id.localeCompare(b.id));
+    return {
+        id: "token-counter-registry",
+        priority: 100,
+        isAvailable: () => true,
+        supports: () => true,
+        countText(input, modelId) {
+            const notes = [];
+            for (const adapter of adapters) {
+                if (!adapter.supports(modelId)) {
+                    continue;
+                }
+                try {
+                    if (!adapter.isAvailable()) {
+                        notes.push(`${adapter.id}:unavailable`);
+                        continue;
+                    }
+                    return adapter.countText(input, modelId);
+                }
+                catch (error) {
+                    const message = error instanceof Error ? error.message : "unknown adapter failure";
+                    notes.push(`${adapter.id}:failed:${message}`);
+                }
+            }
+            const result = fallback.countText(input, modelId);
+            return { ...result, notes: [...notes, ...result.notes] };
+        },
+    };
+}
+function countWithOptionalTokenizerModule(moduleValue, specifier, input, modelId) {
+    const moduleObject = unwrapDefaultModule(moduleValue);
+    const jsTiktoken = moduleObject;
+    if (typeof jsTiktoken.encodingForModel === "function") {
+        const encoding = jsTiktoken.encodingForModel(modelId);
+        return countWithEncoding(encoding, specifier, input, modelId, "model-encoding");
+    }
+    if (typeof jsTiktoken.getEncoding === "function") {
+        const encoding = jsTiktoken.getEncoding(selectOpenAiEncoding(modelId));
+        return countWithEncoding(encoding, specifier, input, modelId, "fallback-encoding");
+    }
+    const generic = moduleObject;
+    if (typeof generic.encode === "function") {
+        return createTokenResult(generic.encode(input).length, "exact", "medium", specifier, modelId, ["generic-encode"]);
+    }
+    return undefined;
+}
+function countWithEncoding(encoding, adapterId, input, modelId, note) {
+    return createTokenResult(encoding.encode(input).length, "exact", "high", adapterId, modelId, [note]);
+}
+function unwrapDefaultModule(moduleValue) {
+    if (moduleValue && typeof moduleValue === "object" && "default" in moduleValue) {
+        return moduleValue.default;
+    }
+    return moduleValue;
+}
+function createTokenResult(tokens, method, confidence, adapterId, modelId, notes) {
+    return {
+        tokens: Math.max(0, Math.ceil(tokens)),
+        method,
+        confidence,
+        adapterId,
+        modelId,
+        notes,
+    };
+}
+function isOpenAiStyleModel(modelId) {
+    const normalized = modelId.toLowerCase();
+    return (normalized.includes("openai") ||
+        normalized.startsWith("gpt-") ||
+        normalized.startsWith("o1") ||
+        normalized.startsWith("o3") ||
+        normalized.startsWith("o4") ||
+        normalized.includes("chatgpt"));
+}
+function selectOpenAiEncoding(modelId) {
+    const normalized = modelId.toLowerCase();
+    if (normalized.includes("gpt-4o") || normalized.includes("gpt-5") || normalized.startsWith("o")) {
+        return "o200k_base";
+    }
+    return "cl100k_base";
+}
+function isAsciiAlphaNumeric(codePoint) {
+    return ((codePoint >= 48 && codePoint <= 57) ||
+        (codePoint >= 65 && codePoint <= 90) ||
+        (codePoint >= 97 && codePoint <= 122));
+}
+function isAsciiPunctuation(codePoint) {
+    return codePoint >= 33 && codePoint <= 126;
+}
+function isHangul(codePoint) {
+    // Composed syllables (가-힣)
+    if (codePoint >= 0xac00 && codePoint <= 0xd7af)
+        return true;
+    // Jamo: initial (ㄱ-ㅎ), medial (ㅏ-ㅣ)
+    if (codePoint >= 0x1100 && codePoint <= 0x11ff)
+        return true;
+    // Compatibility jamo (ㄱ-ㅎ, ㅏ-ㅣ) and Hangul letters
+    if (codePoint >= 0x3130 && codePoint <= 0x318f)
+        return true;
+    // Extended jamo
+    if (codePoint >= 0xa960 && codePoint <= 0xa97c)
+        return true;
+    return false;
+}
+function isHiraganaOrKatakana(codePoint) {
+    // Hiragana (ぁ-より)
+    if (codePoint >= 0x3040 && codePoint <= 0x309f)
+        return true;
+    // Katakana (ァ-ヿ) + halfwidth katakana
+    if (codePoint >= 0x30a0 && codePoint <= 0x30ff)
+        return true;
+    if (codePoint >= 0xff65 && codePoint <= 0xff9f)
+        return true;
+    return false;
+}
+function isCjkIdeograph(codePoint) {
+    // CJK Unified Ideographs (main block: 中, 国, etc.)
+    if (codePoint >= 0x4e00 && codePoint <= 0x9fff)
+        return true;
+    // CJK Extension A (rare)
+    if (codePoint >= 0x3400 && codePoint <= 0x4dbf)
+        return true;
+    // CJK compatibility ideographs
+    if (codePoint >= 0xf900 && codePoint <= 0xfaff)
+        return true;
+    return false;
+}
+function isEmojiOrWideSymbol(codePoint) {
+    // Main emoji blocks
+    if (codePoint >= 0x1f000)
+        return true;
+    // Misc symbols, dingbats
+    if (codePoint >= 0x2600 && codePoint <= 0x27bf)
+        return true;
+    // CJK fullwidth/special forms that occupy double width
+    if (codePoint >= 0xff01 && codePoint <= 0xff60)
+        return true;
+    if (codePoint >= 0xffe0 && codePoint <= 0xffe6)
+        return true;
+    return false;
+}
+function looksJsonLike(input) {
+    const trimmed = input.trim();
+    return (trimmed.startsWith("{") && trimmed.endsWith("}")) || (trimmed.startsWith("[") && trimmed.endsWith("]"));
+}
+function looksCodeLike(input, punctuation, whitespace) {
+    if (/\b(function|const|let|class|interface|import|export|return|async|await)\b/.test(input)) {
+        return true;
+    }
+    return punctuation > input.length / 8 && whitespace > input.length / 20;
+}
+//# sourceMappingURL=context-budget-token-counter.js.map

package/dist/core/context-budget-token-counter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-budget-token-counter.js","sourceRoot":"","sources":["../../src/core/context-budget-token-counter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AA8C5C,MAAM,aAAa,GAAG,aAAa,CAAC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC;AAErD,MAAM,UAAU,8BAA8B,GAAyB;IACtE,OAAO;QACN,OAAO,CAAC,SAAS,EAAE;YAClB,IAAI,CAAC;gBACJ,OAAO,aAAa,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACzC,CAAC;YAAC,MAAM,CAAC;gBACR,OAAO,SAAS,CAAC;YAClB,CAAC;QAAA,CACD;QACD,IAAI,CAAC,SAAS,EAAE;YACf,OAAO,aAAa,CAAC,SAAS,CAAY,CAAC;QAAA,CAC3C;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,0BAA0B,GAAwB;IACjE,OAAO;QACN,EAAE,EAAE,oBAAoB;QACxB,QAAQ,EAAE,CAAC;QACX,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI;QACvB,QAAQ,EAAE,GAAG,EAAE,CAAC,IAAI;QACpB,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE;YACzB,OAAO,kBAAkB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAAA,CAC1C;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,kBAAkB,CAAC,KAAa,EAAE,OAAO,GAAG,SAAS,EAAoB;IACxF,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,iBAAiB,CAAC,CAAC,EAAE,WAAW,EAAE,QAAQ,EAAE,oBAAoB,EAAE,OAAO,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;IACpG,CAAC;IAED,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACtB,UAAU,IAAI,CAAC,CAAC;QACjB,CAAC;aAAM,IAAI,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,oBAAoB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC5C,IAAI,IAAI,CAAC,CAAC;QACX,CAAC;aAAM,IAAI,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YACtC,GAAG,IAAI,CAAC,CAAC;QACV,CAAC;aAAM,IAAI,mBAAmB,CAAC,SAAS,CAAC,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YAC3D,SAAS,IAAI,CAAC,CAAC;QAChB,CAAC;aAAM,IAAI,mBAAmB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC3C,WAAW,IAAI,CAAC,CAAC;QAClB,CAAC;aAAM,IAAI,kBAAkB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1C,WAAW,IAAI,CAAC,CAAC;QAClB,CAAC;aAAM,CAAC;YACP,KAAK,IAAI,CAAC,CAAC;QACZ,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,EAAE,WAAW,EAAE,UAAU,CAAC,CAAC;IAC/D,MAAM,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,aAAa,GAAG,CAAC,MAAM,GAAG,GAAG,GAAG,IAAI,GAAG,WAAW,GAAG,KAAK,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9F,MAAM,IAAI,GACT,SAAS,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAChC,UAAU,GAAG,EAAE;QACf,WAAW,GAAG,GAAG;QACjB,MAAM,GAAG,IAAI;QACb,IAAI,GAAG,GAAG;QACV,GAAG,GAAG,GAAG;QACT,WAAW,GAAG,GAAG;QACjB,KAAK,GAAG,CAAC,CAAC;IACX,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChD,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,IAAI,MAAM,GAAG,CAAC;QAAE,gBAAgB,CAAC,IAAI,CAAC,UAAU,MAAM,EAAE,CAAC,CAAC;IAC1D,IAAI,IAAI,GAAG,CAAC;QAAE,gBAAgB,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC;IACpD,IAAI,GAAG,GAAG,CAAC;QAAE,gBAAgB,CAAC,IAAI,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IACjD,IAAI,SAAS,GAAG,CAAC;QAAE,gBAAgB,CAAC,IAAI,CAAC,SAAS,SAAS,EAAE,CAAC,CAAC;IAC/D,IAAI,WAAW,GAAG,CAAC;QAAE,gBAAgB,CAAC,IAAI,CAAC,SAAS,WAAW,EAAE,CAAC,CAAC;IACnE,MAAM,KAAK,GAAG;QACb,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,YAAY;QACrC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,eAAe;QACxC,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY;QACnF,eAAe,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG;KAC5C,CAAC;IACF,MAAM,UAAU,GACf,aAAa,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;IACxE,OAAO,iBAAiB,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,oBAAoB,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;AAAA,CAChG;AAED,MAAM,UAAU,0BAA0B,CACzC,MAAM,GAAyB,8BAA8B,EAAE,EACzC;IACtB,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,eAAe,EAAE,UAAU,CAAU,CAAC;IAC3E,OAAO;QACN,EAAE,EAAE,eAAe;QACnB,QAAQ,EAAE,EAAE;QACZ,WAAW,GAAG;YACb,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,SAAS,CAAC,CAAC;QAAA,CACjF;QACD,QAAQ,CAAC,OAAO,EAAE;YACjB,OAAO,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAAA,CACnC;QACD,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE;YACzB,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;gBACtC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE,CAAC;oBAC7C,SAAS;gBACV,CAAC;gBACD,MAAM,MAAM,GAAG,gCAAgC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;gBACnG,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBAC1B,OAAO,MAAM,CAAC;gBACf,CAAC;YACF,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QAAA,CACvE;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,4BAA4B,CAC3C,MAAM,GAAyB,8BAA8B,EAAE,EACzC;IACtB,MAAM,YAAY,GAAG,CAAC,gBAAgB,EAAE,UAAU,CAAU,CAAC;IAC7D,OAAO;QACN,EAAE,EAAE,iBAAiB;QACrB,QAAQ,EAAE,EAAE;QACZ,WAAW,GAAG;YACb,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,SAAS,CAAC,CAAC;QAAA,CACjF;QACD,QAAQ,CAAC,OAAO,EAAE;YACjB,OAAO,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAAA,CACnC;QACD,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE;YACzB,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;gBACtC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,SAAS,EAAE,CAAC;oBAC7C,SAAS;gBACV,CAAC;gBACD,MAAM,MAAM,GAAG,gCAAgC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;gBACnG,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBAC1B,OAAO,MAAM,CAAC;gBACf,CAAC;YACF,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAAA,CACzE;KACD,CAAC;AAAA,CACF;AAED,MAAM,UAAU,yBAAyB,CACxC,IAAgC,EAChC,MAAM,GAAyB,8BAA8B,EAAE,EACzC;IACtB,MAAM,QAAQ,GAAG,0BAA0B,EAAE,CAAC;IAC9C,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACzB,OAAO,QAAQ,CAAC;IACjB,CAAC;IACD,MAAM,QAAQ,GACb,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,0BAA0B,CAAC,MAAM,CAAC,CAAC,CAAC;IACxG,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;QACrB,QAAQ,CAAC,IAAI,CAAC,4BAA4B,CAAC,MAAM,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,0BAA0B,CAAC,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC;AAAA,CAC1D;AAED,MAAM,UAAU,0BAA0B,CAAC,OAAO,GAAgC,EAAE,EAAuB;IAC1G,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,0BAA0B,EAAE,CAAC;IAClE,MAAM,QAAQ,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACnH,OAAO;QACN,EAAE,EAAE,wBAAwB;QAC5B,QAAQ,EAAE,GAAG;QACb,WAAW,EAAE,GAAG,EAAE,CAAC,IAAI;QACvB,QAAQ,EAAE,GAAG,EAAE,CAAC,IAAI;QACpB,SAAS,CAAC,KAAK,EAAE,OAAO,EAAE;YACzB,MAAM,KAAK,GAAa,EAAE,CAAC;YAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;gBAChC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBAChC,SAAS;gBACV,CAAC;gBACD,IAAI,CAAC;oBACJ,IAAI,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC;wBAC5B,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;wBACxC,SAAS;oBACV,CAAC;oBACD,OAAO,OAAO,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;gBAC1C,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,yBAAyB,CAAC;oBACnF,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,EAAE,WAAW,OAAO,EAAE,CAAC,CAAC;gBAC/C,CAAC;YACF,CAAC;YACD,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAClD,OAAO,EAAE,GAAG,MAAM,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,EAAE,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;QAAA,CACzD;KACD,CAAC;AAAA,CACF;AAED,SAAS,gCAAgC,CACxC,WAAoB,EACpB,SAAiB,EACjB,KAAa,EACb,OAAe,EACgB;IAC/B,MAAM,YAAY,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,YAAgC,CAAC;IACpD,IAAI,OAAO,UAAU,CAAC,gBAAgB,KAAK,UAAU,EAAE,CAAC;QACvD,MAAM,QAAQ,GAAG,UAAU,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QACtD,OAAO,iBAAiB,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,gBAAgB,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,OAAO,UAAU,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;QAClD,MAAM,QAAQ,GAAG,UAAU,CAAC,WAAW,CAAC,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC;QACvE,OAAO,iBAAiB,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,mBAAmB,CAAC,CAAC;IACpF,CAAC;IACD,MAAM,OAAO,GAAG,YAAmC,CAAC;IACpD,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;QAC1C,OAAO,iBAAiB,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC;IACnH,CAAC;IACD,OAAO,SAAS,CAAC;AAAA,CACjB;AAED,SAAS,iBAAiB,CACzB,QAAuB,EACvB,SAAiB,EACjB,KAAa,EACb,OAAe,EACf,IAAY,EACO;IACnB,OAAO,iBAAiB,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;AAAA,CACrG;AAED,SAAS,mBAAmB,CAAC,WAAoB,EAAW;IAC3D,IAAI,WAAW,IAAI,OAAO,WAAW,KAAK,QAAQ,IAAI,SAAS,IAAI,WAAW,EAAE,CAAC;QAChF,OAAQ,WAA6C,CAAC,OAAO,CAAC;IAC/D,CAAC;IACD,OAAO,WAAW,CAAC;AAAA,CACnB;AAED,SAAS,iBAAiB,CACzB,MAAc,EACd,MAAqC,EACrC,UAAwC,EACxC,SAAiB,EACjB,OAAe,EACf,KAAwB,EACL;IACnB,OAAO;QACN,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM;QACN,UAAU;QACV,SAAS;QACT,OAAO;QACP,KAAK;KACL,CAAC;AAAA,CACF;AAED,SAAS,kBAAkB,CAAC,OAAe,EAAW;IACrD,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACzC,OAAO,CACN,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAC7B,UAAU,CAAC,UAAU,CAAC,MAAM,CAAC;QAC7B,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC;QAC3B,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC;QAC3B,UAAU,CAAC,UAAU,CAAC,IAAI,CAAC;QAC3B,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,CAC9B,CAAC;AAAA,CACF;AAED,SAAS,oBAAoB,CAAC,OAAe,EAAU;IACtD,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IACzC,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,UAAU,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACjG,OAAO,YAAY,CAAC;IACrB,CAAC;IACD,OAAO,aAAa,CAAC;AAAA,CACrB;AAED,SAAS,mBAAmB,CAAC,SAAiB,EAAW;IACxD,OAAO,CACN,CAAC,SAAS,IAAI,EAAE,IAAI,SAAS,IAAI,EAAE,CAAC;QACpC,CAAC,SAAS,IAAI,EAAE,IAAI,SAAS,IAAI,EAAE,CAAC;QACpC,CAAC,SAAS,IAAI,EAAE,IAAI,SAAS,IAAI,GAAG,CAAC,CACrC,CAAC;AAAA,CACF;AAED,SAAS,kBAAkB,CAAC,SAAiB,EAAW;IACvD,OAAO,SAAS,IAAI,EAAE,IAAI,SAAS,IAAI,GAAG,CAAC;AAAA,CAC3C;AAED,SAAS,QAAQ,CAAC,SAAiB,EAAW;IAC7C,+BAA2B;IAC3B,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,4CAAoC;IACpC,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,2DAAmD;IACnD,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,gBAAgB;IAChB,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,OAAO,KAAK,CAAC;AAAA,CACb;AAED,SAAS,oBAAoB,CAAC,SAAiB,EAAW;IACzD,wBAAkB;IAClB,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,0CAAsC;IACtC,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,OAAO,KAAK,CAAC;AAAA,CACb;AAED,SAAS,cAAc,CAAC,SAAiB,EAAW;IACnD,sDAAkD;IAClD,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,yBAAyB;IACzB,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,+BAA+B;IAC/B,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,OAAO,KAAK,CAAC;AAAA,CACb;AAED,SAAS,mBAAmB,CAAC,SAAiB,EAAW;IACxD,oBAAoB;IACpB,IAAI,SAAS,IAAI,OAAO;QAAE,OAAO,IAAI,CAAC;IACtC,yBAAyB;IACzB,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,uDAAuD;IACvD,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,IAAI,SAAS,IAAI,MAAM,IAAI,SAAS,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IAC5D,OAAO,KAAK,CAAC;AAAA,CACb;AAED,SAAS,aAAa,CAAC,KAAa,EAAW;IAC9C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;AAAA,CAChH;AAED,SAAS,aAAa,CAAC,KAAa,EAAE,WAAmB,EAAE,UAAkB,EAAW;IACvF,IAAI,2EAA2E,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7F,OAAO,IAAI,CAAC;IACb,CAAC;IACD,OAAO,WAAW,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,EAAE,CAAC;AAAA,CACxE","sourcesContent":["import { createRequire } from \"node:module\";\n\nexport type ContextBudgetTokenCountMethod = \"exact\" | \"estimated\";\nexport type ContextBudgetTokenConfidence = \"high\" | \"medium\" | \"low\";\nexport type ContextBudgetTokenizerMode = \"auto\" | \"fallback\" | \"openai-js\" | \"openai-wasm\";\n\nexport interface TokenCountResult {\n\treadonly tokens: number;\n\treadonly method: ContextBudgetTokenCountMethod;\n\treadonly confidence: ContextBudgetTokenConfidence;\n\treadonly adapterId: string;\n\treadonly modelId: string;\n\treadonly notes: readonly string[];\n}\n\nexport interface TokenCounterAdapter {\n\treadonly id: string;\n\treadonly priority: number;\n\tisAvailable(): boolean;\n\tsupports(modelId: string): boolean;\n\tcountText(input: string, modelId: string): TokenCountResult;\n}\n\nexport interface OptionalModuleLoader {\n\tresolve(specifier: string): string | undefined;\n\tload(specifier: string): unknown;\n}\n\nexport interface TokenCounterRegistryOptions {\n\treadonly adapters?: readonly TokenCounterAdapter[];\n\treadonly fallback?: TokenCounterAdapter;\n}\n\ninterface EncodeCapable {\n\tencode(input: string): readonly unknown[];\n}\n\ninterface JsTiktokenModule {\n\tencodingForModel?: (modelId: string) => EncodeCapable;\n\tgetEncoding?: (encoding: string) => EncodeCapable;\n}\n\ninterface GenericEncodeModule {\n\tencode?: (input: string) => readonly unknown[];\n}\n\nconst requireModule = createRequire(import.meta.url);\n\nexport function createNodeOptionalModuleLoader(): OptionalModuleLoader {\n\treturn {\n\t\tresolve(specifier) {\n\t\t\ttry {\n\t\t\t\treturn requireModule.resolve(specifier);\n\t\t\t} catch {\n\t\t\t\treturn undefined;\n\t\t\t}\n\t\t},\n\t\tload(specifier) {\n\t\t\treturn requireModule(specifier) as unknown;\n\t\t},\n\t};\n}\n\nexport function createFallbackTokenCounter(): TokenCounterAdapter {\n\treturn {\n\t\tid: \"fallback-estimator\",\n\t\tpriority: 0,\n\t\tisAvailable: () => true,\n\t\tsupports: () => true,\n\t\tcountText(input, modelId) {\n\t\t\treturn estimateTextTokens(input, modelId);\n\t\t},\n\t};\n}\n\nexport function estimateTextTokens(input: string, modelId = \"unknown\"): TokenCountResult {\n\tif (input.length === 0) {\n\t\treturn createTokenResult(0, \"estimated\", \"medium\", \"fallback-estimator\", modelId, [\"empty-input\"]);\n\t}\n\n\tlet asciiWord = 0;\n\tlet whitespace = 0;\n\tlet cjk = 0;\n\tlet hangul = 0;\n\tlet kana = 0;\n\tlet emojiOrWide = 0;\n\tlet punctuation = 0;\n\tlet other = 0;\n\n\tfor (const char of input) {\n\t\tconst codePoint = char.codePointAt(0) ?? 0;\n\t\tif (/\\s/u.test(char)) {\n\t\t\twhitespace += 1;\n\t\t} else if (isHangul(codePoint)) {\n\t\t\thangul += 1;\n\t\t} else if (isHiraganaOrKatakana(codePoint)) {\n\t\t\tkana += 1;\n\t\t} else if (isCjkIdeograph(codePoint)) {\n\t\t\tcjk += 1;\n\t\t} else if (isAsciiAlphaNumeric(codePoint) || char === \"_\") {\n\t\t\tasciiWord += 1;\n\t\t} else if (isEmojiOrWideSymbol(codePoint)) {\n\t\t\temojiOrWide += 1;\n\t\t} else if (isAsciiPunctuation(codePoint)) {\n\t\t\tpunctuation += 1;\n\t\t} else {\n\t\t\tother += 1;\n\t\t}\n\t}\n\n\tconst codeLike = looksCodeLike(input, punctuation, whitespace);\n\tconst jsonLike = looksJsonLike(input);\n\tconst nonAsciiRatio = (hangul + cjk + kana + emojiOrWide + other) / Math.max(1, input.length);\n\tconst base =\n\t\tasciiWord / (codeLike ? 3.2 : 4) +\n\t\twhitespace / 12 +\n\t\tpunctuation / 2.1 +\n\t\thangul / 0.95 +\n\t\tkana / 1.0 +\n\t\tcjk / 1.2 +\n\t\temojiOrWide * 1.8 +\n\t\tother / 2;\n\tconst adjusted = base * (jsonLike ? 1.12 : 1) * (codeLike ? 1.08 : 1);\n\tconst tokens = Math.max(1, Math.ceil(adjusted));\n\tconst compositionParts: string[] = [];\n\tif (hangul > 0) compositionParts.push(`hangul:${hangul}`);\n\tif (kana > 0) compositionParts.push(`kana:${kana}`);\n\tif (cjk > 0) compositionParts.push(`cjk:${cjk}`);\n\tif (asciiWord > 0) compositionParts.push(`ascii:${asciiWord}`);\n\tif (emojiOrWide > 0) compositionParts.push(`emoji:${emojiOrWide}`);\n\tconst notes = [\n\t\tcodeLike ? \"code-like\" : \"prose-like\",\n\t\tjsonLike ? \"json-like\" : \"not-json-like\",\n\t\tnonAsciiRatio > 0 ? `non-ascii:${(nonAsciiRatio * 100).toFixed(0)}%` : \"latin-only\",\n\t\t`composition(${compositionParts.join(\",\")})`,\n\t];\n\tconst confidence: ContextBudgetTokenConfidence =\n\t\tnonAsciiRatio > 0.4 ? \"low\" : nonAsciiRatio > 0.15 ? \"medium\" : \"high\";\n\treturn createTokenResult(tokens, \"estimated\", confidence, \"fallback-estimator\", modelId, notes);\n}\n\nexport function createOpenAiJsTokenCounter(\n\tloader: OptionalModuleLoader = createNodeOptionalModuleLoader(),\n): TokenCounterAdapter {\n\tconst packageNames = [\"js-tiktoken\", \"gpt-tokenizer\", \"tiktoken\"] as const;\n\treturn {\n\t\tid: \"openai-bpe-js\",\n\t\tpriority: 80,\n\t\tisAvailable() {\n\t\t\treturn packageNames.some((specifier) => loader.resolve(specifier) !== undefined);\n\t\t},\n\t\tsupports(modelId) {\n\t\t\treturn isOpenAiStyleModel(modelId);\n\t\t},\n\t\tcountText(input, modelId) {\n\t\t\tfor (const specifier of packageNames) {\n\t\t\t\tif (loader.resolve(specifier) === undefined) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tconst result = countWithOptionalTokenizerModule(loader.load(specifier), specifier, input, modelId);\n\t\t\t\tif (result !== undefined) {\n\t\t\t\t\treturn result;\n\t\t\t\t}\n\t\t\t}\n\t\t\tthrow new Error(\"no supported OpenAI JS tokenizer module shape found\");\n\t\t},\n\t};\n}\n\nexport function createOpenAiWasmTokenCounter(\n\tloader: OptionalModuleLoader = createNodeOptionalModuleLoader(),\n): TokenCounterAdapter {\n\tconst packageNames = [\"@dqbd/tiktoken\", \"tiktoken\"] as const;\n\treturn {\n\t\tid: \"openai-bpe-wasm\",\n\t\tpriority: 90,\n\t\tisAvailable() {\n\t\t\treturn packageNames.some((specifier) => loader.resolve(specifier) !== undefined);\n\t\t},\n\t\tsupports(modelId) {\n\t\t\treturn isOpenAiStyleModel(modelId);\n\t\t},\n\t\tcountText(input, modelId) {\n\t\t\tfor (const specifier of packageNames) {\n\t\t\t\tif (loader.resolve(specifier) === undefined) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tconst result = countWithOptionalTokenizerModule(loader.load(specifier), specifier, input, modelId);\n\t\t\t\tif (result !== undefined) {\n\t\t\t\t\treturn result;\n\t\t\t\t}\n\t\t\t}\n\t\t\tthrow new Error(\"no supported OpenAI WASM tokenizer module shape found\");\n\t\t},\n\t};\n}\n\nexport function createTokenCounterForMode(\n\tmode: ContextBudgetTokenizerMode,\n\tloader: OptionalModuleLoader = createNodeOptionalModuleLoader(),\n): TokenCounterAdapter {\n\tconst fallback = createFallbackTokenCounter();\n\tif (mode === \"fallback\") {\n\t\treturn fallback;\n\t}\n\tconst adapters =\n\t\tmode === \"openai-wasm\" ? [createOpenAiWasmTokenCounter(loader)] : [createOpenAiJsTokenCounter(loader)];\n\tif (mode === \"auto\") {\n\t\tadapters.push(createOpenAiWasmTokenCounter(loader));\n\t}\n\treturn createTokenCounterRegistry({ adapters, fallback });\n}\n\nexport function createTokenCounterRegistry(options: TokenCounterRegistryOptions = {}): TokenCounterAdapter {\n\tconst fallback = options.fallback ?? createFallbackTokenCounter();\n\tconst adapters = [...(options.adapters ?? [])].sort((a, b) => b.priority - a.priority || a.id.localeCompare(b.id));\n\treturn {\n\t\tid: \"token-counter-registry\",\n\t\tpriority: 100,\n\t\tisAvailable: () => true,\n\t\tsupports: () => true,\n\t\tcountText(input, modelId) {\n\t\t\tconst notes: string[] = [];\n\t\t\tfor (const adapter of adapters) {\n\t\t\t\tif (!adapter.supports(modelId)) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\ttry {\n\t\t\t\t\tif (!adapter.isAvailable()) {\n\t\t\t\t\t\tnotes.push(`${adapter.id}:unavailable`);\n\t\t\t\t\t\tcontinue;\n\t\t\t\t\t}\n\t\t\t\t\treturn adapter.countText(input, modelId);\n\t\t\t\t} catch (error) {\n\t\t\t\t\tconst message = error instanceof Error ? error.message : \"unknown adapter failure\";\n\t\t\t\t\tnotes.push(`${adapter.id}:failed:${message}`);\n\t\t\t\t}\n\t\t\t}\n\t\t\tconst result = fallback.countText(input, modelId);\n\t\t\treturn { ...result, notes: [...notes, ...result.notes] };\n\t\t},\n\t};\n}\n\nfunction countWithOptionalTokenizerModule(\n\tmoduleValue: unknown,\n\tspecifier: string,\n\tinput: string,\n\tmodelId: string,\n): TokenCountResult | undefined {\n\tconst moduleObject = unwrapDefaultModule(moduleValue);\n\tconst jsTiktoken = moduleObject as JsTiktokenModule;\n\tif (typeof jsTiktoken.encodingForModel === \"function\") {\n\t\tconst encoding = jsTiktoken.encodingForModel(modelId);\n\t\treturn countWithEncoding(encoding, specifier, input, modelId, \"model-encoding\");\n\t}\n\tif (typeof jsTiktoken.getEncoding === \"function\") {\n\t\tconst encoding = jsTiktoken.getEncoding(selectOpenAiEncoding(modelId));\n\t\treturn countWithEncoding(encoding, specifier, input, modelId, \"fallback-encoding\");\n\t}\n\tconst generic = moduleObject as GenericEncodeModule;\n\tif (typeof generic.encode === \"function\") {\n\t\treturn createTokenResult(generic.encode(input).length, \"exact\", \"medium\", specifier, modelId, [\"generic-encode\"]);\n\t}\n\treturn undefined;\n}\n\nfunction countWithEncoding(\n\tencoding: EncodeCapable,\n\tadapterId: string,\n\tinput: string,\n\tmodelId: string,\n\tnote: string,\n): TokenCountResult {\n\treturn createTokenResult(encoding.encode(input).length, \"exact\", \"high\", adapterId, modelId, [note]);\n}\n\nfunction unwrapDefaultModule(moduleValue: unknown): unknown {\n\tif (moduleValue && typeof moduleValue === \"object\" && \"default\" in moduleValue) {\n\t\treturn (moduleValue as { readonly default: unknown }).default;\n\t}\n\treturn moduleValue;\n}\n\nfunction createTokenResult(\n\ttokens: number,\n\tmethod: ContextBudgetTokenCountMethod,\n\tconfidence: ContextBudgetTokenConfidence,\n\tadapterId: string,\n\tmodelId: string,\n\tnotes: readonly string[],\n): TokenCountResult {\n\treturn {\n\t\ttokens: Math.max(0, Math.ceil(tokens)),\n\t\tmethod,\n\t\tconfidence,\n\t\tadapterId,\n\t\tmodelId,\n\t\tnotes,\n\t};\n}\n\nfunction isOpenAiStyleModel(modelId: string): boolean {\n\tconst normalized = modelId.toLowerCase();\n\treturn (\n\t\tnormalized.includes(\"openai\") ||\n\t\tnormalized.startsWith(\"gpt-\") ||\n\t\tnormalized.startsWith(\"o1\") ||\n\t\tnormalized.startsWith(\"o3\") ||\n\t\tnormalized.startsWith(\"o4\") ||\n\t\tnormalized.includes(\"chatgpt\")\n\t);\n}\n\nfunction selectOpenAiEncoding(modelId: string): string {\n\tconst normalized = modelId.toLowerCase();\n\tif (normalized.includes(\"gpt-4o\") || normalized.includes(\"gpt-5\") || normalized.startsWith(\"o\")) {\n\t\treturn \"o200k_base\";\n\t}\n\treturn \"cl100k_base\";\n}\n\nfunction isAsciiAlphaNumeric(codePoint: number): boolean {\n\treturn (\n\t\t(codePoint >= 48 && codePoint <= 57) ||\n\t\t(codePoint >= 65 && codePoint <= 90) ||\n\t\t(codePoint >= 97 && codePoint <= 122)\n\t);\n}\n\nfunction isAsciiPunctuation(codePoint: number): boolean {\n\treturn codePoint >= 33 && codePoint <= 126;\n}\n\nfunction isHangul(codePoint: number): boolean {\n\t// Composed syllables (가-힣)\n\tif (codePoint >= 0xac00 && codePoint <= 0xd7af) return true;\n\t// Jamo: initial (ㄱ-ㅎ), medial (ㅏ-ㅣ)\n\tif (codePoint >= 0x1100 && codePoint <= 0x11ff) return true;\n\t// Compatibility jamo (ㄱ-ㅎ, ㅏ-ㅣ) and Hangul letters\n\tif (codePoint >= 0x3130 && codePoint <= 0x318f) return true;\n\t// Extended jamo\n\tif (codePoint >= 0xa960 && codePoint <= 0xa97c) return true;\n\treturn false;\n}\n\nfunction isHiraganaOrKatakana(codePoint: number): boolean {\n\t// Hiragana (ぁ-より)\n\tif (codePoint >= 0x3040 && codePoint <= 0x309f) return true;\n\t// Katakana (ァ-ヿ) + halfwidth katakana\n\tif (codePoint >= 0x30a0 && codePoint <= 0x30ff) return true;\n\tif (codePoint >= 0xff65 && codePoint <= 0xff9f) return true;\n\treturn false;\n}\n\nfunction isCjkIdeograph(codePoint: number): boolean {\n\t// CJK Unified Ideographs (main block: 中, 国, etc.)\n\tif (codePoint >= 0x4e00 && codePoint <= 0x9fff) return true;\n\t// CJK Extension A (rare)\n\tif (codePoint >= 0x3400 && codePoint <= 0x4dbf) return true;\n\t// CJK compatibility ideographs\n\tif (codePoint >= 0xf900 && codePoint <= 0xfaff) return true;\n\treturn false;\n}\n\nfunction isEmojiOrWideSymbol(codePoint: number): boolean {\n\t// Main emoji blocks\n\tif (codePoint >= 0x1f000) return true;\n\t// Misc symbols, dingbats\n\tif (codePoint >= 0x2600 && codePoint <= 0x27bf) return true;\n\t// CJK fullwidth/special forms that occupy double width\n\tif (codePoint >= 0xff01 && codePoint <= 0xff60) return true;\n\tif (codePoint >= 0xffe0 && codePoint <= 0xffe6) return true;\n\treturn false;\n}\n\nfunction looksJsonLike(input: string): boolean {\n\tconst trimmed = input.trim();\n\treturn (trimmed.startsWith(\"{\") && trimmed.endsWith(\"}\")) || (trimmed.startsWith(\"[\") && trimmed.endsWith(\"]\"));\n}\n\nfunction looksCodeLike(input: string, punctuation: number, whitespace: number): boolean {\n\tif (/\\b(function|const|let|class|interface|import|export|return|async|await)\\b/.test(input)) {\n\t\treturn true;\n\t}\n\treturn punctuation > input.length / 8 && whitespace > input.length / 20;\n}\n"]}

package/dist/core/context-budget-v2-coverage.d.ts

@@ -0,0 +1,5 @@
+export declare function computeCoverageGap(query: string | undefined, selectedTexts: readonly string[]): {
+    gap: boolean;
+    missing: readonly string[];
+};
+//# sourceMappingURL=context-budget-v2-coverage.d.ts.map

package/dist/core/context-budget-v2-coverage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-budget-v2-coverage.d.ts","sourceRoot":"","sources":["../../src/core/context-budget-v2-coverage.ts"],"names":[],"mappings":"AAmBA,wBAAgB,kBAAkB,CACjC,KAAK,EAAE,MAAM,GAAG,SAAS,EACzB,aAAa,EAAE,SAAS,MAAM,EAAE,GAC9B;IAAE,GAAG,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAA;CAAE,CAgB9C","sourcesContent":["const COVERAGE_STOP_WORDS = new Set([\n\t\"the\",\n\t\"a\",\n\t\"an\",\n\t\"of\",\n\t\"to\",\n\t\"in\",\n\t\"on\",\n\t\"for\",\n\t\"and\",\n\t\"or\",\n\t\"is\",\n\t\"are\",\n\t\"with\",\n\t\"how\",\n\t\"i\",\n\t\"you\",\n]);\n\nexport function computeCoverageGap(\n\tquery: string | undefined,\n\tselectedTexts: readonly string[],\n): { gap: boolean; missing: readonly string[] } {\n\tif (!query || query.trim() === \"\") {\n\t\treturn { gap: false, missing: [] };\n\t}\n\tconst queryTokens = tokenizeLight(query);\n\tif (queryTokens.length === 0) {\n\t\treturn { gap: false, missing: [] };\n\t}\n\tconst union = new Set<string>();\n\tfor (const text of selectedTexts) {\n\t\tfor (const token of tokenizeLight(text)) {\n\t\t\tunion.add(token);\n\t\t}\n\t}\n\tconst missing = queryTokens.filter((token) => !union.has(token));\n\treturn { gap: missing.length > 0, missing };\n}\n\nfunction tokenizeLight(text: string): readonly string[] {\n\tconst lower = text.toLowerCase();\n\tconst raw = lower.match(/[\\p{L}\\p{N}\\p{Extended_Pictographic}]+/gu);\n\tif (!raw) return [];\n\tconst seen = new Set<string>();\n\tconst out: string[] = [];\n\tfor (const token of raw) {\n\t\tif (isShortAsciiToken(token)) continue;\n\t\tif (COVERAGE_STOP_WORDS.has(token)) continue;\n\t\tif (!seen.has(token)) {\n\t\t\tseen.add(token);\n\t\t\tout.push(token);\n\t\t}\n\t}\n\treturn out;\n}\n\nfunction isShortAsciiToken(token: string): boolean {\n\treturn token.length <= 2 && /^[a-z0-9]+$/u.test(token);\n}\n"]}

package/dist/core/context-budget-v2-coverage.js

@@ -0,0 +1,58 @@
+const COVERAGE_STOP_WORDS = new Set([
+    "the",
+    "a",
+    "an",
+    "of",
+    "to",
+    "in",
+    "on",
+    "for",
+    "and",
+    "or",
+    "is",
+    "are",
+    "with",
+    "how",
+    "i",
+    "you",
+]);
+export function computeCoverageGap(query, selectedTexts) {
+    if (!query || query.trim() === "") {
+        return { gap: false, missing: [] };
+    }
+    const queryTokens = tokenizeLight(query);
+    if (queryTokens.length === 0) {
+        return { gap: false, missing: [] };
+    }
+    const union = new Set();
+    for (const text of selectedTexts) {
+        for (const token of tokenizeLight(text)) {
+            union.add(token);
+        }
+    }
+    const missing = queryTokens.filter((token) => !union.has(token));
+    return { gap: missing.length > 0, missing };
+}
+function tokenizeLight(text) {
+    const lower = text.toLowerCase();
+    const raw = lower.match(/[\p{L}\p{N}\p{Extended_Pictographic}]+/gu);
+    if (!raw)
+        return [];
+    const seen = new Set();
+    const out = [];
+    for (const token of raw) {
+        if (isShortAsciiToken(token))
+            continue;
+        if (COVERAGE_STOP_WORDS.has(token))
+            continue;
+        if (!seen.has(token)) {
+            seen.add(token);
+            out.push(token);
+        }
+    }
+    return out;
+}
+function isShortAsciiToken(token) {
+    return token.length <= 2 && /^[a-z0-9]+$/u.test(token);
+}
+//# sourceMappingURL=context-budget-v2-coverage.js.map

package/dist/core/context-budget-v2-coverage.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-budget-v2-coverage.js","sourceRoot":"","sources":["../../src/core/context-budget-v2-coverage.ts"],"names":[],"mappings":"AAAA,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IACnC,KAAK;IACL,GAAG;IACH,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,MAAM;IACN,KAAK;IACL,GAAG;IACH,KAAK;CACL,CAAC,CAAC;AAEH,MAAM,UAAU,kBAAkB,CACjC,KAAyB,EACzB,aAAgC,EACe;IAC/C,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;QACnC,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACpC,CAAC;IACD,MAAM,WAAW,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9B,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IACpC,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;QAClC,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;YACzC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClB,CAAC;IACF,CAAC;IACD,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;IACjE,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,OAAO,EAAE,CAAC;AAAA,CAC5C;AAED,SAAS,aAAa,CAAC,IAAY,EAAqB;IACvD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACjC,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,0CAA0C,CAAC,CAAC;IACpE,IAAI,CAAC,GAAG;QAAE,OAAO,EAAE,CAAC;IACpB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QACzB,IAAI,iBAAiB,CAAC,KAAK,CAAC;YAAE,SAAS;QACvC,IAAI,mBAAmB,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,SAAS;QAC7C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACtB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAChB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjB,CAAC;IACF,CAAC;IACD,OAAO,GAAG,CAAC;AAAA,CACX;AAED,SAAS,iBAAiB,CAAC,KAAa,EAAW;IAClD,OAAO,KAAK,CAAC,MAAM,IAAI,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAAA,CACvD","sourcesContent":["const COVERAGE_STOP_WORDS = new Set([\n\t\"the\",\n\t\"a\",\n\t\"an\",\n\t\"of\",\n\t\"to\",\n\t\"in\",\n\t\"on\",\n\t\"for\",\n\t\"and\",\n\t\"or\",\n\t\"is\",\n\t\"are\",\n\t\"with\",\n\t\"how\",\n\t\"i\",\n\t\"you\",\n]);\n\nexport function computeCoverageGap(\n\tquery: string | undefined,\n\tselectedTexts: readonly string[],\n): { gap: boolean; missing: readonly string[] } {\n\tif (!query || query.trim() === \"\") {\n\t\treturn { gap: false, missing: [] };\n\t}\n\tconst queryTokens = tokenizeLight(query);\n\tif (queryTokens.length === 0) {\n\t\treturn { gap: false, missing: [] };\n\t}\n\tconst union = new Set<string>();\n\tfor (const text of selectedTexts) {\n\t\tfor (const token of tokenizeLight(text)) {\n\t\t\tunion.add(token);\n\t\t}\n\t}\n\tconst missing = queryTokens.filter((token) => !union.has(token));\n\treturn { gap: missing.length > 0, missing };\n}\n\nfunction tokenizeLight(text: string): readonly string[] {\n\tconst lower = text.toLowerCase();\n\tconst raw = lower.match(/[\\p{L}\\p{N}\\p{Extended_Pictographic}]+/gu);\n\tif (!raw) return [];\n\tconst seen = new Set<string>();\n\tconst out: string[] = [];\n\tfor (const token of raw) {\n\t\tif (isShortAsciiToken(token)) continue;\n\t\tif (COVERAGE_STOP_WORDS.has(token)) continue;\n\t\tif (!seen.has(token)) {\n\t\t\tseen.add(token);\n\t\t\tout.push(token);\n\t\t}\n\t}\n\treturn out;\n}\n\nfunction isShortAsciiToken(token: string): boolean {\n\treturn token.length <= 2 && /^[a-z0-9]+$/u.test(token);\n}\n"]}

package/dist/core/context-budget-v2-plan-hash.d.ts

@@ -0,0 +1,12 @@
+import type { PlannedItemV2 } from "./context-budget-v2-scoring.ts";
+import type { SelectedRepresentationV2, TierBudgetAllocationV2 } from "./context-budget-v2-types.ts";
+export declare function contentHashOf(text: string): string;
+export declare function computePlanHash(input: {
+    readonly policyVersion: string;
+    readonly promptHash: string | undefined;
+    readonly planned: readonly PlannedItemV2[];
+    readonly selection: ReadonlyMap<string, SelectedRepresentationV2>;
+    readonly allocations: readonly TierBudgetAllocationV2[];
+    readonly omittedItemIds: readonly string[];
+}): string;
+//# sourceMappingURL=context-budget-v2-plan-hash.d.ts.map

package/dist/core/context-budget-v2-plan-hash.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-budget-v2-plan-hash.d.ts","sourceRoot":"","sources":["../../src/core/context-budget-v2-plan-hash.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,KAAK,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,8BAA8B,CAAC;AAErG,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAElD;AAED,wBAAgB,eAAe,CAAC,KAAK,EAAE;IACtC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,SAAS,CAAC;IACxC,QAAQ,CAAC,OAAO,EAAE,SAAS,aAAa,EAAE,CAAC;IAC3C,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,wBAAwB,CAAC,CAAC;IAClE,QAAQ,CAAC,WAAW,EAAE,SAAS,sBAAsB,EAAE,CAAC;IACxD,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAC;CAC3C,GAAG,MAAM,CAgCT","sourcesContent":["import { createHash } from \"node:crypto\";\nimport type { PlannedItemV2 } from \"./context-budget-v2-scoring.ts\";\nimport type { SelectedRepresentationV2, TierBudgetAllocationV2 } from \"./context-budget-v2-types.ts\";\n\nexport function contentHashOf(text: string): string {\n\treturn createHash(\"sha256\").update(text, \"utf8\").digest(\"hex\");\n}\n\nexport function computePlanHash(input: {\n\treadonly policyVersion: string;\n\treadonly promptHash: string | undefined;\n\treadonly planned: readonly PlannedItemV2[];\n\treadonly selection: ReadonlyMap<string, SelectedRepresentationV2>;\n\treadonly allocations: readonly TierBudgetAllocationV2[];\n\treadonly omittedItemIds: readonly string[];\n}): string {\n\tconst itemsCanonical = input.planned\n\t\t.map((planned) => {\n\t\t\tconst selected = input.selection.get(planned.item.id);\n\t\t\treturn {\n\t\t\t\tid: planned.item.id,\n\t\t\t\ttier: planned.item.tier,\n\t\t\t\tcontentHash: planned.contentHash,\n\t\t\t\tkind: selected?.kind ?? \"omit\",\n\t\t\t\ttokens: selected?.estimatedTokens ?? 0,\n\t\t\t};\n\t\t})\n\t\t.sort((a, b) => a.id.localeCompare(b.id));\n\tconst tiersCanonical = [...input.allocations]\n\t\t.map((allocation) => [\n\t\t\tallocation.tier,\n\t\t\tallocation.floorTokens,\n\t\t\tallocation.ceilingTokens,\n\t\t\tallocation.allocatedTokens,\n\t\t\tallocation.usedTokens,\n\t\t\tallocation.hardTokens,\n\t\t])\n\t\t.sort((a, b) => String(a[0]).localeCompare(String(b[0])));\n\tconst omittedCanonical = [...input.omittedItemIds].sort();\n\tconst canonical = {\n\t\tpolicyVersion: input.policyVersion,\n\t\tpromptHash: input.promptHash ?? null,\n\t\titems: itemsCanonical,\n\t\ttiers: tiersCanonical,\n\t\tomitted: omittedCanonical,\n\t};\n\treturn createHash(\"sha256\").update(JSON.stringify(canonical), \"utf8\").digest(\"hex\");\n}\n"]}

package/dist/core/context-budget-v2-plan-hash.js

@@ -0,0 +1,38 @@
+import { createHash } from "node:crypto";
+export function contentHashOf(text) {
+    return createHash("sha256").update(text, "utf8").digest("hex");
+}
+export function computePlanHash(input) {
+    const itemsCanonical = input.planned
+        .map((planned) => {
+        const selected = input.selection.get(planned.item.id);
+        return {
+            id: planned.item.id,
+            tier: planned.item.tier,
+            contentHash: planned.contentHash,
+            kind: selected?.kind ?? "omit",
+            tokens: selected?.estimatedTokens ?? 0,
+        };
+    })
+        .sort((a, b) => a.id.localeCompare(b.id));
+    const tiersCanonical = [...input.allocations]
+        .map((allocation) => [
+        allocation.tier,
+        allocation.floorTokens,
+        allocation.ceilingTokens,
+        allocation.allocatedTokens,
+        allocation.usedTokens,
+        allocation.hardTokens,
+    ])
+        .sort((a, b) => String(a[0]).localeCompare(String(b[0])));
+    const omittedCanonical = [...input.omittedItemIds].sort();
+    const canonical = {
+        policyVersion: input.policyVersion,
+        promptHash: input.promptHash ?? null,
+        items: itemsCanonical,
+        tiers: tiersCanonical,
+        omitted: omittedCanonical,
+    };
+    return createHash("sha256").update(JSON.stringify(canonical), "utf8").digest("hex");
+}
+//# sourceMappingURL=context-budget-v2-plan-hash.js.map

package/dist/core/context-budget-v2-plan-hash.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-budget-v2-plan-hash.js","sourceRoot":"","sources":["../../src/core/context-budget-v2-plan-hash.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAIzC,MAAM,UAAU,aAAa,CAAC,IAAY,EAAU;IACnD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAAA,CAC/D;AAED,MAAM,UAAU,eAAe,CAAC,KAO/B,EAAU;IACV,MAAM,cAAc,GAAG,KAAK,CAAC,OAAO;SAClC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;QACjB,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACtD,OAAO;YACN,EAAE,EAAE,OAAO,CAAC,IAAI,CAAC,EAAE;YACnB,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,IAAI;YACvB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,MAAM;YAC9B,MAAM,EAAE,QAAQ,EAAE,eAAe,IAAI,CAAC;SACtC,CAAC;IAAA,CACF,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3C,MAAM,cAAc,GAAG,CAAC,GAAG,KAAK,CAAC,WAAW,CAAC;SAC3C,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC;QACpB,UAAU,CAAC,IAAI;QACf,UAAU,CAAC,WAAW;QACtB,UAAU,CAAC,aAAa;QACxB,UAAU,CAAC,eAAe;QAC1B,UAAU,CAAC,UAAU;QACrB,UAAU,CAAC,UAAU;KACrB,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,gBAAgB,GAAG,CAAC,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1D,MAAM,SAAS,GAAG;QACjB,aAAa,EAAE,KAAK,CAAC,aAAa;QAClC,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,IAAI;QACpC,KAAK,EAAE,cAAc;QACrB,KAAK,EAAE,cAAc;QACrB,OAAO,EAAE,gBAAgB;KACzB,CAAC;IACF,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAAA,CACpF","sourcesContent":["import { createHash } from \"node:crypto\";\nimport type { PlannedItemV2 } from \"./context-budget-v2-scoring.ts\";\nimport type { SelectedRepresentationV2, TierBudgetAllocationV2 } from \"./context-budget-v2-types.ts\";\n\nexport function contentHashOf(text: string): string {\n\treturn createHash(\"sha256\").update(text, \"utf8\").digest(\"hex\");\n}\n\nexport function computePlanHash(input: {\n\treadonly policyVersion: string;\n\treadonly promptHash: string | undefined;\n\treadonly planned: readonly PlannedItemV2[];\n\treadonly selection: ReadonlyMap<string, SelectedRepresentationV2>;\n\treadonly allocations: readonly TierBudgetAllocationV2[];\n\treadonly omittedItemIds: readonly string[];\n}): string {\n\tconst itemsCanonical = input.planned\n\t\t.map((planned) => {\n\t\t\tconst selected = input.selection.get(planned.item.id);\n\t\t\treturn {\n\t\t\t\tid: planned.item.id,\n\t\t\t\ttier: planned.item.tier,\n\t\t\t\tcontentHash: planned.contentHash,\n\t\t\t\tkind: selected?.kind ?? \"omit\",\n\t\t\t\ttokens: selected?.estimatedTokens ?? 0,\n\t\t\t};\n\t\t})\n\t\t.sort((a, b) => a.id.localeCompare(b.id));\n\tconst tiersCanonical = [...input.allocations]\n\t\t.map((allocation) => [\n\t\t\tallocation.tier,\n\t\t\tallocation.floorTokens,\n\t\t\tallocation.ceilingTokens,\n\t\t\tallocation.allocatedTokens,\n\t\t\tallocation.usedTokens,\n\t\t\tallocation.hardTokens,\n\t\t])\n\t\t.sort((a, b) => String(a[0]).localeCompare(String(b[0])));\n\tconst omittedCanonical = [...input.omittedItemIds].sort();\n\tconst canonical = {\n\t\tpolicyVersion: input.policyVersion,\n\t\tpromptHash: input.promptHash ?? null,\n\t\titems: itemsCanonical,\n\t\ttiers: tiersCanonical,\n\t\tomitted: omittedCanonical,\n\t};\n\treturn createHash(\"sha256\").update(JSON.stringify(canonical), \"utf8\").digest(\"hex\");\n}\n"]}

package/dist/core/context-budget-v2-planner.d.ts

@@ -0,0 +1,3 @@
+import { type PromptContextBudgetInputV2, type PromptContextBudgetPlanV2 } from "./context-budget-v2-types.ts";
+export declare function planPromptContextBudgetV2(input: PromptContextBudgetInputV2): PromptContextBudgetPlanV2;
+//# sourceMappingURL=context-budget-v2-planner.d.ts.map

package/dist/core/context-budget-v2-planner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-budget-v2-planner.d.ts","sourceRoot":"","sources":["../../src/core/context-budget-v2-planner.ts"],"names":[],"mappings":"AAmBA,OAAO,EAGN,KAAK,0BAA0B,EAE/B,KAAK,yBAAyB,EAI9B,MAAM,8BAA8B,CAAC;AAGtC,wBAAgB,yBAAyB,CAAC,KAAK,EAAE,0BAA0B,GAAG,yBAAyB,CAoHtG","sourcesContent":["import {\n\ttype ContextBudgetItemV2,\n\ttype ContextBudgetTierV2,\n\ttype ContextSourceRefV2,\n\tDEFAULT_HEADROOM_QUALITY_POLICY,\n} from \"./context-budget-headroom.ts\";\nimport { computePlanHash } from \"./context-budget-v2-plan-hash.ts\";\nimport { compareOptionalForSelection } from \"./context-budget-v2-scoring.ts\";\nimport {\n\tapplyPlannerRedundancyPenalties,\n\tbuildTierAllocations,\n\tcreatePlannedItems,\n\tcreateTierUsage,\n\temitCoverageDiagnostics,\n\temitOmissionDiagnostics,\n\tselectHardItem,\n\tselectOptionalItem,\n} from \"./context-budget-v2-selection.ts\";\nimport { allocateTiers, computeTierDemand } from \"./context-budget-v2-tiers.ts\";\nimport {\n\tCONTEXT_BUDGET_POLICY_VERSION_V2,\n\tDEFAULT_TIER_POLICY_V2,\n\ttype PromptContextBudgetInputV2,\n\ttype PromptContextBudgetObservabilityV2,\n\ttype PromptContextBudgetPlanV2,\n\ttype QualityDiagnosticV2,\n\ttype SelectedRepresentationV2,\n\ttype TierBudgetPolicyV2,\n} from \"./context-budget-v2-types.ts\";\nimport { getTokenOptimizerRuntimeStatus } from \"./token-optimizer.ts\";\n\nexport function planPromptContextBudgetV2(input: PromptContextBudgetInputV2): PromptContextBudgetPlanV2 {\n\tconst modelId = input.modelId ?? \"unknown\";\n\tconst qualityPolicy = input.qualityPolicy ?? DEFAULT_HEADROOM_QUALITY_POLICY;\n\tconst policyVersion = input.policyVersion ?? CONTEXT_BUDGET_POLICY_VERSION_V2;\n\tconst diagnostics: QualityDiagnosticV2[] = [];\n\n\tconst maxTokens = normalizeBudgetTokens(\"maxTokens\", input.maxTokens, diagnostics);\n\tconst responseReserve = normalizeBudgetTokens(\n\t\t\"responseReserveTokens\",\n\t\tinput.responseReserveTokens ?? 0,\n\t\tdiagnostics,\n\t);\n\tconst safetyMargin = normalizeBudgetTokens(\n\t\t\"safetyMarginTokens\",\n\t\tinput.safetyMarginTokens ?? Math.ceil(maxTokens * 0.05),\n\t\tdiagnostics,\n\t);\n\tconst available = Math.max(0, maxTokens - responseReserve - safetyMargin);\n\n\tconst tierPolicy: Readonly<Record<ContextBudgetTierV2, TierBudgetPolicyV2>> = {\n\t\t...DEFAULT_TIER_POLICY_V2,\n\t\t...(input.tierPolicy ?? {}),\n\t};\n\tconst basePlanned = createPlannedItems(input.items, input.tokenCounter, modelId);\n\tapplyPlannerRedundancyPenalties(basePlanned);\n\n\tconst demand = computeTierDemand(basePlanned);\n\tconst allocation = allocateTiers(available, tierPolicy, demand);\n\tconst selection = new Map<string, SelectedRepresentationV2>();\n\tconst tierUsed = createTierUsage();\n\tconst omitted: ContextBudgetItemV2[] = [];\n\tconst retrievalFallbacks: ContextSourceRefV2[] = [];\n\tlet usedTokens = 0;\n\n\tfor (const planned of basePlanned) {\n\t\tif (!planned.isHard) continue;\n\t\tusedTokens += selectHardItem(planned, selection, tierUsed);\n\t}\n\tif (usedTokens > available) {\n\t\tdiagnostics.push({\n\t\t\treason: \"hard_pin_over_capacity\",\n\t\t\tdetail: `hard-pinned context uses ${usedTokens} tokens against ${available} available tokens`,\n\t\t});\n\t}\n\n\tconst optionalPlanned = basePlanned.filter((planned) => !planned.isHard).sort(compareOptionalForSelection);\n\tfor (const planned of optionalPlanned) {\n\t\tusedTokens = selectOptionalItem(planned, {\n\t\t\tallocation,\n\t\t\tavailable,\n\t\t\tdiagnostics,\n\t\t\tomitted,\n\t\t\tqualityPolicy,\n\t\t\tretrievalFallbacks,\n\t\t\tselection,\n\t\t\ttierUsed,\n\t\t\tusedTokens,\n\t\t});\n\t}\n\n\tconst omittedHighPriority = omitted.some((item) => item.priority === \"high\" || item.required === true);\n\temitOmissionDiagnostics(omitted, diagnostics, retrievalFallbacks);\n\temitCoverageDiagnostics(input.query, selection, diagnostics);\n\n\tconst tierAllocations = buildTierAllocations(allocation, demand, tierUsed);\n\tconst rawTokens = basePlanned.reduce((sum, planned) => sum + planned.fullTokens, 0);\n\tconst omittedTokens = omitted.reduce(\n\t\t(sum, item) => sum + (basePlanned.find((planned) => planned.item.id === item.id)?.fullTokens ?? 0),\n\t\t0,\n\t);\n\tconst omittedItemIds = omitted.map((item) => item.id);\n\tconst planHash = computePlanHash({\n\t\tpolicyVersion,\n\t\tpromptHash: input.promptHash,\n\t\tplanned: basePlanned,\n\t\tselection,\n\t\tallocations: tierAllocations,\n\t\tomittedItemIds,\n\t});\n\tconst observability = buildObservability({\n\t\tavailable,\n\t\tdiagnostics,\n\t\tomittedItemIds,\n\t\tomittedTokens,\n\t\tplanHash,\n\t\trawTokens,\n\t\tretrievalFallbacks,\n\t\tselection,\n\t\tusedTokens,\n\t});\n\n\treturn {\n\t\tpolicyVersion,\n\t\tpromptHash: input.promptHash,\n\t\tplanHash,\n\t\tmaxTokens,\n\t\tresponseReserveTokens: responseReserve,\n\t\tsafetyMarginTokens: safetyMargin,\n\t\tavailableTokens: available,\n\t\tusedTokens,\n\t\tomittedTokens,\n\t\trawTokens,\n\t\ttokenSavingsRatio: rawTokens > 0 ? Math.max(0, 1 - usedTokens / rawTokens) : 0,\n\t\temergency: usedTokens > available || diagnostics.some((diagnostic) => diagnostic.reason === \"invalid_budget\"),\n\t\tomittedHighPriority,\n\t\ttierAllocations,\n\t\tselectedRepresentations: [...selection.values()].sort((a, b) => a.itemId.localeCompare(b.itemId)),\n\t\tincludedItemIds: basePlanned\n\t\t\t.map((planned) => planned.item.id)\n\t\t\t.filter((id) => selection.has(id))\n\t\t\t.sort(),\n\t\tomittedItemIds,\n\t\tdiagnostics,\n\t\tretrievalFallbacks,\n\t\tobservability,\n\t};\n}\n\nfunction normalizeBudgetTokens(field: string, value: number, diagnostics: QualityDiagnosticV2[]): number {\n\tif (!Number.isFinite(value) || value < 0) {\n\t\tdiagnostics.push({\n\t\t\treason: \"invalid_budget\",\n\t\t\tdetail: `${field} must be a non-negative finite number, got ${value}`,\n\t\t});\n\t\treturn 0;\n\t}\n\treturn Math.floor(value);\n}\n\nfunction buildObservability(input: {\n\treadonly available: number;\n\treadonly diagnostics: readonly QualityDiagnosticV2[];\n\treadonly omittedItemIds: readonly string[];\n\treadonly omittedTokens: number;\n\treadonly planHash: string;\n\treadonly rawTokens: number;\n\treadonly retrievalFallbacks: readonly ContextSourceRefV2[];\n\treadonly selection: ReadonlyMap<string, SelectedRepresentationV2>;\n\treadonly usedTokens: number;\n}): PromptContextBudgetObservabilityV2 {\n\tconst selected = [...input.selection.values()];\n\treturn {\n\t\tcounts: {\n\t\t\tselected: selected.length,\n\t\t\tomitted: input.omittedItemIds.length,\n\t\t\tpointer: selected.filter((representation) => representation.kind === \"pointer\").length,\n\t\t\tcompressed: selected.filter((representation) => representation.kind === \"headroom-compressed\").length,\n\t\t\tfull: selected.filter((representation) => representation.kind === \"full\").length,\n\t\t\tretrievalFallback: countUniqueRetrievalFallbacks(input.retrievalFallbacks),\n\t\t},\n\t\tdiagnosticReasons: [...new Set(input.diagnostics.map((diagnostic) => diagnostic.reason))].sort(),\n\t\ttokens: {\n\t\t\tavailable: input.available,\n\t\t\tused: input.usedTokens,\n\t\t\traw: input.rawTokens,\n\t\t\tomitted: input.omittedTokens,\n\t\t\ttokenSavings: Math.max(0, input.rawTokens - input.usedTokens),\n\t\t},\n\t\tplanHash: input.planHash,\n\t\ttokenOptimizer: getTokenOptimizerRuntimeStatus(),\n\t};\n}\n\nfunction countUniqueRetrievalFallbacks(retrievalFallbacks: readonly ContextSourceRefV2[]): number {\n\treturn new Set(\n\t\tretrievalFallbacks.map((ref) =>\n\t\t\t[ref.uri, ref.contentHash, ref.symbol ?? \"\", ref.range?.startLine ?? \"\", ref.range?.endLine ?? \"\"].join(\"\\0\"),\n\t\t),\n\t).size;\n}\n"]}