open-multi-agent-kit 0.80.7 → 0.90.0

package/dist/core/context-budget-system-prompt-items.js

@@ -0,0 +1,143 @@
+import { fnv1aHex } from "./context-budget-governor-v2.js";
+import { scoreContextFileRelevance, scoreSkillRelevance } from "./context-budget-relevance.js";
+const CONTEXT_EXCERPT_CHARS = 1200;
+const MIN_EXCERPT_CHARS = 400;
+const MAX_EXCERPT_CHARS = 3000;
+const MAX_INACTIVE_SKILLS = 15;
+const CONTEXT_FULL_BUDGET_RATIO = 0.5;
+export function createSystemPromptBudgetItems(input, resourceBudget) {
+    const items = [];
+    const contextFullCount = input.options.includeFullContextFiles !== false ? input.contextFiles.length : 0;
+    const excerptChars = computeExcerptChars(resourceBudget, contextFullCount);
+    for (const contextFile of input.contextFiles) {
+        const queryScore = scoreContextFileRelevance({ path: contextFile.path, content: contextFile.content, isGlobal: contextFile.isGlobal ?? false }, input.options.queryContext);
+        pushContextItems(items, contextFile, queryScore, excerptChars, input.options.includeFullContextFiles !== false);
+    }
+    const visibleSkills = input.skills.filter((skill) => !skill.disableModelInvocation);
+    if (input.includeSkills && input.options.includeSkillInventory !== false && visibleSkills.length > 0) {
+        pushSkillItems(items, visibleSkills, input.options);
+    }
+    return items;
+}
+function pushContextItems(items, contextFile, queryScore, excerptChars, includeFullContextFiles) {
+    const scope = contextFile.isGlobal ? "parent" : "project";
+    const tagName = contextFile.isGlobal ? "parent_instructions" : "project_instructions";
+    items.push({
+        id: `context-pointer:${contextFile.path}`,
+        tier: contextFile.isGlobal ? "system" : "current-files",
+        priority: contextFile.isGlobal ? "hard" : "high",
+        relevance: (contextFile.isGlobal ? 1 : 0.8) * 0.4 + queryScore * 0.6,
+        text: renderContextPointer(contextFile, scope),
+    });
+    if (!includeFullContextFiles)
+        return;
+    items.push({
+        id: `context-full:${contextFile.path}`,
+        tier: contextFile.isGlobal ? "system" : "current-files",
+        priority: contextFile.isGlobal ? "high" : "medium",
+        relevance: (contextFile.isGlobal ? 0.9 : 0.6) * 0.4 + queryScore * 0.6,
+        redundancyKey: contextFile.path,
+        sourceRef: {
+            uri: `file://${contextFile.path}`,
+            contentHash: fnv1aHex(contextFile.content),
+            retrievable: true,
+        },
+        text: renderContextFull(contextFile, tagName, excerptChars),
+    });
+}
+function pushSkillItems(items, visibleSkills, options) {
+    items.push({ id: "skill-header", tier: "skills", priority: "hard", text: renderSkillHeader() });
+    const activeSkillNames = new Set(options.activeSkillNames ?? []);
+    const maxInactive = options.maxInactiveSkills ?? MAX_INACTIVE_SKILLS;
+    for (const skill of visibleSkills.filter((skill) => activeSkillNames.has(skill.name))) {
+        items.push(renderSkillBudgetItem(skill, "hard", 1));
+    }
+    const sortedInactive = visibleSkills
+        .filter((skill) => !activeSkillNames.has(skill.name))
+        .sort((a, b) => scoreSkillRelevance(b, options.queryContext) - scoreSkillRelevance(a, options.queryContext));
+    for (const skill of sortedInactive.slice(0, maxInactive)) {
+        items.push(renderSkillBudgetItem(skill, "low", Math.max(0.25, scoreSkillRelevance(skill, options.queryContext))));
+    }
+    items.push({ id: "skill-footer", tier: "skills", priority: "hard", text: "</available_skills>" });
+    const omittedInactiveCount = sortedInactive.length - Math.min(sortedInactive.length, maxInactive);
+    if (omittedInactiveCount > 0) {
+        items.push({
+            id: "skill-omitted-summary",
+            tier: "skills",
+            priority: "low",
+            relevance: 0.1,
+            text: `<!-- ${omittedInactiveCount} additional skills available. Use 'read' to load a specific skill file when a task matches. -->`,
+        });
+    }
+}
+function renderSkillBudgetItem(skill, priority, relevance) {
+    return {
+        id: `skill:${skill.name}`,
+        tier: "skills",
+        priority,
+        relevance,
+        redundancyKey: skill.name,
+        text: renderSkillEntry(skill),
+    };
+}
+function computeExcerptChars(resourceBudget, contextFullCount) {
+    if (contextFullCount <= 0) {
+        return CONTEXT_EXCERPT_CHARS;
+    }
+    const allocatedTokens = Math.floor(resourceBudget * CONTEXT_FULL_BUDGET_RATIO) / contextFullCount;
+    const estimatedChars = Math.floor(allocatedTokens * 4);
+    return Math.max(MIN_EXCERPT_CHARS, Math.min(MAX_EXCERPT_CHARS, estimatedChars));
+}
+function renderContextPointer(contextFile, scope) {
+    const marker = contextFile.containsJailbreak ? " sanitized=true" : "";
+    return `<context_file_pointer scope="${escapeXml(scope)}" path="${escapeXml(contextFile.path)}"${marker} chars="${contextFile.content.length}" />`;
+}
+function renderContextFull(contextFile, tagName, excerptChars = CONTEXT_EXCERPT_CHARS) {
+    const content = contextFile.content.length > excerptChars
+        ? boundedExcerpt(contextFile.content, excerptChars)
+        : contextFile.content;
+    const truncated = content.length < contextFile.content.length ? " truncated=true" : "";
+    return `<${tagName} path="${escapeXml(contextFile.path)}"${truncated}>\n${content}\n</${tagName}>`;
+}
+function boundedExcerpt(content, excerptChars = CONTEXT_EXCERPT_CHARS) {
+    const headChars = Math.floor(excerptChars * 0.7);
+    const tailChars = excerptChars - headChars;
+    let head = content.slice(0, headChars);
+    const lastSpace = head.search(/\s+\S*$/);
+    if (lastSpace > headChars * 0.5) {
+        head = head.slice(0, lastSpace);
+    }
+    let tail = content.slice(-tailChars);
+    const firstSpace = tail.search(/\S+\s+/);
+    if (firstSpace > 0 && firstSpace < tailChars * 0.5) {
+        tail = tail.slice(firstSpace).trimStart();
+    }
+    return `${head}\n\n[...context-budget excerpt omitted ${content.length - excerptChars} chars...]\n\n${tail}`;
+}
+function renderSkillHeader() {
+    return [
+        "\n\nThe following skills provide specialized instructions for specific tasks.",
+        "Use the read tool to load a skill's file when the task matches its description.",
+        "When a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.",
+        "",
+        "<available_skills>",
+    ].join("\n");
+}
+function renderSkillEntry(skill) {
+    return [
+        "  <skill>",
+        `    <name>${escapeXml(skill.name)}</name>`,
+        `    <description>${escapeXml(skill.description)}</description>`,
+        `    <location>${escapeXml(skill.filePath)}</location>`,
+        "  </skill>",
+    ].join("\n");
+}
+export function escapeXml(str) {
+    return str
+        .replace(/&/g, "&amp;")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;")
+        .replace(/"/g, "&quot;")
+        .replace(/'/g, "&apos;");
+}
+//# sourceMappingURL=context-budget-system-prompt-items.js.map

package/dist/core/context-budget-system-prompt-items.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-budget-system-prompt-items.js","sourceRoot":"","sources":["../../src/core/context-budget-system-prompt-items.ts"],"names":[],"mappings":"AAAA,OAAO,EAA4B,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AACrF,OAAO,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,+BAA+B,CAAC;AAiB/F,MAAM,qBAAqB,GAAG,IAAI,CAAC;AACnC,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,iBAAiB,GAAG,IAAI,CAAC;AAC/B,MAAM,mBAAmB,GAAG,EAAE,CAAC;AAC/B,MAAM,yBAAyB,GAAG,GAAG,CAAC;AAEtC,MAAM,UAAU,6BAA6B,CAC5C,KAAkC,EAClC,cAAsB,EACE;IACxB,MAAM,KAAK,GAA0B,EAAE,CAAC;IACxC,MAAM,gBAAgB,GAAG,KAAK,CAAC,OAAO,CAAC,uBAAuB,KAAK,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACzG,MAAM,YAAY,GAAG,mBAAmB,CAAC,cAAc,EAAE,gBAAgB,CAAC,CAAC;IAE3E,KAAK,MAAM,WAAW,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;QAC9C,MAAM,UAAU,GAAG,yBAAyB,CAC3C,EAAE,IAAI,EAAE,WAAW,CAAC,IAAI,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,CAAC,QAAQ,IAAI,KAAK,EAAE,EACjG,KAAK,CAAC,OAAO,CAAC,YAAY,CAC1B,CAAC;QACF,gBAAgB,CAAC,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,uBAAuB,KAAK,KAAK,CAAC,CAAC;IACjH,CAAC;IAED,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACpF,IAAI,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,OAAO,CAAC,qBAAqB,KAAK,KAAK,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtG,cAAc,CAAC,KAAK,EAAE,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,KAAK,CAAC;AAAA,CACb;AAED,SAAS,gBAAgB,CACxB,KAA4B,EAC5B,WAAwB,EACxB,UAAkB,EAClB,YAAoB,EACpB,uBAAgC,EACzB;IACP,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC;IAC1D,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,sBAAsB,CAAC;IACtF,KAAK,CAAC,IAAI,CAAC;QACV,EAAE,EAAE,mBAAmB,WAAW,CAAC,IAAI,EAAE;QACzC,IAAI,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe;QACvD,QAAQ,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM;QAChD,SAAS,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,UAAU,GAAG,GAAG;QACpE,IAAI,EAAE,oBAAoB,CAAC,WAAW,EAAE,KAAK,CAAC;KAC9C,CAAC,CAAC;IACH,IAAI,CAAC,uBAAuB;QAAE,OAAO;IACrC,KAAK,CAAC,IAAI,CAAC;QACV,EAAE,EAAE,gBAAgB,WAAW,CAAC,IAAI,EAAE;QACtC,IAAI,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,eAAe;QACvD,QAAQ,EAAE,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;QAClD,SAAS,EAAE,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,UAAU,GAAG,GAAG;QACtE,aAAa,EAAE,WAAW,CAAC,IAAI;QAC/B,SAAS,EAAE;YACV,GAAG,EAAE,UAAU,WAAW,CAAC,IAAI,EAAE;YACjC,WAAW,EAAE,QAAQ,CAAC,WAAW,CAAC,OAAO,CAAC;YAC1C,WAAW,EAAE,IAAI;SACjB;QACD,IAAI,EAAE,iBAAiB,CAAC,WAAW,EAAE,OAAO,EAAE,YAAY,CAAC;KAC3D,CAAC,CAAC;AAAA,CACH;AAED,SAAS,cAAc,CACtB,KAA4B,EAC5B,aAA+B,EAC/B,OAA+C,EACxC;IACP,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,iBAAiB,EAAE,EAAE,CAAC,CAAC;IAChG,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,gBAAgB,IAAI,EAAE,CAAC,CAAC;IACjE,MAAM,WAAW,GAAG,OAAO,CAAC,iBAAiB,IAAI,mBAAmB,CAAC;IACrE,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACvF,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IACrD,CAAC;IAED,MAAM,cAAc,GAAG,aAAa;SAClC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;SACpD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,mBAAmB,CAAC,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,mBAAmB,CAAC,CAAC,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC;IAC9G,KAAK,MAAM,KAAK,IAAI,cAAc,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,EAAE,CAAC;QAC1D,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,mBAAmB,CAAC,KAAK,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;IACnH,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAC,CAAC;IAElG,MAAM,oBAAoB,GAAG,cAAc,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IAClG,IAAI,oBAAoB,GAAG,CAAC,EAAE,CAAC;QAC9B,KAAK,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,uBAAuB;YAC3B,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,KAAK;YACf,SAAS,EAAE,GAAG;YACd,IAAI,EAAE,QAAQ,oBAAoB,iGAAiG;SACnI,CAAC,CAAC;IACJ,CAAC;AAAA,CACD;AAED,SAAS,qBAAqB,CAAC,KAAY,EAAE,QAAwB,EAAE,SAAiB,EAAuB;IAC9G,OAAO;QACN,EAAE,EAAE,SAAS,KAAK,CAAC,IAAI,EAAE;QACzB,IAAI,EAAE,QAAQ;QACd,QAAQ;QACR,SAAS;QACT,aAAa,EAAE,KAAK,CAAC,IAAI;QACzB,IAAI,EAAE,gBAAgB,CAAC,KAAK,CAAC;KAC7B,CAAC;AAAA,CACF;AAED,SAAS,mBAAmB,CAAC,cAAsB,EAAE,gBAAwB,EAAU;IACtF,IAAI,gBAAgB,IAAI,CAAC,EAAE,CAAC;QAC3B,OAAO,qBAAqB,CAAC;IAC9B,CAAC;IACD,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,yBAAyB,CAAC,GAAG,gBAAgB,CAAC;IAClG,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,cAAc,CAAC,CAAC,CAAC;AAAA,CAChF;AAED,SAAS,oBAAoB,CAAC,WAAwB,EAAE,KAAa,EAAU;IAC9E,MAAM,MAAM,GAAG,WAAW,CAAC,iBAAiB,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;IACtE,OAAO,gCAAgC,SAAS,CAAC,KAAK,CAAC,WAAW,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,MAAM,WAAW,WAAW,CAAC,OAAO,CAAC,MAAM,MAAM,CAAC;AAAA,CACnJ;AAED,SAAS,iBAAiB,CAAC,WAAwB,EAAE,OAAe,EAAE,YAAY,GAAG,qBAAqB,EAAU;IACnH,MAAM,OAAO,GACZ,WAAW,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY;QACxC,CAAC,CAAC,cAAc,CAAC,WAAW,CAAC,OAAO,EAAE,YAAY,CAAC;QACnD,CAAC,CAAC,WAAW,CAAC,OAAO,CAAC;IACxB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,CAAC;IACvF,OAAO,IAAI,OAAO,UAAU,SAAS,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS,MAAM,OAAO,OAAO,OAAO,GAAG,CAAC;AAAA,CACnG;AAED,SAAS,cAAc,CAAC,OAAe,EAAE,YAAY,GAAG,qBAAqB,EAAU;IACtF,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,YAAY,GAAG,SAAS,CAAC;IAC3C,IAAI,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACvC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACzC,IAAI,SAAS,GAAG,SAAS,GAAG,GAAG,EAAE,CAAC;QACjC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACjC,CAAC;IACD,IAAI,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC;IACrC,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACzC,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,GAAG,SAAS,GAAG,GAAG,EAAE,CAAC;QACpD,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,SAAS,EAAE,CAAC;IAC3C,CAAC;IACD,OAAO,GAAG,IAAI,0CAA0C,OAAO,CAAC,MAAM,GAAG,YAAY,iBAAiB,IAAI,EAAE,CAAC;AAAA,CAC7G;AAED,SAAS,iBAAiB,GAAW;IACpC,OAAO;QACN,+EAA+E;QAC/E,iFAAiF;QACjF,8KAA8K;QAC9K,EAAE;QACF,oBAAoB;KACpB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACb;AAED,SAAS,gBAAgB,CAAC,KAAY,EAAU;IAC/C,OAAO;QACN,WAAW;QACX,aAAa,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS;QAC3C,oBAAoB,SAAS,CAAC,KAAK,CAAC,WAAW,CAAC,gBAAgB;QAChE,iBAAiB,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,aAAa;QACvD,YAAY;KACZ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACb;AAED,MAAM,UAAU,SAAS,CAAC,GAAW,EAAU;IAC9C,OAAO,GAAG;SACR,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC;SACtB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,MAAM,CAAC;SACrB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC;SACvB,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;AAAA,CAC1B","sourcesContent":["import { type ContextBudgetItemV2, fnv1aHex } from \"./context-budget-governor-v2.ts\";\nimport { scoreContextFileRelevance, scoreSkillRelevance } from \"./context-budget-relevance.ts\";\nimport type { ContextFile } from \"./resource-loader.ts\";\nimport type { Skill } from \"./skills.ts\";\n\ninterface SystemPromptBudgetItemInput {\n\treadonly contextFiles: readonly ContextFile[];\n\treadonly skills: readonly Skill[];\n\treadonly includeSkills: boolean;\n\treadonly options: {\n\t\treadonly activeSkillNames?: readonly string[];\n\t\treadonly includeSkillInventory?: boolean;\n\t\treadonly includeFullContextFiles?: boolean;\n\t\treadonly maxInactiveSkills?: number;\n\t\treadonly queryContext?: string;\n\t};\n}\n\nconst CONTEXT_EXCERPT_CHARS = 1200;\nconst MIN_EXCERPT_CHARS = 400;\nconst MAX_EXCERPT_CHARS = 3000;\nconst MAX_INACTIVE_SKILLS = 15;\nconst CONTEXT_FULL_BUDGET_RATIO = 0.5;\n\nexport function createSystemPromptBudgetItems(\n\tinput: SystemPromptBudgetItemInput,\n\tresourceBudget: number,\n): ContextBudgetItemV2[] {\n\tconst items: ContextBudgetItemV2[] = [];\n\tconst contextFullCount = input.options.includeFullContextFiles !== false ? input.contextFiles.length : 0;\n\tconst excerptChars = computeExcerptChars(resourceBudget, contextFullCount);\n\n\tfor (const contextFile of input.contextFiles) {\n\t\tconst queryScore = scoreContextFileRelevance(\n\t\t\t{ path: contextFile.path, content: contextFile.content, isGlobal: contextFile.isGlobal ?? false },\n\t\t\tinput.options.queryContext,\n\t\t);\n\t\tpushContextItems(items, contextFile, queryScore, excerptChars, input.options.includeFullContextFiles !== false);\n\t}\n\n\tconst visibleSkills = input.skills.filter((skill) => !skill.disableModelInvocation);\n\tif (input.includeSkills && input.options.includeSkillInventory !== false && visibleSkills.length > 0) {\n\t\tpushSkillItems(items, visibleSkills, input.options);\n\t}\n\treturn items;\n}\n\nfunction pushContextItems(\n\titems: ContextBudgetItemV2[],\n\tcontextFile: ContextFile,\n\tqueryScore: number,\n\texcerptChars: number,\n\tincludeFullContextFiles: boolean,\n): void {\n\tconst scope = contextFile.isGlobal ? \"parent\" : \"project\";\n\tconst tagName = contextFile.isGlobal ? \"parent_instructions\" : \"project_instructions\";\n\titems.push({\n\t\tid: `context-pointer:${contextFile.path}`,\n\t\ttier: contextFile.isGlobal ? \"system\" : \"current-files\",\n\t\tpriority: contextFile.isGlobal ? \"hard\" : \"high\",\n\t\trelevance: (contextFile.isGlobal ? 1 : 0.8) * 0.4 + queryScore * 0.6,\n\t\ttext: renderContextPointer(contextFile, scope),\n\t});\n\tif (!includeFullContextFiles) return;\n\titems.push({\n\t\tid: `context-full:${contextFile.path}`,\n\t\ttier: contextFile.isGlobal ? \"system\" : \"current-files\",\n\t\tpriority: contextFile.isGlobal ? \"high\" : \"medium\",\n\t\trelevance: (contextFile.isGlobal ? 0.9 : 0.6) * 0.4 + queryScore * 0.6,\n\t\tredundancyKey: contextFile.path,\n\t\tsourceRef: {\n\t\t\turi: `file://${contextFile.path}`,\n\t\t\tcontentHash: fnv1aHex(contextFile.content),\n\t\t\tretrievable: true,\n\t\t},\n\t\ttext: renderContextFull(contextFile, tagName, excerptChars),\n\t});\n}\n\nfunction pushSkillItems(\n\titems: ContextBudgetItemV2[],\n\tvisibleSkills: readonly Skill[],\n\toptions: SystemPromptBudgetItemInput[\"options\"],\n): void {\n\titems.push({ id: \"skill-header\", tier: \"skills\", priority: \"hard\", text: renderSkillHeader() });\n\tconst activeSkillNames = new Set(options.activeSkillNames ?? []);\n\tconst maxInactive = options.maxInactiveSkills ?? MAX_INACTIVE_SKILLS;\n\tfor (const skill of visibleSkills.filter((skill) => activeSkillNames.has(skill.name))) {\n\t\titems.push(renderSkillBudgetItem(skill, \"hard\", 1));\n\t}\n\n\tconst sortedInactive = visibleSkills\n\t\t.filter((skill) => !activeSkillNames.has(skill.name))\n\t\t.sort((a, b) => scoreSkillRelevance(b, options.queryContext) - scoreSkillRelevance(a, options.queryContext));\n\tfor (const skill of sortedInactive.slice(0, maxInactive)) {\n\t\titems.push(renderSkillBudgetItem(skill, \"low\", Math.max(0.25, scoreSkillRelevance(skill, options.queryContext))));\n\t}\n\titems.push({ id: \"skill-footer\", tier: \"skills\", priority: \"hard\", text: \"</available_skills>\" });\n\n\tconst omittedInactiveCount = sortedInactive.length - Math.min(sortedInactive.length, maxInactive);\n\tif (omittedInactiveCount > 0) {\n\t\titems.push({\n\t\t\tid: \"skill-omitted-summary\",\n\t\t\ttier: \"skills\",\n\t\t\tpriority: \"low\",\n\t\t\trelevance: 0.1,\n\t\t\ttext: `<!-- ${omittedInactiveCount} additional skills available. Use 'read' to load a specific skill file when a task matches. -->`,\n\t\t});\n\t}\n}\n\nfunction renderSkillBudgetItem(skill: Skill, priority: \"hard\" | \"low\", relevance: number): ContextBudgetItemV2 {\n\treturn {\n\t\tid: `skill:${skill.name}`,\n\t\ttier: \"skills\",\n\t\tpriority,\n\t\trelevance,\n\t\tredundancyKey: skill.name,\n\t\ttext: renderSkillEntry(skill),\n\t};\n}\n\nfunction computeExcerptChars(resourceBudget: number, contextFullCount: number): number {\n\tif (contextFullCount <= 0) {\n\t\treturn CONTEXT_EXCERPT_CHARS;\n\t}\n\tconst allocatedTokens = Math.floor(resourceBudget * CONTEXT_FULL_BUDGET_RATIO) / contextFullCount;\n\tconst estimatedChars = Math.floor(allocatedTokens * 4);\n\treturn Math.max(MIN_EXCERPT_CHARS, Math.min(MAX_EXCERPT_CHARS, estimatedChars));\n}\n\nfunction renderContextPointer(contextFile: ContextFile, scope: string): string {\n\tconst marker = contextFile.containsJailbreak ? \" sanitized=true\" : \"\";\n\treturn `<context_file_pointer scope=\"${escapeXml(scope)}\" path=\"${escapeXml(contextFile.path)}\"${marker} chars=\"${contextFile.content.length}\" />`;\n}\n\nfunction renderContextFull(contextFile: ContextFile, tagName: string, excerptChars = CONTEXT_EXCERPT_CHARS): string {\n\tconst content =\n\t\tcontextFile.content.length > excerptChars\n\t\t\t? boundedExcerpt(contextFile.content, excerptChars)\n\t\t\t: contextFile.content;\n\tconst truncated = content.length < contextFile.content.length ? \" truncated=true\" : \"\";\n\treturn `<${tagName} path=\"${escapeXml(contextFile.path)}\"${truncated}>\\n${content}\\n</${tagName}>`;\n}\n\nfunction boundedExcerpt(content: string, excerptChars = CONTEXT_EXCERPT_CHARS): string {\n\tconst headChars = Math.floor(excerptChars * 0.7);\n\tconst tailChars = excerptChars - headChars;\n\tlet head = content.slice(0, headChars);\n\tconst lastSpace = head.search(/\\s+\\S*$/);\n\tif (lastSpace > headChars * 0.5) {\n\t\thead = head.slice(0, lastSpace);\n\t}\n\tlet tail = content.slice(-tailChars);\n\tconst firstSpace = tail.search(/\\S+\\s+/);\n\tif (firstSpace > 0 && firstSpace < tailChars * 0.5) {\n\t\ttail = tail.slice(firstSpace).trimStart();\n\t}\n\treturn `${head}\\n\\n[...context-budget excerpt omitted ${content.length - excerptChars} chars...]\\n\\n${tail}`;\n}\n\nfunction renderSkillHeader(): string {\n\treturn [\n\t\t\"\\n\\nThe following skills provide specialized instructions for specific tasks.\",\n\t\t\"Use the read tool to load a skill's file when the task matches its description.\",\n\t\t\"When a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.\",\n\t\t\"\",\n\t\t\"<available_skills>\",\n\t].join(\"\\n\");\n}\n\nfunction renderSkillEntry(skill: Skill): string {\n\treturn [\n\t\t\"  <skill>\",\n\t\t`    <name>${escapeXml(skill.name)}</name>`,\n\t\t`    <description>${escapeXml(skill.description)}</description>`,\n\t\t`    <location>${escapeXml(skill.filePath)}</location>`,\n\t\t\"  </skill>\",\n\t].join(\"\\n\");\n}\n\nexport function escapeXml(str: string): string {\n\treturn str\n\t\t.replace(/&/g, \"&amp;\")\n\t\t.replace(/</g, \"&lt;\")\n\t\t.replace(/>/g, \"&gt;\")\n\t\t.replace(/\"/g, \"&quot;\")\n\t\t.replace(/'/g, \"&apos;\");\n}\n"]}

package/dist/core/context-budget-system-prompt.d.ts

@@ -0,0 +1,31 @@
+import { type PromptContextBudgetPlanV2 } from "./context-budget-governor-v2.ts";
+import { type ContextBudgetTokenizerMode, type TokenCounterAdapter } from "./context-budget-token-counter.ts";
+import type { ContextFile } from "./resource-loader.ts";
+import type { Skill } from "./skills.ts";
+export interface SystemPromptContextBudgetOptions {
+    readonly maxPromptTokens: number;
+    readonly responseReserveTokens?: number;
+    readonly modelId?: string;
+    readonly tokenizerMode?: ContextBudgetTokenizerMode;
+    readonly activeSkillNames?: readonly string[];
+    readonly includeSkillInventory?: boolean;
+    readonly includeFullContextFiles?: boolean;
+    readonly tokenCounter?: TokenCounterAdapter;
+    /** Maximum number of inactive (non-active) skills to include as items. Default: 15. */
+    readonly maxInactiveSkills?: number;
+    /** Current user query text for relevance-aware skill ranking. */
+    readonly queryContext?: string;
+}
+export interface SystemPromptBudgetedResourcesInput {
+    readonly basePrompt: string;
+    readonly contextFiles: readonly ContextFile[];
+    readonly skills: readonly Skill[];
+    readonly includeSkills: boolean;
+    readonly options: SystemPromptContextBudgetOptions;
+}
+export interface SystemPromptBudgetedResources {
+    readonly text: string;
+    readonly plan: PromptContextBudgetPlanV2;
+}
+export declare function renderSystemPromptBudgetedResources(input: SystemPromptBudgetedResourcesInput): SystemPromptBudgetedResources;
+//# sourceMappingURL=context-budget-system-prompt.d.ts.map

package/dist/core/context-budget-system-prompt.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-budget-system-prompt.d.ts","sourceRoot":"","sources":["../../src/core/context-budget-system-prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EAGN,KAAK,yBAAyB,EAE9B,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EACN,KAAK,0BAA0B,EAE/B,KAAK,mBAAmB,EACxB,MAAM,mCAAmC,CAAC;AAC3C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAC;AACxD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEzC,MAAM,WAAW,gCAAgC;IAChD,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IACxC,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,aAAa,CAAC,EAAE,0BAA0B,CAAC;IACpD,QAAQ,CAAC,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC9C,QAAQ,CAAC,qBAAqB,CAAC,EAAE,OAAO,CAAC;IACzC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,OAAO,CAAC;IAC3C,QAAQ,CAAC,YAAY,CAAC,EAAE,mBAAmB,CAAC;IAC5C,uFAAuF;IACvF,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAC;IACpC,iEAAiE;IACjE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAC;CAC/B;AAED,MAAM,WAAW,kCAAkC;IAClD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,YAAY,EAAE,SAAS,WAAW,EAAE,CAAC;IAC9C,QAAQ,CAAC,MAAM,EAAE,SAAS,KAAK,EAAE,CAAC;IAClC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,gCAAgC,CAAC;CACnD;AAED,MAAM,WAAW,6BAA6B;IAC7C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,yBAAyB,CAAC;CACzC;AAED,wBAAgB,mCAAmC,CAClD,KAAK,EAAE,kCAAkC,GACvC,6BAA6B,CA4B/B","sourcesContent":["import {\n\tCONTEXT_BUDGET_POLICY_VERSION_V2,\n\ttype ContextBudgetItemV2,\n\ttype PromptContextBudgetPlanV2,\n\tplanPromptContextBudgetV2,\n} from \"./context-budget-governor-v2.ts\";\nimport { createSystemPromptBudgetItems, escapeXml } from \"./context-budget-system-prompt-items.ts\";\nimport {\n\ttype ContextBudgetTokenizerMode,\n\tcreateTokenCounterForMode,\n\ttype TokenCounterAdapter,\n} from \"./context-budget-token-counter.ts\";\nimport type { ContextFile } from \"./resource-loader.ts\";\nimport type { Skill } from \"./skills.ts\";\n\nexport interface SystemPromptContextBudgetOptions {\n\treadonly maxPromptTokens: number;\n\treadonly responseReserveTokens?: number;\n\treadonly modelId?: string;\n\treadonly tokenizerMode?: ContextBudgetTokenizerMode;\n\treadonly activeSkillNames?: readonly string[];\n\treadonly includeSkillInventory?: boolean;\n\treadonly includeFullContextFiles?: boolean;\n\treadonly tokenCounter?: TokenCounterAdapter;\n\t/** Maximum number of inactive (non-active) skills to include as items. Default: 15. */\n\treadonly maxInactiveSkills?: number;\n\t/** Current user query text for relevance-aware skill ranking. */\n\treadonly queryContext?: string;\n}\n\nexport interface SystemPromptBudgetedResourcesInput {\n\treadonly basePrompt: string;\n\treadonly contextFiles: readonly ContextFile[];\n\treadonly skills: readonly Skill[];\n\treadonly includeSkills: boolean;\n\treadonly options: SystemPromptContextBudgetOptions;\n}\n\nexport interface SystemPromptBudgetedResources {\n\treadonly text: string;\n\treadonly plan: PromptContextBudgetPlanV2;\n}\n\nexport function renderSystemPromptBudgetedResources(\n\tinput: SystemPromptBudgetedResourcesInput,\n): SystemPromptBudgetedResources {\n\tconst tokenCounter =\n\t\tinput.options.tokenCounter ?? createTokenCounterForMode(input.options.tokenizerMode ?? \"fallback\");\n\tconst modelId = input.options.modelId ?? \"unknown\";\n\tconst baseTokens = tokenCounter.countText(input.basePrompt, modelId).tokens;\n\tconst resourceBudget = Math.max(0, input.options.maxPromptTokens - baseTokens);\n\tconst items = createSystemPromptBudgetItems(input, resourceBudget);\n\tconst plan = planPromptContextBudgetV2({\n\t\tmaxTokens: resourceBudget,\n\t\tresponseReserveTokens: input.options.responseReserveTokens ?? 0,\n\t\tmodelId,\n\t\tpolicyVersion: CONTEXT_BUDGET_POLICY_VERSION_V2,\n\t\tquery: input.options.queryContext,\n\t\titems,\n\t\ttokenCounter,\n\t});\n\tconst included = new Set(plan.includedItemIds);\n\tconst filtered = deduplicatePointerFull(items, included);\n\tconst text = filtered\n\t\t.filter((item) => included.has(item.id))\n\t\t.map(\n\t\t\t(item) =>\n\t\t\t\tplan.selectedRepresentations.find((representation) => representation.itemId === item.id)?.text ?? item.text,\n\t\t)\n\t\t.join(\"\\n\")\n\t\t.trimEnd();\n\tconst note = renderBudgetNote(plan, baseTokens);\n\treturn { text: text ? `${text}\\n${note}` : note, plan };\n}\n\nfunction renderBudgetNote(plan: PromptContextBudgetPlanV2, baseTokens: number): string {\n\tconst omitted = plan.omittedItemIds.length;\n\tconst { counts, diagnosticReasons, tokenOptimizer, tokens } = plan.observability;\n\tconst renderedDiagnosticReasons =\n\t\tdiagnosticReasons.length === 0\n\t\t\t? '    <diagnostic_reasons count=\"0\" />'\n\t\t\t: [\n\t\t\t\t\t`    <diagnostic_reasons count=\"${formatObservableInteger(diagnosticReasons.length)}\">`,\n\t\t\t\t\t...diagnosticReasons.map((reason) => `      <reason>${escapeXml(reason)}</reason>`),\n\t\t\t\t\t\"    </diagnostic_reasons>\",\n\t\t\t\t].join(\"\\n\");\n\treturn [\n\t\t\"<context_budget>\",\n\t\t`  <policy>${escapeXml(plan.policyVersion)}</policy>`,\n\t\t`  <plan_hash>${escapeXml(plan.planHash)}</plan_hash>`,\n\t\t`  <base_prompt_tokens>${formatObservableInteger(baseTokens)}</base_prompt_tokens>`,\n\t\t`  <resource_tokens_used>${formatObservableInteger(plan.usedTokens)}</resource_tokens_used>`,\n\t\t`  <resource_tokens_omitted>${formatObservableInteger(plan.omittedTokens)}</resource_tokens_omitted>`,\n\t\t`  <omitted_items>${formatObservableInteger(omitted)}</omitted_items>`,\n\t\t`  <emergency>${plan.emergency ? \"true\" : \"false\"}</emergency>`,\n\t\t\"  <decision_observability>\",\n\t\t`    <counts selected=\"${formatObservableInteger(counts.selected)}\" omitted=\"${formatObservableInteger(counts.omitted)}\" pointer=\"${formatObservableInteger(counts.pointer)}\" compressed=\"${formatObservableInteger(counts.compressed)}\" full=\"${formatObservableInteger(counts.full)}\" retrieval_fallbacks=\"${formatObservableInteger(counts.retrievalFallback)}\" />`,\n\t\t`    <tokens available=\"${formatObservableInteger(tokens.available)}\" used=\"${formatObservableInteger(tokens.used)}\" raw=\"${formatObservableInteger(tokens.raw)}\" omitted=\"${formatObservableInteger(tokens.omitted)}\" token_savings=\"${formatObservableInteger(tokens.tokenSavings)}\" />`,\n\t\trenderedDiagnosticReasons,\n\t\t`    <token_optimizer optimizer_id=\"${escapeXml(tokenOptimizer.optimizerId)}\" status=\"${escapeXml(tokenOptimizer.status)}\" active=\"${tokenOptimizer.active ? \"true\" : \"false\"}\" active_context_budget_optimizer=\"${escapeXml(tokenOptimizer.activeContextBudgetOptimizer)}\" compatibility_only=\"${tokenOptimizer.compatibilityOnly ? \"true\" : \"false\"}\" />`,\n\t\t\"  </decision_observability>\",\n\t\t\"  <note>Some low-priority resource inventory may be represented by pointers. Use read on referenced paths when needed.</note>\",\n\t\t\"</context_budget>\",\n\t].join(\"\\n\");\n}\n\nfunction formatObservableInteger(value: number): string {\n\tif (!Number.isFinite(value)) {\n\t\treturn \"0\";\n\t}\n\treturn String(Math.max(0, Math.floor(value)));\n}\n\n/**\n * R3: When a context-full item is included, exclude the matching pointer\n * so the same file does not consume tokens twice.\n */\nfunction deduplicatePointerFull(\n\titems: readonly ContextBudgetItemV2[],\n\tincluded: ReadonlySet<string>,\n): readonly ContextBudgetItemV2[] {\n\tconst includedFullPaths = new Set<string>();\n\tfor (const item of items) {\n\t\tif (item.id.startsWith(\"context-full:\") && included.has(item.id)) {\n\t\t\tconst path = item.id.slice(\"context-full:\".length);\n\t\t\tincludedFullPaths.add(path);\n\t\t}\n\t}\n\tif (includedFullPaths.size === 0) {\n\t\treturn items;\n\t}\n\treturn items.filter((item) => {\n\t\tif (!item.id.startsWith(\"context-pointer:\")) {\n\t\t\treturn true;\n\t\t}\n\t\tconst path = item.id.slice(\"context-pointer:\".length);\n\t\treturn !includedFullPaths.has(path);\n\t});\n}\n"]}

package/dist/core/context-budget-system-prompt.js

@@ -0,0 +1,87 @@
+import { CONTEXT_BUDGET_POLICY_VERSION_V2, planPromptContextBudgetV2, } from "./context-budget-governor-v2.js";
+import { createSystemPromptBudgetItems, escapeXml } from "./context-budget-system-prompt-items.js";
+import { createTokenCounterForMode, } from "./context-budget-token-counter.js";
+export function renderSystemPromptBudgetedResources(input) {
+    const tokenCounter = input.options.tokenCounter ?? createTokenCounterForMode(input.options.tokenizerMode ?? "fallback");
+    const modelId = input.options.modelId ?? "unknown";
+    const baseTokens = tokenCounter.countText(input.basePrompt, modelId).tokens;
+    const resourceBudget = Math.max(0, input.options.maxPromptTokens - baseTokens);
+    const items = createSystemPromptBudgetItems(input, resourceBudget);
+    const plan = planPromptContextBudgetV2({
+        maxTokens: resourceBudget,
+        responseReserveTokens: input.options.responseReserveTokens ?? 0,
+        modelId,
+        policyVersion: CONTEXT_BUDGET_POLICY_VERSION_V2,
+        query: input.options.queryContext,
+        items,
+        tokenCounter,
+    });
+    const included = new Set(plan.includedItemIds);
+    const filtered = deduplicatePointerFull(items, included);
+    const text = filtered
+        .filter((item) => included.has(item.id))
+        .map((item) => plan.selectedRepresentations.find((representation) => representation.itemId === item.id)?.text ?? item.text)
+        .join("\n")
+        .trimEnd();
+    const note = renderBudgetNote(plan, baseTokens);
+    return { text: text ? `${text}\n${note}` : note, plan };
+}
+function renderBudgetNote(plan, baseTokens) {
+    const omitted = plan.omittedItemIds.length;
+    const { counts, diagnosticReasons, tokenOptimizer, tokens } = plan.observability;
+    const renderedDiagnosticReasons = diagnosticReasons.length === 0
+        ? '    <diagnostic_reasons count="0" />'
+        : [
+            `    <diagnostic_reasons count="${formatObservableInteger(diagnosticReasons.length)}">`,
+            ...diagnosticReasons.map((reason) => `      <reason>${escapeXml(reason)}</reason>`),
+            "    </diagnostic_reasons>",
+        ].join("\n");
+    return [
+        "<context_budget>",
+        `  <policy>${escapeXml(plan.policyVersion)}</policy>`,
+        `  <plan_hash>${escapeXml(plan.planHash)}</plan_hash>`,
+        `  <base_prompt_tokens>${formatObservableInteger(baseTokens)}</base_prompt_tokens>`,
+        `  <resource_tokens_used>${formatObservableInteger(plan.usedTokens)}</resource_tokens_used>`,
+        `  <resource_tokens_omitted>${formatObservableInteger(plan.omittedTokens)}</resource_tokens_omitted>`,
+        `  <omitted_items>${formatObservableInteger(omitted)}</omitted_items>`,
+        `  <emergency>${plan.emergency ? "true" : "false"}</emergency>`,
+        "  <decision_observability>",
+        `    <counts selected="${formatObservableInteger(counts.selected)}" omitted="${formatObservableInteger(counts.omitted)}" pointer="${formatObservableInteger(counts.pointer)}" compressed="${formatObservableInteger(counts.compressed)}" full="${formatObservableInteger(counts.full)}" retrieval_fallbacks="${formatObservableInteger(counts.retrievalFallback)}" />`,
+        `    <tokens available="${formatObservableInteger(tokens.available)}" used="${formatObservableInteger(tokens.used)}" raw="${formatObservableInteger(tokens.raw)}" omitted="${formatObservableInteger(tokens.omitted)}" token_savings="${formatObservableInteger(tokens.tokenSavings)}" />`,
+        renderedDiagnosticReasons,
+        `    <token_optimizer optimizer_id="${escapeXml(tokenOptimizer.optimizerId)}" status="${escapeXml(tokenOptimizer.status)}" active="${tokenOptimizer.active ? "true" : "false"}" active_context_budget_optimizer="${escapeXml(tokenOptimizer.activeContextBudgetOptimizer)}" compatibility_only="${tokenOptimizer.compatibilityOnly ? "true" : "false"}" />`,
+        "  </decision_observability>",
+        "  <note>Some low-priority resource inventory may be represented by pointers. Use read on referenced paths when needed.</note>",
+        "</context_budget>",
+    ].join("\n");
+}
+function formatObservableInteger(value) {
+    if (!Number.isFinite(value)) {
+        return "0";
+    }
+    return String(Math.max(0, Math.floor(value)));
+}
+/**
+ * R3: When a context-full item is included, exclude the matching pointer
+ * so the same file does not consume tokens twice.
+ */
+function deduplicatePointerFull(items, included) {
+    const includedFullPaths = new Set();
+    for (const item of items) {
+        if (item.id.startsWith("context-full:") && included.has(item.id)) {
+            const path = item.id.slice("context-full:".length);
+            includedFullPaths.add(path);
+        }
+    }
+    if (includedFullPaths.size === 0) {
+        return items;
+    }
+    return items.filter((item) => {
+        if (!item.id.startsWith("context-pointer:")) {
+            return true;
+        }
+        const path = item.id.slice("context-pointer:".length);
+        return !includedFullPaths.has(path);
+    });
+}
+//# sourceMappingURL=context-budget-system-prompt.js.map

package/dist/core/context-budget-system-prompt.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-budget-system-prompt.js","sourceRoot":"","sources":["../../src/core/context-budget-system-prompt.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,gCAAgC,EAGhC,yBAAyB,GACzB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAAE,6BAA6B,EAAE,SAAS,EAAE,MAAM,yCAAyC,CAAC;AACnG,OAAO,EAEN,yBAAyB,GAEzB,MAAM,mCAAmC,CAAC;AAgC3C,MAAM,UAAU,mCAAmC,CAClD,KAAyC,EACT;IAChC,MAAM,YAAY,GACjB,KAAK,CAAC,OAAO,CAAC,YAAY,IAAI,yBAAyB,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,IAAI,UAAU,CAAC,CAAC;IACpG,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC;IACnD,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC,MAAM,CAAC;IAC5E,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,OAAO,CAAC,eAAe,GAAG,UAAU,CAAC,CAAC;IAC/E,MAAM,KAAK,GAAG,6BAA6B,CAAC,KAAK,EAAE,cAAc,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,yBAAyB,CAAC;QACtC,SAAS,EAAE,cAAc;QACzB,qBAAqB,EAAE,KAAK,CAAC,OAAO,CAAC,qBAAqB,IAAI,CAAC;QAC/D,OAAO;QACP,aAAa,EAAE,gCAAgC;QAC/C,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,YAAY;QACjC,KAAK;QACL,YAAY;KACZ,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,sBAAsB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IACzD,MAAM,IAAI,GAAG,QAAQ;SACnB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;SACvC,GAAG,CACH,CAAC,IAAI,EAAE,EAAE,CACR,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,KAAK,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC,IAAI,CAC5G;SACA,IAAI,CAAC,IAAI,CAAC;SACV,OAAO,EAAE,CAAC;IACZ,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,EAAE,UAAU,CAAC,CAAC;IAChD,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;AAAA,CACxD;AAED,SAAS,gBAAgB,CAAC,IAA+B,EAAE,UAAkB,EAAU;IACtF,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;IAC3C,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC;IACjF,MAAM,yBAAyB,GAC9B,iBAAiB,CAAC,MAAM,KAAK,CAAC;QAC7B,CAAC,CAAC,sCAAsC;QACxC,CAAC,CAAC;YACA,kCAAkC,uBAAuB,CAAC,iBAAiB,CAAC,MAAM,CAAC,IAAI;YACvF,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,iBAAiB,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC;YACnF,2BAA2B;SAC3B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChB,OAAO;QACN,kBAAkB;QAClB,aAAa,SAAS,CAAC,IAAI,CAAC,aAAa,CAAC,WAAW;QACrD,gBAAgB,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,cAAc;QACtD,yBAAyB,uBAAuB,CAAC,UAAU,CAAC,uBAAuB;QACnF,2BAA2B,uBAAuB,CAAC,IAAI,CAAC,UAAU,CAAC,yBAAyB;QAC5F,8BAA8B,uBAAuB,CAAC,IAAI,CAAC,aAAa,CAAC,4BAA4B;QACrG,oBAAoB,uBAAuB,CAAC,OAAO,CAAC,kBAAkB;QACtE,gBAAgB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,cAAc;QAC/D,4BAA4B;QAC5B,yBAAyB,uBAAuB,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,uBAAuB,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,uBAAuB,CAAC,MAAM,CAAC,OAAO,CAAC,iBAAiB,uBAAuB,CAAC,MAAM,CAAC,UAAU,CAAC,WAAW,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,uBAAuB,CAAC,MAAM,CAAC,iBAAiB,CAAC,MAAM;QACtW,0BAA0B,uBAAuB,CAAC,MAAM,CAAC,SAAS,CAAC,WAAW,uBAAuB,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,uBAAuB,CAAC,MAAM,CAAC,GAAG,CAAC,cAAc,uBAAuB,CAAC,MAAM,CAAC,OAAO,CAAC,oBAAoB,uBAAuB,CAAC,MAAM,CAAC,YAAY,CAAC,MAAM;QAC1R,yBAAyB;QACzB,sCAAsC,SAAS,CAAC,cAAc,CAAC,WAAW,CAAC,aAAa,SAAS,CAAC,cAAc,CAAC,MAAM,CAAC,aAAa,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,sCAAsC,SAAS,CAAC,cAAc,CAAC,4BAA4B,CAAC,yBAAyB,cAAc,CAAC,iBAAiB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,MAAM;QAC3V,6BAA6B;QAC7B,+HAA+H;QAC/H,mBAAmB;KACnB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAAA,CACb;AAED,SAAS,uBAAuB,CAAC,KAAa,EAAU;IACvD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC;IACZ,CAAC;IACD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAAA,CAC9C;AAED;;;GAGG;AACH,SAAS,sBAAsB,CAC9B,KAAqC,EACrC,QAA6B,EACI;IACjC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;IAC5C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAClE,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,eAAe,CAAC,MAAM,CAAC,CAAC;YACnD,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;IACF,CAAC;IACD,IAAI,iBAAiB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;QAClC,OAAO,KAAK,CAAC;IACd,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;QAC7B,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,kBAAkB,CAAC,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC;QACb,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QACtD,OAAO,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAAA,CACpC,CAAC,CAAC;AAAA,CACH","sourcesContent":["import {\n\tCONTEXT_BUDGET_POLICY_VERSION_V2,\n\ttype ContextBudgetItemV2,\n\ttype PromptContextBudgetPlanV2,\n\tplanPromptContextBudgetV2,\n} from \"./context-budget-governor-v2.ts\";\nimport { createSystemPromptBudgetItems, escapeXml } from \"./context-budget-system-prompt-items.ts\";\nimport {\n\ttype ContextBudgetTokenizerMode,\n\tcreateTokenCounterForMode,\n\ttype TokenCounterAdapter,\n} from \"./context-budget-token-counter.ts\";\nimport type { ContextFile } from \"./resource-loader.ts\";\nimport type { Skill } from \"./skills.ts\";\n\nexport interface SystemPromptContextBudgetOptions {\n\treadonly maxPromptTokens: number;\n\treadonly responseReserveTokens?: number;\n\treadonly modelId?: string;\n\treadonly tokenizerMode?: ContextBudgetTokenizerMode;\n\treadonly activeSkillNames?: readonly string[];\n\treadonly includeSkillInventory?: boolean;\n\treadonly includeFullContextFiles?: boolean;\n\treadonly tokenCounter?: TokenCounterAdapter;\n\t/** Maximum number of inactive (non-active) skills to include as items. Default: 15. */\n\treadonly maxInactiveSkills?: number;\n\t/** Current user query text for relevance-aware skill ranking. */\n\treadonly queryContext?: string;\n}\n\nexport interface SystemPromptBudgetedResourcesInput {\n\treadonly basePrompt: string;\n\treadonly contextFiles: readonly ContextFile[];\n\treadonly skills: readonly Skill[];\n\treadonly includeSkills: boolean;\n\treadonly options: SystemPromptContextBudgetOptions;\n}\n\nexport interface SystemPromptBudgetedResources {\n\treadonly text: string;\n\treadonly plan: PromptContextBudgetPlanV2;\n}\n\nexport function renderSystemPromptBudgetedResources(\n\tinput: SystemPromptBudgetedResourcesInput,\n): SystemPromptBudgetedResources {\n\tconst tokenCounter =\n\t\tinput.options.tokenCounter ?? createTokenCounterForMode(input.options.tokenizerMode ?? \"fallback\");\n\tconst modelId = input.options.modelId ?? \"unknown\";\n\tconst baseTokens = tokenCounter.countText(input.basePrompt, modelId).tokens;\n\tconst resourceBudget = Math.max(0, input.options.maxPromptTokens - baseTokens);\n\tconst items = createSystemPromptBudgetItems(input, resourceBudget);\n\tconst plan = planPromptContextBudgetV2({\n\t\tmaxTokens: resourceBudget,\n\t\tresponseReserveTokens: input.options.responseReserveTokens ?? 0,\n\t\tmodelId,\n\t\tpolicyVersion: CONTEXT_BUDGET_POLICY_VERSION_V2,\n\t\tquery: input.options.queryContext,\n\t\titems,\n\t\ttokenCounter,\n\t});\n\tconst included = new Set(plan.includedItemIds);\n\tconst filtered = deduplicatePointerFull(items, included);\n\tconst text = filtered\n\t\t.filter((item) => included.has(item.id))\n\t\t.map(\n\t\t\t(item) =>\n\t\t\t\tplan.selectedRepresentations.find((representation) => representation.itemId === item.id)?.text ?? item.text,\n\t\t)\n\t\t.join(\"\\n\")\n\t\t.trimEnd();\n\tconst note = renderBudgetNote(plan, baseTokens);\n\treturn { text: text ? `${text}\\n${note}` : note, plan };\n}\n\nfunction renderBudgetNote(plan: PromptContextBudgetPlanV2, baseTokens: number): string {\n\tconst omitted = plan.omittedItemIds.length;\n\tconst { counts, diagnosticReasons, tokenOptimizer, tokens } = plan.observability;\n\tconst renderedDiagnosticReasons =\n\t\tdiagnosticReasons.length === 0\n\t\t\t? '    <diagnostic_reasons count=\"0\" />'\n\t\t\t: [\n\t\t\t\t\t`    <diagnostic_reasons count=\"${formatObservableInteger(diagnosticReasons.length)}\">`,\n\t\t\t\t\t...diagnosticReasons.map((reason) => `      <reason>${escapeXml(reason)}</reason>`),\n\t\t\t\t\t\"    </diagnostic_reasons>\",\n\t\t\t\t].join(\"\\n\");\n\treturn [\n\t\t\"<context_budget>\",\n\t\t`  <policy>${escapeXml(plan.policyVersion)}</policy>`,\n\t\t`  <plan_hash>${escapeXml(plan.planHash)}</plan_hash>`,\n\t\t`  <base_prompt_tokens>${formatObservableInteger(baseTokens)}</base_prompt_tokens>`,\n\t\t`  <resource_tokens_used>${formatObservableInteger(plan.usedTokens)}</resource_tokens_used>`,\n\t\t`  <resource_tokens_omitted>${formatObservableInteger(plan.omittedTokens)}</resource_tokens_omitted>`,\n\t\t`  <omitted_items>${formatObservableInteger(omitted)}</omitted_items>`,\n\t\t`  <emergency>${plan.emergency ? \"true\" : \"false\"}</emergency>`,\n\t\t\"  <decision_observability>\",\n\t\t`    <counts selected=\"${formatObservableInteger(counts.selected)}\" omitted=\"${formatObservableInteger(counts.omitted)}\" pointer=\"${formatObservableInteger(counts.pointer)}\" compressed=\"${formatObservableInteger(counts.compressed)}\" full=\"${formatObservableInteger(counts.full)}\" retrieval_fallbacks=\"${formatObservableInteger(counts.retrievalFallback)}\" />`,\n\t\t`    <tokens available=\"${formatObservableInteger(tokens.available)}\" used=\"${formatObservableInteger(tokens.used)}\" raw=\"${formatObservableInteger(tokens.raw)}\" omitted=\"${formatObservableInteger(tokens.omitted)}\" token_savings=\"${formatObservableInteger(tokens.tokenSavings)}\" />`,\n\t\trenderedDiagnosticReasons,\n\t\t`    <token_optimizer optimizer_id=\"${escapeXml(tokenOptimizer.optimizerId)}\" status=\"${escapeXml(tokenOptimizer.status)}\" active=\"${tokenOptimizer.active ? \"true\" : \"false\"}\" active_context_budget_optimizer=\"${escapeXml(tokenOptimizer.activeContextBudgetOptimizer)}\" compatibility_only=\"${tokenOptimizer.compatibilityOnly ? \"true\" : \"false\"}\" />`,\n\t\t\"  </decision_observability>\",\n\t\t\"  <note>Some low-priority resource inventory may be represented by pointers. Use read on referenced paths when needed.</note>\",\n\t\t\"</context_budget>\",\n\t].join(\"\\n\");\n}\n\nfunction formatObservableInteger(value: number): string {\n\tif (!Number.isFinite(value)) {\n\t\treturn \"0\";\n\t}\n\treturn String(Math.max(0, Math.floor(value)));\n}\n\n/**\n * R3: When a context-full item is included, exclude the matching pointer\n * so the same file does not consume tokens twice.\n */\nfunction deduplicatePointerFull(\n\titems: readonly ContextBudgetItemV2[],\n\tincluded: ReadonlySet<string>,\n): readonly ContextBudgetItemV2[] {\n\tconst includedFullPaths = new Set<string>();\n\tfor (const item of items) {\n\t\tif (item.id.startsWith(\"context-full:\") && included.has(item.id)) {\n\t\t\tconst path = item.id.slice(\"context-full:\".length);\n\t\t\tincludedFullPaths.add(path);\n\t\t}\n\t}\n\tif (includedFullPaths.size === 0) {\n\t\treturn items;\n\t}\n\treturn items.filter((item) => {\n\t\tif (!item.id.startsWith(\"context-pointer:\")) {\n\t\t\treturn true;\n\t\t}\n\t\tconst path = item.id.slice(\"context-pointer:\".length);\n\t\treturn !includedFullPaths.has(path);\n\t});\n}\n"]}

package/dist/core/context-budget-token-counter.d.ts

@@ -0,0 +1,34 @@
+export type ContextBudgetTokenCountMethod = "exact" | "estimated";
+export type ContextBudgetTokenConfidence = "high" | "medium" | "low";
+export type ContextBudgetTokenizerMode = "auto" | "fallback" | "openai-js" | "openai-wasm";
+export interface TokenCountResult {
+    readonly tokens: number;
+    readonly method: ContextBudgetTokenCountMethod;
+    readonly confidence: ContextBudgetTokenConfidence;
+    readonly adapterId: string;
+    readonly modelId: string;
+    readonly notes: readonly string[];
+}
+export interface TokenCounterAdapter {
+    readonly id: string;
+    readonly priority: number;
+    isAvailable(): boolean;
+    supports(modelId: string): boolean;
+    countText(input: string, modelId: string): TokenCountResult;
+}
+export interface OptionalModuleLoader {
+    resolve(specifier: string): string | undefined;
+    load(specifier: string): unknown;
+}
+export interface TokenCounterRegistryOptions {
+    readonly adapters?: readonly TokenCounterAdapter[];
+    readonly fallback?: TokenCounterAdapter;
+}
+export declare function createNodeOptionalModuleLoader(): OptionalModuleLoader;
+export declare function createFallbackTokenCounter(): TokenCounterAdapter;
+export declare function estimateTextTokens(input: string, modelId?: string): TokenCountResult;
+export declare function createOpenAiJsTokenCounter(loader?: OptionalModuleLoader): TokenCounterAdapter;
+export declare function createOpenAiWasmTokenCounter(loader?: OptionalModuleLoader): TokenCounterAdapter;
+export declare function createTokenCounterForMode(mode: ContextBudgetTokenizerMode, loader?: OptionalModuleLoader): TokenCounterAdapter;
+export declare function createTokenCounterRegistry(options?: TokenCounterRegistryOptions): TokenCounterAdapter;
+//# sourceMappingURL=context-budget-token-counter.d.ts.map

package/dist/core/context-budget-token-counter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context-budget-token-counter.d.ts","sourceRoot":"","sources":["../../src/core/context-budget-token-counter.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,6BAA6B,GAAG,OAAO,GAAG,WAAW,CAAC;AAClE,MAAM,MAAM,4BAA4B,GAAG,MAAM,GAAG,QAAQ,GAAG,KAAK,CAAC;AACrE,MAAM,MAAM,0BAA0B,GAAG,MAAM,GAAG,UAAU,GAAG,WAAW,GAAG,aAAa,CAAC;AAE3F,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,6BAA6B,CAAC;IAC/C,QAAQ,CAAC,UAAU,EAAE,4BAA4B,CAAC;IAClD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;CAClC;AAED,MAAM,WAAW,mBAAmB;IACnC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,WAAW,IAAI,OAAO,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC;IACnC,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,gBAAgB,CAAC;CAC5D;AAED,MAAM,WAAW,oBAAoB;IACpC,OAAO,CAAC,SAAS,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC;IAC/C,IAAI,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC;CACjC;AAED,MAAM,WAAW,2BAA2B;IAC3C,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,mBAAmB,EAAE,CAAC;IACnD,QAAQ,CAAC,QAAQ,CAAC,EAAE,mBAAmB,CAAC;CACxC;AAiBD,wBAAgB,8BAA8B,IAAI,oBAAoB,CAarE;AAED,wBAAgB,0BAA0B,IAAI,mBAAmB,CAUhE;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,SAAY,GAAG,gBAAgB,CAgEvF;AAED,wBAAgB,0BAA0B,CACzC,MAAM,GAAE,oBAAuD,GAC7D,mBAAmB,CAwBrB;AAED,wBAAgB,4BAA4B,CAC3C,MAAM,GAAE,oBAAuD,GAC7D,mBAAmB,CAwBrB;AAED,wBAAgB,yBAAyB,CACxC,IAAI,EAAE,0BAA0B,EAChC,MAAM,GAAE,oBAAuD,GAC7D,mBAAmB,CAWrB;AAED,wBAAgB,0BAA0B,CAAC,OAAO,GAAE,2BAAgC,GAAG,mBAAmB,CA6BzG","sourcesContent":["import { createRequire } from \"node:module\";\n\nexport type ContextBudgetTokenCountMethod = \"exact\" | \"estimated\";\nexport type ContextBudgetTokenConfidence = \"high\" | \"medium\" | \"low\";\nexport type ContextBudgetTokenizerMode = \"auto\" | \"fallback\" | \"openai-js\" | \"openai-wasm\";\n\nexport interface TokenCountResult {\n\treadonly tokens: number;\n\treadonly method: ContextBudgetTokenCountMethod;\n\treadonly confidence: ContextBudgetTokenConfidence;\n\treadonly adapterId: string;\n\treadonly modelId: string;\n\treadonly notes: readonly string[];\n}\n\nexport interface TokenCounterAdapter {\n\treadonly id: string;\n\treadonly priority: number;\n\tisAvailable(): boolean;\n\tsupports(modelId: string): boolean;\n\tcountText(input: string, modelId: string): TokenCountResult;\n}\n\nexport interface OptionalModuleLoader {\n\tresolve(specifier: string): string | undefined;\n\tload(specifier: string): unknown;\n}\n\nexport interface TokenCounterRegistryOptions {\n\treadonly adapters?: readonly TokenCounterAdapter[];\n\treadonly fallback?: TokenCounterAdapter;\n}\n\ninterface EncodeCapable {\n\tencode(input: string): readonly unknown[];\n}\n\ninterface JsTiktokenModule {\n\tencodingForModel?: (modelId: string) => EncodeCapable;\n\tgetEncoding?: (encoding: string) => EncodeCapable;\n}\n\ninterface GenericEncodeModule {\n\tencode?: (input: string) => readonly unknown[];\n}\n\nconst requireModule = createRequire(import.meta.url);\n\nexport function createNodeOptionalModuleLoader(): OptionalModuleLoader {\n\treturn {\n\t\tresolve(specifier) {\n\t\t\ttry {\n\t\t\t\treturn requireModule.resolve(specifier);\n\t\t\t} catch {\n\t\t\t\treturn undefined;\n\t\t\t}\n\t\t},\n\t\tload(specifier) {\n\t\t\treturn requireModule(specifier) as unknown;\n\t\t},\n\t};\n}\n\nexport function createFallbackTokenCounter(): TokenCounterAdapter {\n\treturn {\n\t\tid: \"fallback-estimator\",\n\t\tpriority: 0,\n\t\tisAvailable: () => true,\n\t\tsupports: () => true,\n\t\tcountText(input, modelId) {\n\t\t\treturn estimateTextTokens(input, modelId);\n\t\t},\n\t};\n}\n\nexport function estimateTextTokens(input: string, modelId = \"unknown\"): TokenCountResult {\n\tif (input.length === 0) {\n\t\treturn createTokenResult(0, \"estimated\", \"medium\", \"fallback-estimator\", modelId, [\"empty-input\"]);\n\t}\n\n\tlet asciiWord = 0;\n\tlet whitespace = 0;\n\tlet cjk = 0;\n\tlet hangul = 0;\n\tlet kana = 0;\n\tlet emojiOrWide = 0;\n\tlet punctuation = 0;\n\tlet other = 0;\n\n\tfor (const char of input) {\n\t\tconst codePoint = char.codePointAt(0) ?? 0;\n\t\tif (/\\s/u.test(char)) {\n\t\t\twhitespace += 1;\n\t\t} else if (isHangul(codePoint)) {\n\t\t\thangul += 1;\n\t\t} else if (isHiraganaOrKatakana(codePoint)) {\n\t\t\tkana += 1;\n\t\t} else if (isCjkIdeograph(codePoint)) {\n\t\t\tcjk += 1;\n\t\t} else if (isAsciiAlphaNumeric(codePoint) || char === \"_\") {\n\t\t\tasciiWord += 1;\n\t\t} else if (isEmojiOrWideSymbol(codePoint)) {\n\t\t\temojiOrWide += 1;\n\t\t} else if (isAsciiPunctuation(codePoint)) {\n\t\t\tpunctuation += 1;\n\t\t} else {\n\t\t\tother += 1;\n\t\t}\n\t}\n\n\tconst codeLike = looksCodeLike(input, punctuation, whitespace);\n\tconst jsonLike = looksJsonLike(input);\n\tconst nonAsciiRatio = (hangul + cjk + kana + emojiOrWide + other) / Math.max(1, input.length);\n\tconst base =\n\t\tasciiWord / (codeLike ? 3.2 : 4) +\n\t\twhitespace / 12 +\n\t\tpunctuation / 2.1 +\n\t\thangul / 0.95 +\n\t\tkana / 1.0 +\n\t\tcjk / 1.2 +\n\t\temojiOrWide * 1.8 +\n\t\tother / 2;\n\tconst adjusted = base * (jsonLike ? 1.12 : 1) * (codeLike ? 1.08 : 1);\n\tconst tokens = Math.max(1, Math.ceil(adjusted));\n\tconst compositionParts: string[] = [];\n\tif (hangul > 0) compositionParts.push(`hangul:${hangul}`);\n\tif (kana > 0) compositionParts.push(`kana:${kana}`);\n\tif (cjk > 0) compositionParts.push(`cjk:${cjk}`);\n\tif (asciiWord > 0) compositionParts.push(`ascii:${asciiWord}`);\n\tif (emojiOrWide > 0) compositionParts.push(`emoji:${emojiOrWide}`);\n\tconst notes = [\n\t\tcodeLike ? \"code-like\" : \"prose-like\",\n\t\tjsonLike ? \"json-like\" : \"not-json-like\",\n\t\tnonAsciiRatio > 0 ? `non-ascii:${(nonAsciiRatio * 100).toFixed(0)}%` : \"latin-only\",\n\t\t`composition(${compositionParts.join(\",\")})`,\n\t];\n\tconst confidence: ContextBudgetTokenConfidence =\n\t\tnonAsciiRatio > 0.4 ? \"low\" : nonAsciiRatio > 0.15 ? \"medium\" : \"high\";\n\treturn createTokenResult(tokens, \"estimated\", confidence, \"fallback-estimator\", modelId, notes);\n}\n\nexport function createOpenAiJsTokenCounter(\n\tloader: OptionalModuleLoader = createNodeOptionalModuleLoader(),\n): TokenCounterAdapter {\n\tconst packageNames = [\"js-tiktoken\", \"gpt-tokenizer\", \"tiktoken\"] as const;\n\treturn {\n\t\tid: \"openai-bpe-js\",\n\t\tpriority: 80,\n\t\tisAvailable() {\n\t\t\treturn packageNames.some((specifier) => loader.resolve(specifier) !== undefined);\n\t\t},\n\t\tsupports(modelId) {\n\t\t\treturn isOpenAiStyleModel(modelId);\n\t\t},\n\t\tcountText(input, modelId) {\n\t\t\tfor (const specifier of packageNames) {\n\t\t\t\tif (loader.resolve(specifier) === undefined) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tconst result = countWithOptionalTokenizerModule(loader.load(specifier), specifier, input, modelId);\n\t\t\t\tif (result !== undefined) {\n\t\t\t\t\treturn result;\n\t\t\t\t}\n\t\t\t}\n\t\t\tthrow new Error(\"no supported OpenAI JS tokenizer module shape found\");\n\t\t},\n\t};\n}\n\nexport function createOpenAiWasmTokenCounter(\n\tloader: OptionalModuleLoader = createNodeOptionalModuleLoader(),\n): TokenCounterAdapter {\n\tconst packageNames = [\"@dqbd/tiktoken\", \"tiktoken\"] as const;\n\treturn {\n\t\tid: \"openai-bpe-wasm\",\n\t\tpriority: 90,\n\t\tisAvailable() {\n\t\t\treturn packageNames.some((specifier) => loader.resolve(specifier) !== undefined);\n\t\t},\n\t\tsupports(modelId) {\n\t\t\treturn isOpenAiStyleModel(modelId);\n\t\t},\n\t\tcountText(input, modelId) {\n\t\t\tfor (const specifier of packageNames) {\n\t\t\t\tif (loader.resolve(specifier) === undefined) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tconst result = countWithOptionalTokenizerModule(loader.load(specifier), specifier, input, modelId);\n\t\t\t\tif (result !== undefined) {\n\t\t\t\t\treturn result;\n\t\t\t\t}\n\t\t\t}\n\t\t\tthrow new Error(\"no supported OpenAI WASM tokenizer module shape found\");\n\t\t},\n\t};\n}\n\nexport function createTokenCounterForMode(\n\tmode: ContextBudgetTokenizerMode,\n\tloader: OptionalModuleLoader = createNodeOptionalModuleLoader(),\n): TokenCounterAdapter {\n\tconst fallback = createFallbackTokenCounter();\n\tif (mode === \"fallback\") {\n\t\treturn fallback;\n\t}\n\tconst adapters =\n\t\tmode === \"openai-wasm\" ? [createOpenAiWasmTokenCounter(loader)] : [createOpenAiJsTokenCounter(loader)];\n\tif (mode === \"auto\") {\n\t\tadapters.push(createOpenAiWasmTokenCounter(loader));\n\t}\n\treturn createTokenCounterRegistry({ adapters, fallback });\n}\n\nexport function createTokenCounterRegistry(options: TokenCounterRegistryOptions = {}): TokenCounterAdapter {\n\tconst fallback = options.fallback ?? createFallbackTokenCounter();\n\tconst adapters = [...(options.adapters ?? [])].sort((a, b) => b.priority - a.priority || a.id.localeCompare(b.id));\n\treturn {\n\t\tid: \"token-counter-registry\",\n\t\tpriority: 100,\n\t\tisAvailable: () => true,\n\t\tsupports: () => true,\n\t\tcountText(input, modelId) {\n\t\t\tconst notes: string[] = [];\n\t\t\tfor (const adapter of adapters) {\n\t\t\t\tif (!adapter.supports(modelId)) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\ttry {\n\t\t\t\t\tif (!adapter.isAvailable()) {\n\t\t\t\t\t\tnotes.push(`${adapter.id}:unavailable`);\n\t\t\t\t\t\tcontinue;\n\t\t\t\t\t}\n\t\t\t\t\treturn adapter.countText(input, modelId);\n\t\t\t\t} catch (error) {\n\t\t\t\t\tconst message = error instanceof Error ? error.message : \"unknown adapter failure\";\n\t\t\t\t\tnotes.push(`${adapter.id}:failed:${message}`);\n\t\t\t\t}\n\t\t\t}\n\t\t\tconst result = fallback.countText(input, modelId);\n\t\t\treturn { ...result, notes: [...notes, ...result.notes] };\n\t\t},\n\t};\n}\n\nfunction countWithOptionalTokenizerModule(\n\tmoduleValue: unknown,\n\tspecifier: string,\n\tinput: string,\n\tmodelId: string,\n): TokenCountResult | undefined {\n\tconst moduleObject = unwrapDefaultModule(moduleValue);\n\tconst jsTiktoken = moduleObject as JsTiktokenModule;\n\tif (typeof jsTiktoken.encodingForModel === \"function\") {\n\t\tconst encoding = jsTiktoken.encodingForModel(modelId);\n\t\treturn countWithEncoding(encoding, specifier, input, modelId, \"model-encoding\");\n\t}\n\tif (typeof jsTiktoken.getEncoding === \"function\") {\n\t\tconst encoding = jsTiktoken.getEncoding(selectOpenAiEncoding(modelId));\n\t\treturn countWithEncoding(encoding, specifier, input, modelId, \"fallback-encoding\");\n\t}\n\tconst generic = moduleObject as GenericEncodeModule;\n\tif (typeof generic.encode === \"function\") {\n\t\treturn createTokenResult(generic.encode(input).length, \"exact\", \"medium\", specifier, modelId, [\"generic-encode\"]);\n\t}\n\treturn undefined;\n}\n\nfunction countWithEncoding(\n\tencoding: EncodeCapable,\n\tadapterId: string,\n\tinput: string,\n\tmodelId: string,\n\tnote: string,\n): TokenCountResult {\n\treturn createTokenResult(encoding.encode(input).length, \"exact\", \"high\", adapterId, modelId, [note]);\n}\n\nfunction unwrapDefaultModule(moduleValue: unknown): unknown {\n\tif (moduleValue && typeof moduleValue === \"object\" && \"default\" in moduleValue) {\n\t\treturn (moduleValue as { readonly default: unknown }).default;\n\t}\n\treturn moduleValue;\n}\n\nfunction createTokenResult(\n\ttokens: number,\n\tmethod: ContextBudgetTokenCountMethod,\n\tconfidence: ContextBudgetTokenConfidence,\n\tadapterId: string,\n\tmodelId: string,\n\tnotes: readonly string[],\n): TokenCountResult {\n\treturn {\n\t\ttokens: Math.max(0, Math.ceil(tokens)),\n\t\tmethod,\n\t\tconfidence,\n\t\tadapterId,\n\t\tmodelId,\n\t\tnotes,\n\t};\n}\n\nfunction isOpenAiStyleModel(modelId: string): boolean {\n\tconst normalized = modelId.toLowerCase();\n\treturn (\n\t\tnormalized.includes(\"openai\") ||\n\t\tnormalized.startsWith(\"gpt-\") ||\n\t\tnormalized.startsWith(\"o1\") ||\n\t\tnormalized.startsWith(\"o3\") ||\n\t\tnormalized.startsWith(\"o4\") ||\n\t\tnormalized.includes(\"chatgpt\")\n\t);\n}\n\nfunction selectOpenAiEncoding(modelId: string): string {\n\tconst normalized = modelId.toLowerCase();\n\tif (normalized.includes(\"gpt-4o\") || normalized.includes(\"gpt-5\") || normalized.startsWith(\"o\")) {\n\t\treturn \"o200k_base\";\n\t}\n\treturn \"cl100k_base\";\n}\n\nfunction isAsciiAlphaNumeric(codePoint: number): boolean {\n\treturn (\n\t\t(codePoint >= 48 && codePoint <= 57) ||\n\t\t(codePoint >= 65 && codePoint <= 90) ||\n\t\t(codePoint >= 97 && codePoint <= 122)\n\t);\n}\n\nfunction isAsciiPunctuation(codePoint: number): boolean {\n\treturn codePoint >= 33 && codePoint <= 126;\n}\n\nfunction isHangul(codePoint: number): boolean {\n\t// Composed syllables (가-힣)\n\tif (codePoint >= 0xac00 && codePoint <= 0xd7af) return true;\n\t// Jamo: initial (ㄱ-ㅎ), medial (ㅏ-ㅣ)\n\tif (codePoint >= 0x1100 && codePoint <= 0x11ff) return true;\n\t// Compatibility jamo (ㄱ-ㅎ, ㅏ-ㅣ) and Hangul letters\n\tif (codePoint >= 0x3130 && codePoint <= 0x318f) return true;\n\t// Extended jamo\n\tif (codePoint >= 0xa960 && codePoint <= 0xa97c) return true;\n\treturn false;\n}\n\nfunction isHiraganaOrKatakana(codePoint: number): boolean {\n\t// Hiragana (ぁ-より)\n\tif (codePoint >= 0x3040 && codePoint <= 0x309f) return true;\n\t// Katakana (ァ-ヿ) + halfwidth katakana\n\tif (codePoint >= 0x30a0 && codePoint <= 0x30ff) return true;\n\tif (codePoint >= 0xff65 && codePoint <= 0xff9f) return true;\n\treturn false;\n}\n\nfunction isCjkIdeograph(codePoint: number): boolean {\n\t// CJK Unified Ideographs (main block: 中, 国, etc.)\n\tif (codePoint >= 0x4e00 && codePoint <= 0x9fff) return true;\n\t// CJK Extension A (rare)\n\tif (codePoint >= 0x3400 && codePoint <= 0x4dbf) return true;\n\t// CJK compatibility ideographs\n\tif (codePoint >= 0xf900 && codePoint <= 0xfaff) return true;\n\treturn false;\n}\n\nfunction isEmojiOrWideSymbol(codePoint: number): boolean {\n\t// Main emoji blocks\n\tif (codePoint >= 0x1f000) return true;\n\t// Misc symbols, dingbats\n\tif (codePoint >= 0x2600 && codePoint <= 0x27bf) return true;\n\t// CJK fullwidth/special forms that occupy double width\n\tif (codePoint >= 0xff01 && codePoint <= 0xff60) return true;\n\tif (codePoint >= 0xffe0 && codePoint <= 0xffe6) return true;\n\treturn false;\n}\n\nfunction looksJsonLike(input: string): boolean {\n\tconst trimmed = input.trim();\n\treturn (trimmed.startsWith(\"{\") && trimmed.endsWith(\"}\")) || (trimmed.startsWith(\"[\") && trimmed.endsWith(\"]\"));\n}\n\nfunction looksCodeLike(input: string, punctuation: number, whitespace: number): boolean {\n\tif (/\\b(function|const|let|class|interface|import|export|return|async|await)\\b/.test(input)) {\n\t\treturn true;\n\t}\n\treturn punctuation > input.length / 8 && whitespace > input.length / 20;\n}\n"]}