@ramtinj95/opencode-tokenscope 1.4.0

package/dist/tokenscope-lib/analyzer.js

@@ -0,0 +1,321 @@
+// Analysis classes - ModelResolver, ContentCollector, TokenAnalysisEngine
+import { isToolPart as toolGuard, isReasoningPart as reasoningGuard, isTextPart as textGuard } from "./types";
+import { OPENAI_MODEL_MAP, TRANSFORMERS_MODEL_MAP, PROVIDER_DEFAULTS } from "./config";
+// Model Resolution
+export class ModelResolver {
+    resolveTokenModel(messages) {
+        for (const message of [...messages].reverse()) {
+            const modelID = this.canonicalize(message.info.modelID);
+            const providerID = this.canonicalize(message.info.providerID);
+            const openaiModel = this.resolveOpenAIModel(modelID, providerID);
+            if (openaiModel)
+                return openaiModel;
+            const transformerModel = this.resolveTransformersModel(modelID, providerID);
+            if (transformerModel)
+                return transformerModel;
+        }
+        return { name: "approx", spec: { kind: "approx" } };
+    }
+    resolveOpenAIModel(modelID, providerID) {
+        if (providerID === "openai" || providerID === "opencode" || providerID === "azure") {
+            const mapped = this.mapOpenAI(modelID);
+            return { name: modelID ?? mapped, spec: { kind: "tiktoken", model: mapped } };
+        }
+        if (modelID && OPENAI_MODEL_MAP[modelID]) {
+            return { name: modelID, spec: { kind: "tiktoken", model: OPENAI_MODEL_MAP[modelID] } };
+        }
+        return undefined;
+    }
+    resolveTransformersModel(modelID, providerID) {
+        if (modelID && TRANSFORMERS_MODEL_MAP[modelID]) {
+            return { name: modelID, spec: { kind: "transformers", hub: TRANSFORMERS_MODEL_MAP[modelID] } };
+        }
+        if (providerID && PROVIDER_DEFAULTS[providerID]) {
+            return { name: modelID ?? providerID, spec: PROVIDER_DEFAULTS[providerID] };
+        }
+        // Prefix-based fallbacks
+        if (modelID?.startsWith("claude")) {
+            return { name: modelID, spec: { kind: "transformers", hub: "Xenova/claude-tokenizer" } };
+        }
+        if (modelID?.startsWith("llama")) {
+            return {
+                name: modelID,
+                spec: { kind: "transformers", hub: TRANSFORMERS_MODEL_MAP[modelID] ?? "Xenova/Meta-Llama-3.1-Tokenizer" },
+            };
+        }
+        if (modelID?.startsWith("mistral")) {
+            return { name: modelID, spec: { kind: "transformers", hub: "Xenova/mistral-tokenizer-v3" } };
+        }
+        if (modelID?.startsWith("deepseek")) {
+            return { name: modelID, spec: { kind: "transformers", hub: "deepseek-ai/DeepSeek-V3" } };
+        }
+        return undefined;
+    }
+    mapOpenAI(modelID) {
+        if (!modelID)
+            return "cl100k_base";
+        return OPENAI_MODEL_MAP[modelID] ?? modelID;
+    }
+    canonicalize(value) {
+        return value?.split("/").pop()?.toLowerCase().trim();
+    }
+}
+// Content Collection
+export class ContentCollector {
+    collectSystemPrompts(messages) {
+        const prompts = new Map();
+        for (const message of messages) {
+            if (message.info.role === "system") {
+                const content = this.extractText(message.parts);
+                if (content)
+                    prompts.set(content, content);
+            }
+            if (message.info.role === "assistant") {
+                for (const prompt of message.info.system ?? []) {
+                    const trimmed = (prompt ?? "").trim();
+                    if (trimmed)
+                        prompts.set(trimmed, trimmed);
+                }
+            }
+        }
+        return Array.from(prompts.values()).map((content, index) => ({
+            label: this.identifySystemPrompt(content, index + 1),
+            content,
+        }));
+    }
+    collectMessageTexts(messages, role) {
+        const results = [];
+        let index = 0;
+        for (const message of messages) {
+            if (message.info.role !== role)
+                continue;
+            const content = this.extractText(message.parts);
+            if (!content)
+                continue;
+            index += 1;
+            results.push({ label: `${this.capitalize(role)}#${index}`, content });
+        }
+        return results;
+    }
+    collectToolOutputs(messages) {
+        const toolOutputs = new Map();
+        for (const message of messages) {
+            for (const part of message.parts) {
+                if (!toolGuard(part))
+                    continue;
+                if (part.state.status !== "completed")
+                    continue;
+                const output = (part.state.output ?? "").toString().trim();
+                if (!output)
+                    continue;
+                const toolName = part.tool || "tool";
+                const existing = toolOutputs.get(toolName) || "";
+                toolOutputs.set(toolName, existing + (existing ? "\n\n" : "") + output);
+            }
+        }
+        return Array.from(toolOutputs.entries()).map(([toolName, content]) => ({
+            label: toolName,
+            content,
+        }));
+    }
+    collectToolCallCounts(messages) {
+        const toolCounts = new Map();
+        for (const message of messages) {
+            for (const part of message.parts) {
+                if (!toolGuard(part))
+                    continue;
+                const toolName = part.tool || "tool";
+                if (toolName) {
+                    toolCounts.set(toolName, (toolCounts.get(toolName) || 0) + 1);
+                }
+            }
+        }
+        return toolCounts;
+    }
+    collectAllToolsCalled(messages) {
+        return Array.from(this.collectToolCallCounts(messages).keys()).sort();
+    }
+    collectReasoningTexts(messages) {
+        const results = [];
+        let index = 0;
+        for (const message of messages) {
+            for (const part of message.parts) {
+                if (!reasoningGuard(part))
+                    continue;
+                const text = (part.text ?? "").toString().trim();
+                if (!text)
+                    continue;
+                index += 1;
+                results.push({ label: `Reasoning#${index}`, content: text });
+            }
+        }
+        return results;
+    }
+    extractText(parts) {
+        return parts
+            .filter(textGuard)
+            .map((part) => part.text ?? "")
+            .map((text) => text.trim())
+            .filter(Boolean)
+            .join("\n\n");
+    }
+    identifySystemPrompt(content, index) {
+        const lower = content.toLowerCase();
+        if (lower.includes("opencode") && lower.includes("cli") && content.length > 500)
+            return "System#MainPrompt";
+        if (lower.includes("opencode") && lower.includes("cli") && content.length <= 500)
+            return "System#ShortPrompt";
+        if (lower.includes("agent") && lower.includes("mode"))
+            return "System#AgentMode";
+        if (lower.includes("permission") || lower.includes("allowed") || lower.includes("deny"))
+            return "System#Permissions";
+        if (lower.includes("tool") && (lower.includes("rule") || lower.includes("guideline")))
+            return "System#ToolRules";
+        if (lower.includes("format") || lower.includes("style") || lower.includes("concise"))
+            return "System#Formatting";
+        if (lower.includes("project") || lower.includes("repository") || lower.includes("codebase"))
+            return "System#ProjectContext";
+        if (lower.includes("session") || lower.includes("context") || lower.includes("memory"))
+            return "System#SessionMgmt";
+        if (content.includes("@") && (content.includes(".md") || content.includes(".txt")))
+            return "System#FileRefs";
+        if (content.includes("name:") && content.includes("description:"))
+            return "System#AgentDef";
+        if (lower.includes("code") && (lower.includes("convention") || lower.includes("standard")))
+            return "System#CodeGuidelines";
+        return `System#${index}`;
+    }
+    capitalize(value) {
+        if (!value)
+            return value;
+        return value[0].toUpperCase() + value.slice(1);
+    }
+}
+// Token Analysis Engine
+export class TokenAnalysisEngine {
+    tokenizerManager;
+    contentCollector;
+    constructor(tokenizerManager, contentCollector) {
+        this.tokenizerManager = tokenizerManager;
+        this.contentCollector = contentCollector;
+    }
+    async analyze(sessionID, messages, tokenModel, entryLimit) {
+        const systemPrompts = this.contentCollector.collectSystemPrompts(messages);
+        const userTexts = this.contentCollector.collectMessageTexts(messages, "user");
+        const assistantTexts = this.contentCollector.collectMessageTexts(messages, "assistant");
+        const toolOutputs = this.contentCollector.collectToolOutputs(messages);
+        const reasoningTraces = this.contentCollector.collectReasoningTexts(messages);
+        const allToolsCalled = this.contentCollector.collectAllToolsCalled(messages);
+        const toolCallCounts = this.contentCollector.collectToolCallCounts(messages);
+        const [system, user, assistant, tools, reasoning] = await Promise.all([
+            this.buildCategory("system", systemPrompts, tokenModel, entryLimit),
+            this.buildCategory("user", userTexts, tokenModel, entryLimit),
+            this.buildCategory("assistant", assistantTexts, tokenModel, entryLimit),
+            this.buildCategory("tools", toolOutputs, tokenModel, entryLimit),
+            this.buildCategory("reasoning", reasoningTraces, tokenModel, entryLimit),
+        ]);
+        const analysis = {
+            sessionID,
+            model: tokenModel,
+            categories: { system, user, assistant, tools, reasoning },
+            totalTokens: system.totalTokens + user.totalTokens + assistant.totalTokens + tools.totalTokens + reasoning.totalTokens,
+            inputTokens: 0,
+            outputTokens: 0,
+            reasoningTokens: 0,
+            cacheReadTokens: 0,
+            cacheWriteTokens: 0,
+            assistantMessageCount: 0,
+            mostRecentInput: 0,
+            mostRecentOutput: 0,
+            mostRecentReasoning: 0,
+            mostRecentCacheRead: 0,
+            mostRecentCacheWrite: 0,
+            sessionCost: 0,
+            mostRecentCost: 0,
+            allToolsCalled,
+            toolCallCounts,
+        };
+        this.applyTelemetryAdjustments(analysis, messages);
+        return analysis;
+    }
+    async buildCategory(label, sources, model, entryLimit) {
+        const entries = [];
+        for (const source of sources) {
+            const tokens = await this.tokenizerManager.countTokens(source.content, model);
+            if (tokens > 0) {
+                entries.push({ label: source.label, tokens });
+            }
+        }
+        entries.sort((a, b) => b.tokens - a.tokens);
+        const limited = entries.slice(0, entryLimit);
+        const totalTokens = entries.reduce((sum, entry) => sum + entry.tokens, 0);
+        return { label, totalTokens, entries: limited, allEntries: entries };
+    }
+    applyTelemetryAdjustments(analysis, messages) {
+        const assistants = messages
+            .filter((m) => m.info.role === "assistant" && (m.info?.tokens || m.info?.cost !== undefined))
+            .map((m) => ({ msg: m, tokens: m.info.tokens, cost: m.info.cost ?? 0 }));
+        let totalInput = 0, totalOutput = 0, totalReasoning = 0;
+        let totalCacheRead = 0, totalCacheWrite = 0, totalCost = 0;
+        for (const { tokens, cost } of assistants) {
+            if (tokens) {
+                totalInput += Number(tokens.input) || 0;
+                totalOutput += Number(tokens.output) || 0;
+                totalReasoning += Number(tokens.reasoning) || 0;
+                totalCacheRead += Number(tokens.cache?.read) || 0;
+                totalCacheWrite += Number(tokens.cache?.write) || 0;
+            }
+            totalCost += Number(cost) || 0;
+        }
+        const mostRecentWithUsage = [...assistants]
+            .reverse()
+            .find(({ tokens }) => tokens &&
+            (Number(tokens.input) || 0) +
+                (Number(tokens.output) || 0) +
+                (Number(tokens.reasoning) || 0) +
+                (Number(tokens.cache?.read) || 0) +
+                (Number(tokens.cache?.write) || 0) >
+                0) ?? assistants[assistants.length - 1];
+        let mostRecentInput = 0, mostRecentOutput = 0, mostRecentReasoning = 0;
+        let mostRecentCacheRead = 0, mostRecentCacheWrite = 0, mostRecentCost = 0;
+        if (mostRecentWithUsage) {
+            const t = mostRecentWithUsage.tokens;
+            if (t) {
+                mostRecentInput = Number(t.input) || 0;
+                mostRecentOutput = Number(t.output) || 0;
+                mostRecentReasoning = Number(t.reasoning) || 0;
+                mostRecentCacheRead = Number(t.cache?.read) || 0;
+                mostRecentCacheWrite = Number(t.cache?.write) || 0;
+            }
+            mostRecentCost = Number(mostRecentWithUsage.cost) || 0;
+        }
+        analysis.inputTokens = totalInput;
+        analysis.outputTokens = totalOutput;
+        analysis.reasoningTokens = totalReasoning;
+        analysis.cacheReadTokens = totalCacheRead;
+        analysis.cacheWriteTokens = totalCacheWrite;
+        analysis.assistantMessageCount = assistants.length;
+        analysis.sessionCost = totalCost;
+        analysis.mostRecentCost = mostRecentCost;
+        analysis.mostRecentInput = mostRecentInput;
+        analysis.mostRecentOutput = mostRecentOutput;
+        analysis.mostRecentReasoning = mostRecentReasoning;
+        analysis.mostRecentCacheRead = mostRecentCacheRead;
+        analysis.mostRecentCacheWrite = mostRecentCacheWrite;
+        const recentApiInputTotal = mostRecentInput + mostRecentCacheRead;
+        const localUserAndTools = analysis.categories.user.totalTokens + analysis.categories.tools.totalTokens;
+        const inferredSystemTokens = Math.max(0, recentApiInputTotal - localUserAndTools);
+        if (inferredSystemTokens > 0 && analysis.categories.system.totalTokens === 0) {
+            analysis.categories.system.totalTokens = inferredSystemTokens;
+            analysis.categories.system.entries = [{ label: "System (inferred from API)", tokens: inferredSystemTokens }];
+            analysis.categories.system.allEntries = analysis.categories.system.entries;
+        }
+        analysis.totalTokens =
+            analysis.categories.system.totalTokens +
+                analysis.categories.user.totalTokens +
+                analysis.categories.assistant.totalTokens +
+                analysis.categories.tools.totalTokens +
+                analysis.categories.reasoning.totalTokens;
+    }
+}
+//# sourceMappingURL=analyzer.js.map

package/dist/tokenscope-lib/analyzer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"analyzer.js","sourceRoot":"","sources":["../../tokenscope-lib/analyzer.ts"],"names":[],"mappings":"AAAA,0EAA0E;AAc1E,OAAO,EAAE,UAAU,IAAI,SAAS,EAAE,eAAe,IAAI,cAAc,EAAE,UAAU,IAAI,SAAS,EAAE,MAAM,SAAS,CAAA;AAC7G,OAAO,EAAE,gBAAgB,EAAE,sBAAsB,EAAE,iBAAiB,EAAE,MAAM,UAAU,CAAA;AAGtF,mBAAmB;AAEnB,MAAM,OAAO,aAAa;IACxB,iBAAiB,CAAC,QAA0B;QAC1C,KAAK,MAAM,OAAO,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;YAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACvD,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YAE7D,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAA;YAChE,IAAI,WAAW;gBAAE,OAAO,WAAW,CAAA;YAEnC,MAAM,gBAAgB,GAAG,IAAI,CAAC,wBAAwB,CAAC,OAAO,EAAE,UAAU,CAAC,CAAA;YAC3E,IAAI,gBAAgB;gBAAE,OAAO,gBAAgB,CAAA;QAC/C,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAA;IACrD,CAAC;IAEO,kBAAkB,CAAC,OAAgB,EAAE,UAAmB;QAC9D,IAAI,UAAU,KAAK,QAAQ,IAAI,UAAU,KAAK,UAAU,IAAI,UAAU,KAAK,OAAO,EAAE,CAAC;YACnF,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;YACtC,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,MAAM,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,CAAA;QAC/E,CAAC;QAED,IAAI,OAAO,IAAI,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;YACzC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,gBAAgB,CAAC,OAAO,CAAC,EAAE,EAAE,CAAA;QACxF,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAEO,wBAAwB,CAAC,OAAgB,EAAE,UAAmB;QACpE,IAAI,OAAO,IAAI,sBAAsB,CAAC,OAAO,CAAC,EAAE,CAAC;YAC/C,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,sBAAsB,CAAC,OAAO,CAAC,EAAE,EAAE,CAAA;QAChG,CAAC;QAED,IAAI,UAAU,IAAI,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAC;YAChD,OAAO,EAAE,IAAI,EAAE,OAAO,IAAI,UAAU,EAAE,IAAI,EAAE,iBAAiB,CAAC,UAAU,CAAC,EAAE,CAAA;QAC7E,CAAC;QAED,yBAAyB;QACzB,IAAI,OAAO,EAAE,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,yBAAyB,EAAE,EAAE,CAAA;QAC1F,CAAC;QAED,IAAI,OAAO,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACjC,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,IAAI,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,sBAAsB,CAAC,OAAO,CAAC,IAAI,iCAAiC,EAAE;aAC1G,CAAA;QACH,CAAC;QAED,IAAI,OAAO,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;YACnC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,6BAA6B,EAAE,EAAE,CAAA;QAC9F,CAAC;QAED,IAAI,OAAO,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YACpC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,yBAAyB,EAAE,EAAE,CAAA;QAC1F,CAAC;QAED,OAAO,SAAS,CAAA;IAClB,CAAC;IAEO,SAAS,CAAC,OAAgB;QAChC,IAAI,CAAC,OAAO;YAAE,OAAO,aAAa,CAAA;QAClC,OAAO,gBAAgB,CAAC,OAAO,CAAC,IAAI,OAAO,CAAA;IAC7C,CAAC;IAEO,YAAY,CAAC,KAAc;QACjC,OAAO,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC,IAAI,EAAE,CAAA;IACtD,CAAC;CACF;AAED,qBAAqB;AAErB,MAAM,OAAO,gBAAgB;IAC3B,oBAAoB,CAAC,QAA0B;QAC7C,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAA;QAEzC,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACnC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC/C,IAAI,OAAO;oBAAE,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC5C,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACtC,KAAK,MAAM,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;oBAC/C,MAAM,OAAO,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;oBACrC,IAAI,OAAO;wBAAE,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;gBAC5C,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAC3D,KAAK,EAAE,IAAI,CAAC,oBAAoB,CAAC,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC;YACpD,OAAO;SACR,CAAC,CAAC,CAAA;IACL,CAAC;IAED,mBAAmB,CAAC,QAA0B,EAAE,IAA0B;QACxE,MAAM,OAAO,GAA0B,EAAE,CAAA;QACzC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI;gBAAE,SAAQ;YACxC,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;YAC/C,IAAI,CAAC,OAAO;gBAAE,SAAQ;YAEtB,KAAK,IAAI,CAAC,CAAA;YACV,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,EAAE,OAAO,EAAE,CAAC,CAAA;QACvE,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAED,kBAAkB,CAAC,QAA0B;QAC3C,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAA;QAE7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAAE,SAAQ;gBAE9B,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,WAAW;oBAAE,SAAQ;gBAE/C,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAA;gBAC1D,IAAI,CAAC,MAAM;oBAAE,SAAQ;gBAErB,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,CAAA;gBACpC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;gBAChD,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,QAAQ,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,CAAA;YACzE,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YACrE,KAAK,EAAE,QAAQ;YACf,OAAO;SACR,CAAC,CAAC,CAAA;IACL,CAAC;IAED,qBAAqB,CAAC,QAA0B;QAC9C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAA;QAE5C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;oBAAE,SAAQ;gBAE9B,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,IAAI,MAAM,CAAA;gBACpC,IAAI,QAAQ,EAAE,CAAC;oBACb,UAAU,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;gBAC/D,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,UAAU,CAAA;IACnB,CAAC;IAED,qBAAqB,CAAC,QAA0B;QAC9C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IACvE,CAAC;IAED,qBAAqB,CAAC,QAA0B;QAC9C,MAAM,OAAO,GAA0B,EAAE,CAAA;QACzC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;gBACjC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC;oBAAE,SAAQ;gBAEnC,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,CAAA;gBAChD,IAAI,CAAC,IAAI;oBAAE,SAAQ;gBAEnB,KAAK,IAAI,CAAC,CAAA;gBACV,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,aAAa,KAAK,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAA;YAC9D,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAA;IAChB,CAAC;IAEO,WAAW,CAAC,KAA2B;QAC7C,OAAO,KAAK;aACT,MAAM,CAAC,SAAS,CAAC;aACjB,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;aAC9B,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aAC1B,MAAM,CAAC,OAAO,CAAC;aACf,IAAI,CAAC,MAAM,CAAC,CAAA;IACjB,CAAC;IAEO,oBAAoB,CAAC,OAAe,EAAE,KAAa;QACzD,MAAM,KAAK,GAAG,OAAO,CAAC,WAAW,EAAE,CAAA;QAEnC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,mBAAmB,CAAA;QAC3G,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,MAAM,IAAI,GAAG;YAAE,OAAO,oBAAoB,CAAA;QAC7G,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,kBAAkB,CAAA;QAChF,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,oBAAoB,CAAA;QACpH,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAAE,OAAO,kBAAkB,CAAA;QAChH,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC;YAAE,OAAO,mBAAmB,CAAA;QAChH,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;YACzF,OAAO,uBAAuB,CAAA;QAChC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO,oBAAoB,CAAA;QACnH,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;YAAE,OAAO,iBAAiB,CAAA;QAC5G,IAAI,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC;YAAE,OAAO,iBAAiB,CAAA;QAC3F,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACxF,OAAO,uBAAuB,CAAA;QAEhC,OAAO,UAAU,KAAK,EAAE,CAAA;IAC1B,CAAC;IAEO,UAAU,CAAC,KAAa;QAC9B,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAA;QACxB,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAChD,CAAC;CACF;AAED,wBAAwB;AAExB,MAAM,OAAO,mBAAmB;IAEpB;IACA;IAFV,YACU,gBAAkC,EAClC,gBAAkC;QADlC,qBAAgB,GAAhB,gBAAgB,CAAkB;QAClC,qBAAgB,GAAhB,gBAAgB,CAAkB;IACzC,CAAC;IAEJ,KAAK,CAAC,OAAO,CACX,SAAiB,EACjB,QAA0B,EAC1B,UAAsB,EACtB,UAAkB;QAElB,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAA;QAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QAC7E,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,mBAAmB,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;QACvF,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAA;QACtE,MAAM,eAAe,GAAG,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAA;QAC7E,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAA;QAC5E,MAAM,cAAc,GAAG,IAAI,CAAC,gBAAgB,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAA;QAE5E,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YACpE,IAAI,CAAC,aAAa,CAAC,QAAQ,EAAE,aAAa,EAAE,UAAU,EAAE,UAAU,CAAC;YACnE,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,CAAC;YAC7D,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,cAAc,EAAE,UAAU,EAAE,UAAU,CAAC;YACvE,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,UAAU,CAAC;YAChE,IAAI,CAAC,aAAa,CAAC,WAAW,EAAE,eAAe,EAAE,UAAU,EAAE,UAAU,CAAC;SACzE,CAAC,CAAA;QAEF,MAAM,QAAQ,GAAkB;YAC9B,SAAS;YACT,KAAK,EAAE,UAAU;YACjB,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE;YACzD,WAAW,EACT,MAAM,CAAC,WAAW,GAAG,IAAI,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW,GAAG,KAAK,CAAC,WAAW,GAAG,SAAS,CAAC,WAAW;YAC3G,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,eAAe,EAAE,CAAC;YAClB,eAAe,EAAE,CAAC;YAClB,gBAAgB,EAAE,CAAC;YACnB,qBAAqB,EAAE,CAAC;YACxB,eAAe,EAAE,CAAC;YAClB,gBAAgB,EAAE,CAAC;YACnB,mBAAmB,EAAE,CAAC;YACtB,mBAAmB,EAAE,CAAC;YACtB,oBAAoB,EAAE,CAAC;YACvB,WAAW,EAAE,CAAC;YACd,cAAc,EAAE,CAAC;YACjB,cAAc;YACd,cAAc;SACf,CAAA;QAED,IAAI,CAAC,yBAAyB,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAElD,OAAO,QAAQ,CAAA;IACjB,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,KAAa,EACb,OAA8B,EAC9B,KAAiB,EACjB,UAAkB;QAElB,MAAM,OAAO,GAAoB,EAAE,CAAA;QAEnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;YAC7E,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,OAAO,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,CAAC,CAAA;YAC/C,CAAC;QACH,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;QAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAA;QAC5C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAEzE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,CAAA;IACtE,CAAC;IAEO,yBAAyB,CAAC,QAAuB,EAAE,QAA0B;QACnF,MAAM,UAAU,GAAG,QAAQ;aACxB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,KAAK,WAAW,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,CAAC,IAAI,EAAE,IAAI,KAAK,SAAS,CAAC,CAAC;aAC5F,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;QAE1E,IAAI,UAAU,GAAG,CAAC,EAChB,WAAW,GAAG,CAAC,EACf,cAAc,GAAG,CAAC,CAAA;QACpB,IAAI,cAAc,GAAG,CAAC,EACpB,eAAe,GAAG,CAAC,EACnB,SAAS,GAAG,CAAC,CAAA;QAEf,KAAK,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,UAAU,EAAE,CAAC;YAC1C,IAAI,MAAM,EAAE,CAAC;gBACX,UAAU,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;gBACvC,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBACzC,cAAc,IAAI,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;gBAC/C,cAAc,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;gBACjD,eAAe,IAAI,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;YACrD,CAAC;YACD,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAChC,CAAC;QAED,MAAM,mBAAmB,GAAG,CAAC,GAAG,UAAU,CAAC;aACxC,OAAO,EAAE;aACT,IAAI,CACH,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,CACb,MAAM;YACN,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACzB,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBAC5B,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;gBAC/B,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;gBAClC,CAAC,CACN,IAAI,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAExC,IAAI,eAAe,GAAG,CAAC,EACrB,gBAAgB,GAAG,CAAC,EACpB,mBAAmB,GAAG,CAAC,CAAA;QACzB,IAAI,mBAAmB,GAAG,CAAC,EACzB,oBAAoB,GAAG,CAAC,EACxB,cAAc,GAAG,CAAC,CAAA;QAEpB,IAAI,mBAAmB,EAAE,CAAC;YACxB,MAAM,CAAC,GAAG,mBAAmB,CAAC,MAAM,CAAA;YACpC,IAAI,CAAC,EAAE,CAAC;gBACN,eAAe,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;gBACtC,gBAAgB,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;gBACxC,mBAAmB,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;gBAC9C,mBAAmB,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;gBAChD,oBAAoB,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;YACpD,CAAC;YACD,cAAc,GAAG,MAAM,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxD,CAAC;QAED,QAAQ,CAAC,WAAW,GAAG,UAAU,CAAA;QACjC,QAAQ,CAAC,YAAY,GAAG,WAAW,CAAA;QACnC,QAAQ,CAAC,eAAe,GAAG,cAAc,CAAA;QACzC,QAAQ,CAAC,eAAe,GAAG,cAAc,CAAA;QACzC,QAAQ,CAAC,gBAAgB,GAAG,eAAe,CAAA;QAC3C,QAAQ,CAAC,qBAAqB,GAAG,UAAU,CAAC,MAAM,CAAA;QAClD,QAAQ,CAAC,WAAW,GAAG,SAAS,CAAA;QAChC,QAAQ,CAAC,cAAc,GAAG,cAAc,CAAA;QACxC,QAAQ,CAAC,eAAe,GAAG,eAAe,CAAA;QAC1C,QAAQ,CAAC,gBAAgB,GAAG,gBAAgB,CAAA;QAC5C,QAAQ,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAClD,QAAQ,CAAC,mBAAmB,GAAG,mBAAmB,CAAA;QAClD,QAAQ,CAAC,oBAAoB,GAAG,oBAAoB,CAAA;QAEpD,MAAM,mBAAmB,GAAG,eAAe,GAAG,mBAAmB,CAAA;QACjE,MAAM,iBAAiB,GAAG,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,WAAW,CAAA;QACtG,MAAM,oBAAoB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,mBAAmB,GAAG,iBAAiB,CAAC,CAAA;QAEjF,IAAI,oBAAoB,GAAG,CAAC,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,KAAK,CAAC,EAAE,CAAC;YAC7E,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW,GAAG,oBAAoB,CAAA;YAC7D,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,EAAE,KAAK,EAAE,4BAA4B,EAAE,MAAM,EAAE,oBAAoB,EAAE,CAAC,CAAA;YAC5G,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,OAAO,CAAA;QAC5E,CAAC;QAED,QAAQ,CAAC,WAAW;YAClB,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,WAAW;gBACtC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,WAAW;gBACpC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,WAAW;gBACzC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,WAAW;gBACrC,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC,WAAW,CAAA;IAC7C,CAAC;CACF"}

package/dist/tokenscope-lib/config.d.ts

@@ -0,0 +1,10 @@
+import type { TokenizerSpec, ModelPricing, TokenscopeConfig } from "./types";
+export declare const DEFAULT_ENTRY_LIMIT = 3;
+export declare const VENDOR_ROOT: string;
+export declare function loadModelPricing(): Promise<Record<string, ModelPricing>>;
+export declare const DEFAULT_TOKENSCOPE_CONFIG: TokenscopeConfig;
+export declare function loadTokenscopeConfig(): Promise<TokenscopeConfig>;
+export declare const OPENAI_MODEL_MAP: Record<string, string>;
+export declare const TRANSFORMERS_MODEL_MAP: Record<string, string>;
+export declare const PROVIDER_DEFAULTS: Record<string, TokenizerSpec>;
+//# sourceMappingURL=config.d.ts.map

package/dist/tokenscope-lib/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../tokenscope-lib/config.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAE5E,eAAO,MAAM,mBAAmB,IAAI,CAAA;AACpC,eAAO,MAAM,WAAW,QAA0F,CAAA;AAKlH,wBAAsB,gBAAgB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC,CAY9E;AAID,eAAO,MAAM,yBAAyB,EAAE,gBAKvC,CAAA;AAID,wBAAsB,oBAAoB,IAAI,OAAO,CAAC,gBAAgB,CAAC,CAatE;AAGD,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAiBnD,CAAA;AAGD,eAAO,MAAM,sBAAsB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CA2BzD,CAAA;AAGD,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAM3D,CAAA"}

package/dist/tokenscope-lib/config.js

@@ -0,0 +1,102 @@
+// Configuration constants, model maps, and pricing loader
+import path from "path";
+import fs from "fs/promises";
+import { fileURLToPath } from "url";
+export const DEFAULT_ENTRY_LIMIT = 3;
+export const VENDOR_ROOT = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "vendor", "node_modules");
+// Pricing cache
+let PRICING_CACHE = null;
+export async function loadModelPricing() {
+    if (PRICING_CACHE)
+        return PRICING_CACHE;
+    try {
+        const modelsPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "models.json");
+        const data = await fs.readFile(modelsPath, "utf8");
+        PRICING_CACHE = JSON.parse(data);
+        return PRICING_CACHE;
+    }
+    catch {
+        PRICING_CACHE = { default: { input: 1, output: 3, cacheWrite: 0, cacheRead: 0 } };
+        return PRICING_CACHE;
+    }
+}
+// Tokenscope config defaults and loader
+export const DEFAULT_TOKENSCOPE_CONFIG = {
+    enableContextBreakdown: true,
+    enableToolSchemaEstimation: true,
+    enableCacheEfficiency: true,
+    enableSubagentAnalysis: true,
+};
+let TOKENSCOPE_CONFIG_CACHE = null;
+export async function loadTokenscopeConfig() {
+    if (TOKENSCOPE_CONFIG_CACHE)
+        return TOKENSCOPE_CONFIG_CACHE;
+    try {
+        const configPath = path.join(path.dirname(fileURLToPath(import.meta.url)), "..", "tokenscope-config.json");
+        const data = await fs.readFile(configPath, "utf8");
+        const config = { ...DEFAULT_TOKENSCOPE_CONFIG, ...JSON.parse(data) };
+        TOKENSCOPE_CONFIG_CACHE = config;
+        return config;
+    }
+    catch {
+        TOKENSCOPE_CONFIG_CACHE = DEFAULT_TOKENSCOPE_CONFIG;
+        return DEFAULT_TOKENSCOPE_CONFIG;
+    }
+}
+// OpenAI model mapping for tiktoken
+export const OPENAI_MODEL_MAP = {
+    "gpt-5": "gpt-4o",
+    "o4-mini": "gpt-4o",
+    "o3": "gpt-4o",
+    "o3-mini": "gpt-4o",
+    "o1": "gpt-4o",
+    "o1-pro": "gpt-4o",
+    "gpt-4.1": "gpt-4o",
+    "gpt-4.1-mini": "gpt-4o",
+    "gpt-4o": "gpt-4o",
+    "gpt-4o-mini": "gpt-4o-mini",
+    "gpt-4-turbo": "gpt-4",
+    "gpt-4": "gpt-4",
+    "gpt-3.5-turbo": "gpt-3.5-turbo",
+    "text-embedding-3-large": "text-embedding-3-large",
+    "text-embedding-3-small": "text-embedding-3-small",
+    "text-embedding-ada-002": "text-embedding-ada-002",
+};
+// Transformers model mapping for HuggingFace tokenizers
+export const TRANSFORMERS_MODEL_MAP = {
+    "claude-opus-4": "Xenova/claude-tokenizer",
+    "claude-sonnet-4": "Xenova/claude-tokenizer",
+    "claude-3.7-sonnet": "Xenova/claude-tokenizer",
+    "claude-3.5-sonnet": "Xenova/claude-tokenizer",
+    "claude-3.5-haiku": "Xenova/claude-tokenizer",
+    "claude-3-opus": "Xenova/claude-tokenizer",
+    "claude-3-sonnet": "Xenova/claude-tokenizer",
+    "claude-3-haiku": "Xenova/claude-tokenizer",
+    "claude-2.1": "Xenova/claude-tokenizer",
+    "claude-2.0": "Xenova/claude-tokenizer",
+    "claude-instant-1.2": "Xenova/claude-tokenizer",
+    "llama-4": "Xenova/llama4-tokenizer",
+    "llama-3.3": "unsloth/Llama-3.3-70B-Instruct",
+    "llama-3.2": "Xenova/Llama-3.2-Tokenizer",
+    "llama-3.1": "Xenova/Meta-Llama-3.1-Tokenizer",
+    "llama-3": "Xenova/llama3-tokenizer-new",
+    "llama-2": "Xenova/llama2-tokenizer",
+    "code-llama": "Xenova/llama-code-tokenizer",
+    "deepseek-r1": "deepseek-ai/DeepSeek-R1",
+    "deepseek-v3": "deepseek-ai/DeepSeek-V3",
+    "deepseek-v2": "deepseek-ai/DeepSeek-V2",
+    "mistral-large": "Xenova/mistral-tokenizer-v3",
+    "mistral-small": "Xenova/mistral-tokenizer-v3",
+    "mistral-nemo": "Xenova/Mistral-Nemo-Instruct-Tokenizer",
+    "devstral-small": "Xenova/Mistral-Nemo-Instruct-Tokenizer",
+    "codestral": "Xenova/mistral-tokenizer-v3",
+};
+// Provider default tokenizers
+export const PROVIDER_DEFAULTS = {
+    anthropic: { kind: "transformers", hub: "Xenova/claude-tokenizer" },
+    meta: { kind: "transformers", hub: "Xenova/Meta-Llama-3.1-Tokenizer" },
+    mistral: { kind: "transformers", hub: "Xenova/mistral-tokenizer-v3" },
+    deepseek: { kind: "transformers", hub: "deepseek-ai/DeepSeek-V3" },
+    google: { kind: "transformers", hub: "google/gemma-2-9b-it" },
+};
+//# sourceMappingURL=config.js.map

package/dist/tokenscope-lib/config.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../../tokenscope-lib/config.ts"],"names":[],"mappings":"AAAA,0DAA0D;AAE1D,OAAO,IAAI,MAAM,MAAM,CAAA;AACvB,OAAO,EAAE,MAAM,aAAa,CAAA;AAC5B,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAA;AAGnC,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,CAAA;AACpC,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAA;AAElH,gBAAgB;AAChB,IAAI,aAAa,GAAwC,IAAI,CAAA;AAE7D,MAAM,CAAC,KAAK,UAAU,gBAAgB;IACpC,IAAI,aAAa;QAAE,OAAO,aAAa,CAAA;IAEvC,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,aAAa,CAAC,CAAA;QAC/F,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QAClD,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;QAChC,OAAO,aAAc,CAAA;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,aAAa,GAAG,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,CAAA;QACjF,OAAO,aAAa,CAAA;IACtB,CAAC;AACH,CAAC;AAED,wCAAwC;AAExC,MAAM,CAAC,MAAM,yBAAyB,GAAqB;IACzD,sBAAsB,EAAE,IAAI;IAC5B,0BAA0B,EAAE,IAAI;IAChC,qBAAqB,EAAE,IAAI;IAC3B,sBAAsB,EAAE,IAAI;CAC7B,CAAA;AAED,IAAI,uBAAuB,GAA4B,IAAI,CAAA;AAE3D,MAAM,CAAC,KAAK,UAAU,oBAAoB;IACxC,IAAI,uBAAuB;QAAE,OAAO,uBAAuB,CAAA;IAE3D,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,wBAAwB,CAAC,CAAA;QAC1G,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QAClD,MAAM,MAAM,GAAG,EAAE,GAAG,yBAAyB,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAA;QACpE,uBAAuB,GAAG,MAAM,CAAA;QAChC,OAAO,MAAM,CAAA;IACf,CAAC;IAAC,MAAM,CAAC;QACP,uBAAuB,GAAG,yBAAyB,CAAA;QACnD,OAAO,yBAAyB,CAAA;IAClC,CAAC;AACH,CAAC;AAED,oCAAoC;AACpC,MAAM,CAAC,MAAM,gBAAgB,GAA2B;IACtD,OAAO,EAAE,QAAQ;IACjB,SAAS,EAAE,QAAQ;IACnB,IAAI,EAAE,QAAQ;IACd,SAAS,EAAE,QAAQ;IACnB,IAAI,EAAE,QAAQ;IACd,QAAQ,EAAE,QAAQ;IAClB,SAAS,EAAE,QAAQ;IACnB,cAAc,EAAE,QAAQ;IACxB,QAAQ,EAAE,QAAQ;IAClB,aAAa,EAAE,aAAa;IAC5B,aAAa,EAAE,OAAO;IACtB,OAAO,EAAE,OAAO;IAChB,eAAe,EAAE,eAAe;IAChC,wBAAwB,EAAE,wBAAwB;IAClD,wBAAwB,EAAE,wBAAwB;IAClD,wBAAwB,EAAE,wBAAwB;CACnD,CAAA;AAED,wDAAwD;AACxD,MAAM,CAAC,MAAM,sBAAsB,GAA2B;IAC5D,eAAe,EAAE,yBAAyB;IAC1C,iBAAiB,EAAE,yBAAyB;IAC5C,mBAAmB,EAAE,yBAAyB;IAC9C,mBAAmB,EAAE,yBAAyB;IAC9C,kBAAkB,EAAE,yBAAyB;IAC7C,eAAe,EAAE,yBAAyB;IAC1C,iBAAiB,EAAE,yBAAyB;IAC5C,gBAAgB,EAAE,yBAAyB;IAC3C,YAAY,EAAE,yBAAyB;IACvC,YAAY,EAAE,yBAAyB;IACvC,oBAAoB,EAAE,yBAAyB;IAC/C,SAAS,EAAE,yBAAyB;IACpC,WAAW,EAAE,gCAAgC;IAC7C,WAAW,EAAE,4BAA4B;IACzC,WAAW,EAAE,iCAAiC;IAC9C,SAAS,EAAE,6BAA6B;IACxC,SAAS,EAAE,yBAAyB;IACpC,YAAY,EAAE,6BAA6B;IAC3C,aAAa,EAAE,yBAAyB;IACxC,aAAa,EAAE,yBAAyB;IACxC,aAAa,EAAE,yBAAyB;IACxC,eAAe,EAAE,6BAA6B;IAC9C,eAAe,EAAE,6BAA6B;IAC9C,cAAc,EAAE,wCAAwC;IACxD,gBAAgB,EAAE,wCAAwC;IAC1D,WAAW,EAAE,6BAA6B;CAC3C,CAAA;AAED,8BAA8B;AAC9B,MAAM,CAAC,MAAM,iBAAiB,GAAkC;IAC9D,SAAS,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,yBAAyB,EAAE;IACnE,IAAI,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,iCAAiC,EAAE;IACtE,OAAO,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,6BAA6B,EAAE;IACrE,QAAQ,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,yBAAyB,EAAE;IAClE,MAAM,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,EAAE,sBAAsB,EAAE;CAC9D,CAAA"}

package/dist/tokenscope-lib/context.d.ts

@@ -0,0 +1,81 @@
+import type { TokenscopeConfig, ModelPricing, TokenModel, ContextAnalysisResult } from "./types";
+import { TokenizerManager } from "./tokenizer";
+export declare class ContextAnalyzer {
+    private tokenizerManager;
+    constructor(tokenizerManager: TokenizerManager);
+    /**
+     * Main entry point - analyzes a session using opencode export
+     */
+    analyze(sessionID: string, tokenModel: TokenModel, pricing: ModelPricing, config: TokenscopeConfig): Promise<ContextAnalysisResult>;
+    /**
+     * Execute opencode export and parse the JSON output
+     */
+    private runExport;
+    /**
+     * Analyze context breakdown from cache_write tokens.
+     *
+     * Note: OpenCode's `opencode export` command doesn't include system prompt content
+     * in the output, so we estimate the breakdown from the first cache_write token count
+     * which represents the total cached context size.
+     *
+     * If system prompts become available in future versions, we can enhance this
+     * to tokenize the actual content for more accurate breakdowns.
+     */
+    private analyzeContextBreakdown;
+    /**
+     * Analyze actual system prompt content (for when opencode export includes it)
+     */
+    private analyzeSystemPromptContent;
+    /**
+     * Extract system prompts from exported session
+     */
+    private extractSystemPrompts;
+    /**
+     * Estimate context breakdown from cache token counts when system prompts aren't available.
+     *
+     * Based on typical OpenCode system prompt structure:
+     * - Base System Prompt: ~1,500-2,000 tokens
+     * - Tool Definitions: ~350 tokens per tool (typically 12-15 tools = ~4,500-5,500)
+     * - Environment Context: ~100-200 tokens
+     * - Project Tree: ~300-800 tokens (varies by project)
+     * - Custom Instructions: ~100-500 tokens
+     *
+     * We use the first cache_write value as an estimate of total cached context.
+     */
+    private estimateContextFromCacheTokens;
+    /**
+     * Check if content looks like a file tree
+     */
+    private looksLikeFileTree;
+    /**
+     * Estimate tool schema tokens from tool calls in the session
+     */
+    private estimateToolSchemas;
+    /**
+     * Extract enabled tools from user messages
+     */
+    private extractEnabledTools;
+    /**
+     * Extract tool call argument data for inference
+     */
+    private extractToolCallData;
+    /**
+     * Infer argument types from values
+     */
+    private inferArgTypes;
+    /**
+     * Estimate tokens for a tool schema based on call data
+     *
+     * Formula from plan:
+     * base_tokens = 200  (description + schema overhead)
+     * per_simple_arg = 30
+     * per_complex_arg = 60  (arrays, objects)
+     * description_bonus = 80 (simple) or 120 (complex)
+     */
+    private estimateToolTokens;
+    /**
+     * Calculate cache efficiency metrics
+     */
+    private calculateCacheEfficiency;
+}
+//# sourceMappingURL=context.d.ts.map

package/dist/tokenscope-lib/context.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"context.d.ts","sourceRoot":"","sources":["../../tokenscope-lib/context.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAQV,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,qBAAqB,EACtB,MAAM,SAAS,CAAA;AAChB,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAA;AAE9C,qBAAa,eAAe;IAC1B,OAAO,CAAC,gBAAgB,CAAkB;gBAE9B,gBAAgB,EAAE,gBAAgB;IAI9C;;OAEG;IACG,OAAO,CACX,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,UAAU,EACtB,OAAO,EAAE,YAAY,EACrB,MAAM,EAAE,gBAAgB,GACvB,OAAO,CAAC,qBAAqB,CAAC;IAyBjC;;OAEG;YACW,SAAS;IAsBvB;;;;;;;;;OASG;YACW,uBAAuB;IAyBrC;;OAEG;YACW,0BAA0B;IAqIxC;;OAEG;IACH,OAAO,CAAC,oBAAoB;IAiB5B;;;;;;;;;;;OAWG;IACH,OAAO,CAAC,8BAA8B;IAgDtC;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAiBzB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAwB3B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA0B3B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAoB3B;;OAEG;IACH,OAAO,CAAC,aAAa;IAoBrB;;;;;;;;OAQG;IACH,OAAO,CAAC,kBAAkB;IA6C1B;;OAEG;IACH,OAAO,CAAC,wBAAwB;CA6CjC"}