@townco/agent 0.1.122 → 0.1.123

package/dist/utils/context-size-calculator.d.ts

@@ -24,4 +24,4 @@ export interface ContextSize {
  * @param mcpOverheadTokens - Pre-calculated MCP tool overhead (MCP tool definitions)
  * @param modelContextWindow - Model's max context window size
  */
-export declare function calculateContextSize(messages: SessionMessage[], systemPrompt?: string, toolOverheadTokens?: number, mcpOverheadTokens?: number, modelContextWindow?: number): ContextSize;
+export declare function calculateContextSize(messages: SessionMessage[], systemPrompt?: string, toolOverheadTokens?: number, mcpOverheadTokens?: number, modelContextWindow?: number): Promise<ContextSize>;

package/dist/utils/context-size-calculator.js

@@ -6,10 +6,10 @@ import { countTokens, countToolResultTokens } from "./token-counter.js";
 /**
  * Extract and count tokens from a content block based on its type
  */
-function countContentBlock(block) {
+async function countContentBlock(block) {
     if (block.type === "text") {
         return {
-            textTokens: countTokens(block.text),
+            textTokens: await countTokens(block.text),
             toolInputTokens: 0,
             toolResultTokens: 0,
         };
@@ -18,10 +18,10 @@ function countContentBlock(block) {
         return {
             textTokens: 0,
             toolInputTokens: block.rawInput
-                ? countToolResultTokens(block.rawInput)
+                ? await countToolResultTokens(block.rawInput)
                 : 0,
             toolResultTokens: block.rawOutput
-                ? countToolResultTokens(block.rawOutput)
+                ? await countToolResultTokens(block.rawOutput)
                 : 0,
         };
     }
@@ -37,8 +37,8 @@ function countContentBlock(block) {
  * @param mcpOverheadTokens - Pre-calculated MCP tool overhead (MCP tool definitions)
  * @param modelContextWindow - Model's max context window size
  */
-export function calculateContextSize(messages, systemPrompt, toolOverheadTokens, mcpOverheadTokens, modelContextWindow) {
-    const systemPromptTokens = systemPrompt ? countTokens(systemPrompt) : 0;
+export async function calculateContextSize(messages, systemPrompt, toolOverheadTokens, mcpOverheadTokens, modelContextWindow) {
+    const systemPromptTokens = systemPrompt ? await countTokens(systemPrompt) : 0;
     const toolOverheadTokensEstimate = toolOverheadTokens ?? 0;
     const mcpOverheadTokensEstimate = mcpOverheadTokens ?? 0;
     let userMessagesTokens = 0;
@@ -48,7 +48,7 @@ export function calculateContextSize(messages, systemPrompt, toolOverheadTokens,
     // Go through ALL messages in this context snapshot
     for (const message of messages) {
         for (const block of message.content) {
-            const counts = countContentBlock(block);
+            const counts = await countContentBlock(block);
             // Accumulate based on message role
             if (message.role === "user") {
                 userMessagesTokens += counts.textTokens;
@@ -66,6 +66,8 @@ export function calculateContextSize(messages, systemPrompt, toolOverheadTokens,
         assistantMessagesTokens,
         toolInputTokens,
         toolResultsTokens,
+        toolOverheadTokens: toolOverheadTokensEstimate,
+        mcpOverheadTokens: mcpOverheadTokensEstimate,
         totalEstimated: systemPromptTokens +
             toolOverheadTokensEstimate +
             mcpOverheadTokensEstimate +
@@ -74,13 +76,6 @@ export function calculateContextSize(messages, systemPrompt, toolOverheadTokens,
             toolInputTokens +
             toolResultsTokens,
     };
-    // Only include optional fields if they have values
-    if (toolOverheadTokensEstimate > 0) {
-        result.toolOverheadTokens = toolOverheadTokensEstimate;
-    }
-    if (mcpOverheadTokensEstimate > 0) {
-        result.mcpOverheadTokens = mcpOverheadTokensEstimate;
-    }
     if (modelContextWindow !== undefined) {
         result.modelContextWindow = modelContextWindow;
     }

package/dist/utils/token-counter.d.ts

@@ -1,19 +1,21 @@
 /**
  * Token counting utilities for tracking context size
- * Uses Anthropic's tokenizer for rough approximation
+ * Uses ai-tokenizer for accurate local estimation (97-99% accuracy for Claude)
+ * Source: https://github.com/coder/ai-tokenizer
  */
 /**
- * Count tokens in a string using Anthropic's tokenizer
- * Note: This is a rough approximation for Claude 3+ models
- * For exact counts, use the API's usage_metadata where available
+ * Count tokens in a string using ai-tokenizer
+ * This provides fast local estimation with 97-99% accuracy for Claude models
+ * @param text - The text to count tokens for
+ * @returns Promise resolving to the token count
  */
-export declare function countTokens(text: string): number;
+export declare function countTokens(text: string): Promise<number>;
 /**
  * Count tokens in a tool result (rawOutput)
  * Handles various data types that might be in tool outputs
  */
-export declare function countToolResultTokens(rawOutput: Record<string, unknown> | undefined): number;
+export declare function countToolResultTokens(rawOutput: Record<string, unknown> | undefined): Promise<number>;
 /**
  * Count tokens in multiple tool results
  */
-export declare function countMultipleToolResults(results: Array<Record<string, unknown> | undefined>): number;
+export declare function countMultipleToolResults(results: Array<Record<string, unknown> | undefined>): Promise<number>;

package/dist/utils/token-counter.js

@@ -1,16 +1,34 @@
 /**
  * Token counting utilities for tracking context size
- * Uses Anthropic's tokenizer for rough approximation
+ * Uses ai-tokenizer for accurate local estimation (97-99% accuracy for Claude)
+ * Source: https://github.com/coder/ai-tokenizer
  */
-import { countTokens as anthropicCountTokens } from "@anthropic-ai/tokenizer";
+import { Tokenizer } from "ai-tokenizer";
+import * as claudeEncoding from "ai-tokenizer/encoding/claude";
+// Create a singleton tokenizer instance for Claude
+let _claudeTokenizer = null;
+function getClaudeTokenizer() {
+    if (!_claudeTokenizer) {
+        _claudeTokenizer = new Tokenizer(claudeEncoding);
+    }
+    return _claudeTokenizer;
+}
 /**
- * Count tokens in a string using Anthropic's tokenizer
- * Note: This is a rough approximation for Claude 3+ models
- * For exact counts, use the API's usage_metadata where available
+ * Count tokens in a string using ai-tokenizer
+ * This provides fast local estimation with 97-99% accuracy for Claude models
+ * @param text - The text to count tokens for
+ * @returns Promise resolving to the token count
  */
-export function countTokens(text) {
+export async function countTokens(text) {
+    if (!text || text.length === 0) {
+        return 0;
+    }
     try {
-        return anthropicCountTokens(text);
+        // Use ai-tokenizer with Claude encoding
+        // Achieves 98.48-99.79% accuracy depending on text length
+        const tokenizer = getClaudeTokenizer();
+        const tokens = tokenizer.encode(text);
+        return tokens.length;
     }
     catch (_error) {
         // Fallback to rough estimation if tokenizer fails
@@ -22,14 +40,14 @@ export function countTokens(text) {
  * Count tokens in a tool result (rawOutput)
  * Handles various data types that might be in tool outputs
  */
-export function countToolResultTokens(rawOutput) {
+export async function countToolResultTokens(rawOutput) {
     if (!rawOutput) {
         return 0;
     }
     try {
         // Convert to string representation (as it would be sent to the LLM)
         const text = JSON.stringify(rawOutput);
-        return countTokens(text);
+        return await countTokens(text);
     }
     catch (_error) {
         // If JSON.stringify fails, return 0
@@ -39,6 +57,7 @@ export function countToolResultTokens(rawOutput) {
 /**
  * Count tokens in multiple tool results
  */
-export function countMultipleToolResults(results) {
-    return results.reduce((total, result) => total + countToolResultTokens(result), 0);
+export async function countMultipleToolResults(results) {
+    const counts = await Promise.all(results.map((result) => countToolResultTokens(result)));
+    return counts.reduce((total, count) => total + count, 0);
 }

package/dist/utils/tool-overhead-calculator.d.ts

@@ -14,11 +14,11 @@ export interface ToolMetadata {
  * Note: Different LLM providers (Anthropic, OpenAI, Gemini) serialize tools
  * slightly differently. This is a rough approximation based on the general format.
  */
-export declare function estimateToolDefinitionTokens(tool: ToolMetadata): number;
+export declare function estimateToolDefinitionTokens(tool: ToolMetadata): Promise<number>;
 /**
  * Estimate total tokens for all tool definitions
  */
-export declare function estimateAllToolsOverhead(tools: ToolMetadata[]): number;
+export declare function estimateAllToolsOverhead(tools: ToolMetadata[]): Promise<number>;
 /**
  * Extract metadata from LangChain tools
  * LangChain tools have .name, .description, and .schema properties

package/dist/utils/tool-overhead-calculator.js

@@ -10,7 +10,7 @@ import { countTokens } from "./token-counter.js";
  * Note: Different LLM providers (Anthropic, OpenAI, Gemini) serialize tools
  * slightly differently. This is a rough approximation based on the general format.
  */
-export function estimateToolDefinitionTokens(tool) {
+export async function estimateToolDefinitionTokens(tool) {
     // Rough serialization of how tools are sent to APIs:
     // {"name": "tool_name", "description": "...", "input_schema": {...}}
     const approximateJson = JSON.stringify({
@@ -18,13 +18,14 @@ export function estimateToolDefinitionTokens(tool) {
         description: tool.description,
         input_schema: tool.schema,
     });
-    return countTokens(approximateJson);
+    return await countTokens(approximateJson);
 }
 /**
  * Estimate total tokens for all tool definitions
  */
-export function estimateAllToolsOverhead(tools) {
-    return tools.reduce((total, tool) => total + estimateToolDefinitionTokens(tool), 0);
+export async function estimateAllToolsOverhead(tools) {
+    const tokens = await Promise.all(tools.map((tool) => estimateToolDefinitionTokens(tool)));
+    return tokens.reduce((total, count) => total + count, 0);
 }
 /**
  * Extract metadata from LangChain tools

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@townco/agent",
-  "version": "0.1.122",
+  "version": "0.1.123",
   "type": "module",
   "module": "index.ts",
   "files": [
@@ -83,12 +83,13 @@
     "@opentelemetry/sdk-trace-base": "^1.28.0",
     "@opentelemetry/sdk-trace-node": "^1.28.0",
     "@opentelemetry/semantic-conventions": "^1.28.0",
-    "@townco/apiclient": "0.0.34",
-    "@townco/core": "0.0.92",
-    "@townco/gui-template": "0.1.111",
-    "@townco/tsconfig": "0.1.111",
-    "@townco/tui-template": "0.1.111",
-    "@townco/ui": "0.1.114",
+    "@townco/apiclient": "0.0.35",
+    "@townco/core": "0.0.93",
+    "@townco/gui-template": "0.1.112",
+    "@townco/tsconfig": "0.1.112",
+    "@townco/tui-template": "0.1.112",
+    "@townco/ui": "0.1.115",
+    "ai-tokenizer": "^1.0.6",
     "exa-js": "^2.0.0",
     "hono": "^4.10.4",
     "langchain": "^1.0.3",