@townco/agent 0.1.52 → 0.1.53

package/dist/utils/__tests__/tool-overhead-calculator.test.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/utils/__tests__/tool-overhead-calculator.test.js

@@ -0,0 +1,153 @@
+import { describe, expect, test } from "bun:test";
+import { estimateAllToolsOverhead, estimateToolDefinitionTokens, extractToolMetadata, } from "../tool-overhead-calculator.js";
+describe("tool-overhead-calculator", () => {
+    describe("estimateToolDefinitionTokens", () => {
+        test("estimates tokens for a simple tool", () => {
+            const tool = {
+                name: "simple_tool",
+                description: "A simple tool",
+                schema: {
+                    type: "object",
+                    properties: {
+                        param1: { type: "string" },
+                    },
+                },
+            };
+            const tokens = estimateToolDefinitionTokens(tool);
+            // Should return a positive number
+            expect(tokens).toBeGreaterThan(0);
+            // Rough sanity check: a simple tool should be at least 20 tokens
+            expect(tokens).toBeGreaterThanOrEqual(20);
+        });
+        test("estimates more tokens for a complex tool", () => {
+            const simpleTool = {
+                name: "simple",
+                description: "Simple",
+                schema: { type: "object" },
+            };
+            const complexTool = {
+                name: "complex_tool_with_long_name",
+                description: "A very complex tool with a long description that explains what it does in great detail and provides usage notes and examples.",
+                schema: {
+                    type: "object",
+                    properties: {
+                        param1: {
+                            type: "string",
+                            description: "First parameter description",
+                        },
+                        param2: {
+                            type: "number",
+                            description: "Second parameter description",
+                        },
+                        param3: {
+                            type: "array",
+                            items: { type: "string" },
+                            description: "Third parameter description",
+                        },
+                    },
+                    required: ["param1", "param2"],
+                },
+            };
+            const simpleTokens = estimateToolDefinitionTokens(simpleTool);
+            const complexTokens = estimateToolDefinitionTokens(complexTool);
+            expect(complexTokens).toBeGreaterThan(simpleTokens);
+        });
+        test("handles empty description", () => {
+            const tool = {
+                name: "test",
+                description: "",
+                schema: {},
+            };
+            const tokens = estimateToolDefinitionTokens(tool);
+            expect(tokens).toBeGreaterThan(0);
+        });
+    });
+    describe("estimateAllToolsOverhead", () => {
+        test("returns 0 for empty array", () => {
+            const tools = [];
+            const tokens = estimateAllToolsOverhead(tools);
+            expect(tokens).toBe(0);
+        });
+        test("sums tokens for multiple tools", () => {
+            const tools = [
+                {
+                    name: "tool1",
+                    description: "First tool",
+                    schema: { type: "object" },
+                },
+                {
+                    name: "tool2",
+                    description: "Second tool",
+                    schema: { type: "object" },
+                },
+            ];
+            const totalTokens = estimateAllToolsOverhead(tools);
+            const tool1Tokens = estimateToolDefinitionTokens(tools[0]);
+            const tool2Tokens = estimateToolDefinitionTokens(tools[1]);
+            expect(totalTokens).toBe(tool1Tokens + tool2Tokens);
+        });
+        test("handles large tool collections", () => {
+            const tools = Array.from({ length: 10 }, (_, i) => ({
+                name: `tool_${i}`,
+                description: `Tool number ${i} with description`,
+                schema: {
+                    type: "object",
+                    properties: {
+                        param: { type: "string" },
+                    },
+                },
+            }));
+            const totalTokens = estimateAllToolsOverhead(tools);
+            // With 10 tools, should be substantial
+            expect(totalTokens).toBeGreaterThan(100);
+        });
+    });
+    describe("extractToolMetadata", () => {
+        test("extracts metadata from LangChain tool object", () => {
+            const langchainTool = {
+                name: "test_tool",
+                description: "Test description",
+                schema: {
+                    type: "object",
+                    properties: { x: { type: "number" } },
+                },
+            };
+            const metadata = extractToolMetadata(langchainTool);
+            expect(metadata.name).toBe("test_tool");
+            expect(metadata.description).toBe("Test description");
+            expect(metadata.schema).toEqual({
+                type: "object",
+                properties: { x: { type: "number" } },
+            });
+        });
+        test("handles missing description", () => {
+            const langchainTool = {
+                name: "test_tool",
+                schema: { type: "object" },
+            };
+            const metadata = extractToolMetadata(langchainTool);
+            expect(metadata.name).toBe("test_tool");
+            expect(metadata.description).toBe("");
+            expect(metadata.schema).toEqual({ type: "object" });
+        });
+        test("handles missing schema", () => {
+            const langchainTool = {
+                name: "test_tool",
+                description: "Test",
+            };
+            const metadata = extractToolMetadata(langchainTool);
+            expect(metadata.name).toBe("test_tool");
+            expect(metadata.description).toBe("Test");
+            expect(metadata.schema).toEqual({});
+        });
+        test("handles missing description and schema", () => {
+            const langchainTool = {
+                name: "test_tool",
+            };
+            const metadata = extractToolMetadata(langchainTool);
+            expect(metadata.name).toBe("test_tool");
+            expect(metadata.description).toBe("");
+            expect(metadata.schema).toEqual({});
+        });
+    });
+});

package/dist/utils/context-size-calculator.d.ts

@@ -5,6 +5,8 @@
 import type { SessionMessage } from "../acp-server/session-storage.js";
 export interface ContextSize {
     systemPromptTokens: number;
+    toolOverheadTokens?: number;
+    mcpOverheadTokens?: number;
     userMessagesTokens: number;
     assistantMessagesTokens: number;
     toolInputTokens: number;
@@ -22,8 +24,11 @@ export interface ContextSize {
  *   all messages, and all tool results
  * - We pass this as `llmReportedTokens` for comparison with our estimate
  * - This helps us validate the accuracy of our tokenizer estimates
- * - Tool declarations are NOT counted separately in our estimate since they're
- *   included in the LLM-reported value
+ *
+ * @param messages - Resolved messages from context entry
+ * @param systemPrompt - Base system prompt (without TODO instructions)
+ * @param llmReportedTokens - From API usage_metadata.input_tokens
+ * @param toolOverheadTokens - Pre-calculated tool overhead (built-in/custom/filesystem tool definitions + TODO instructions)
+ * @param mcpOverheadTokens - Pre-calculated MCP tool overhead (MCP tool definitions)
  */
-export declare function calculateContextSize(messages: SessionMessage[], // Resolved messages from context entry
-systemPrompt?: string, llmReportedTokens?: number): ContextSize;
+export declare function calculateContextSize(messages: SessionMessage[], systemPrompt?: string, llmReportedTokens?: number, toolOverheadTokens?: number, mcpOverheadTokens?: number): ContextSize;

package/dist/utils/context-size-calculator.js

@@ -37,12 +37,17 @@ function countContentBlock(block) {
  *   all messages, and all tool results
  * - We pass this as `llmReportedTokens` for comparison with our estimate
  * - This helps us validate the accuracy of our tokenizer estimates
- * - Tool declarations are NOT counted separately in our estimate since they're
- *   included in the LLM-reported value
+ *
+ * @param messages - Resolved messages from context entry
+ * @param systemPrompt - Base system prompt (without TODO instructions)
+ * @param llmReportedTokens - From API usage_metadata.input_tokens
+ * @param toolOverheadTokens - Pre-calculated tool overhead (built-in/custom/filesystem tool definitions + TODO instructions)
+ * @param mcpOverheadTokens - Pre-calculated MCP tool overhead (MCP tool definitions)
  */
-export function calculateContextSize(messages, // Resolved messages from context entry
-systemPrompt, llmReportedTokens) {
+export function calculateContextSize(messages, systemPrompt, llmReportedTokens, toolOverheadTokens, mcpOverheadTokens) {
     const systemPromptTokens = systemPrompt ? countTokens(systemPrompt) : 0;
+    const toolOverheadTokensEstimate = toolOverheadTokens ?? 0;
+    const mcpOverheadTokensEstimate = mcpOverheadTokens ?? 0;
     let userMessagesTokens = 0;
     let assistantMessagesTokens = 0;
     let toolInputTokens = 0;
@@ -62,17 +67,29 @@ systemPrompt, llmReportedTokens) {
             }
         }
     }
-    return {
+    const result = {
         systemPromptTokens,
         userMessagesTokens,
         assistantMessagesTokens,
         toolInputTokens,
         toolResultsTokens,
         totalEstimated: systemPromptTokens +
+            toolOverheadTokensEstimate +
+            mcpOverheadTokensEstimate +
             userMessagesTokens +
             assistantMessagesTokens +
             toolInputTokens +
             toolResultsTokens,
-        llmReportedInputTokens: llmReportedTokens,
     };
+    // Only include optional fields if they have values
+    if (toolOverheadTokensEstimate > 0) {
+        result.toolOverheadTokens = toolOverheadTokensEstimate;
+    }
+    if (mcpOverheadTokensEstimate > 0) {
+        result.mcpOverheadTokens = mcpOverheadTokensEstimate;
+    }
+    if (llmReportedTokens !== undefined) {
+        result.llmReportedInputTokens = llmReportedTokens;
+    }
+    return result;
 }

package/dist/utils/tool-overhead-calculator.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Tool overhead calculation utilities
+ * Estimates token overhead from tool definitions sent to LLM APIs
+ */
+export interface ToolMetadata {
+    name: string;
+    description: string;
+    schema: Record<string, unknown>;
+}
+/**
+ * Estimate tokens for a single tool definition
+ * LLMs receive tools as JSON with name, description, and parameters schema
+ *
+ * Note: Different LLM providers (Anthropic, OpenAI, Gemini) serialize tools
+ * slightly differently. This is a rough approximation based on the general format.
+ */
+export declare function estimateToolDefinitionTokens(tool: ToolMetadata): number;
+/**
+ * Estimate total tokens for all tool definitions
+ */
+export declare function estimateAllToolsOverhead(tools: ToolMetadata[]): number;
+/**
+ * Extract metadata from LangChain tools
+ * LangChain tools have .name, .description, and .schema properties
+ */
+export declare function extractToolMetadata(langchainTool: {
+    name: string;
+    description?: string;
+    schema?: unknown;
+}): ToolMetadata;

package/dist/utils/tool-overhead-calculator.js

@@ -0,0 +1,54 @@
+/**
+ * Tool overhead calculation utilities
+ * Estimates token overhead from tool definitions sent to LLM APIs
+ */
+import { countTokens } from "./token-counter.js";
+/**
+ * Estimate tokens for a single tool definition
+ * LLMs receive tools as JSON with name, description, and parameters schema
+ *
+ * Note: Different LLM providers (Anthropic, OpenAI, Gemini) serialize tools
+ * slightly differently. This is a rough approximation based on the general format.
+ */
+export function estimateToolDefinitionTokens(tool) {
+    // Rough serialization of how tools are sent to APIs:
+    // {"name": "tool_name", "description": "...", "input_schema": {...}}
+    const approximateJson = JSON.stringify({
+        name: tool.name,
+        description: tool.description,
+        input_schema: tool.schema,
+    });
+    return countTokens(approximateJson);
+}
+/**
+ * Estimate total tokens for all tool definitions
+ */
+export function estimateAllToolsOverhead(tools) {
+    return tools.reduce((total, tool) => total + estimateToolDefinitionTokens(tool), 0);
+}
+/**
+ * Extract metadata from LangChain tools
+ * LangChain tools have .name, .description, and .schema properties
+ */
+export function extractToolMetadata(langchainTool) {
+    // LangChain tools may have Zod schemas - serialize them to JSON
+    let schemaObject = {};
+    if (langchainTool.schema) {
+        // If it's a Zod schema, it might have a _def property
+        if (typeof langchainTool.schema === "object" &&
+            langchainTool.schema !== null) {
+            // Try to serialize to JSON, fallback to empty object
+            try {
+                schemaObject = JSON.parse(JSON.stringify(langchainTool.schema));
+            }
+            catch {
+                schemaObject = {};
+            }
+        }
+    }
+    return {
+        name: langchainTool.name,
+        description: langchainTool.description || "",
+        schema: schemaObject,
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@townco/agent",
-  "version": "0.1.52",
+  "version": "0.1.53",
   "type": "module",
   "module": "index.ts",
   "files": [
@@ -73,11 +73,11 @@
     "@opentelemetry/sdk-trace-base": "^1.28.0",
     "@opentelemetry/sdk-trace-node": "^1.28.0",
     "@opentelemetry/semantic-conventions": "^1.28.0",
-    "@townco/core": "0.0.25",
-    "@townco/gui-template": "0.1.44",
-    "@townco/tsconfig": "0.1.44",
-    "@townco/tui-template": "0.1.44",
-    "@townco/ui": "0.1.47",
+    "@townco/core": "0.0.26",
+    "@townco/gui-template": "0.1.45",
+    "@townco/tsconfig": "0.1.45",
+    "@townco/tui-template": "0.1.45",
+    "@townco/ui": "0.1.48",
     "exa-js": "^2.0.0",
     "hono": "^4.10.4",
     "langchain": "^1.0.3",

package/dist/check-jaeger.d.ts

@@ -1,5 +0,0 @@
-/**
- * Quick script to verify Jaeger connectivity
- * Run with: bun check-jaeger.ts
- */
-export {};

package/dist/check-jaeger.js

@@ -1,82 +0,0 @@
-/**
- * Quick script to verify Jaeger connectivity
- * Run with: bun check-jaeger.ts
- */
-export {}; // Make this a module for top-level await
-console.log("🔍 Checking Jaeger connectivity...\n");
-// Check if Jaeger collector is accepting OTLP on gRPC port
-const checkPort = async (port, protocol) => {
-    try {
-        const net = await import("node:net");
-        const socket = new net.Socket();
-        return new Promise((resolve) => {
-            socket.setTimeout(2000);
-            socket.on("connect", () => {
-                console.log(`✅ ${protocol} port ${port} is accepting connections`);
-                socket.destroy();
-                resolve(true);
-            });
-            socket.on("timeout", () => {
-                console.log(`❌ ${protocol} port ${port} timed out`);
-                socket.destroy();
-                resolve(false);
-            });
-            socket.on("error", (error) => {
-                console.log(`❌ ${protocol} port ${port} not reachable: ${error.message}`);
-                resolve(false);
-            });
-            socket.connect(port, "localhost");
-        });
-    }
-    catch (error) {
-        console.log(`❌ Error checking ${protocol} port ${port}:`, error);
-        return false;
-    }
-};
-// Check Jaeger ports
-console.log("Checking Jaeger ports:");
-console.log("─────────────────────────────");
-await checkPort(4317, "OTLP gRPC"); // Collector OTLP gRPC
-await checkPort(4318, "OTLP HTTP"); // Collector OTLP HTTP
-await checkPort(16686, "UI       "); // Query UI
-console.log("\n🔍 Checking Docker container...\n");
-// Check if Jaeger container is running
-try {
-    const { execSync } = await import("node:child_process");
-    const output = execSync("docker ps --filter name=jaeger --format '{{.Status}}'", {
-        encoding: "utf-8",
-    }).trim();
-    if (output) {
-        console.log(`✅ Jaeger container is running: ${output}`);
-        // Check container logs for OTLP
-        try {
-            const logs = execSync("docker logs jaeger 2>&1 | tail -20", {
-                encoding: "utf-8",
-            });
-            console.log("\n📋 Recent container logs:");
-            console.log("─────────────────────────────");
-            console.log(logs);
-        }
-        catch (e) {
-            console.log("⚠️  Could not fetch container logs");
-        }
-    }
-    else {
-        console.log("❌ No Jaeger container running with name 'jaeger'");
-        console.log("\n💡 Start Jaeger with:");
-        console.log("docker run -d --name jaeger \\");
-        console.log("  -e COLLECTOR_OTLP_ENABLED=true \\");
-        console.log("  -p 16686:16686 \\");
-        console.log("  -p 4317:4317 \\");
-        console.log("  -p 4318:4318 \\");
-        console.log("  jaegertracing/all-in-one:latest");
-    }
-}
-catch (error) {
-    console.log("⚠️  Could not check Docker container:", error);
-}
-console.log("\n🎯 Next steps:");
-console.log("─────────────────────────────");
-console.log("1. Make sure ports above show ✅");
-console.log("2. Run the test: ENABLE_TELEMETRY=true bun test-telemetry.ts");
-console.log("3. Check UI: http://localhost:16686");

package/dist/run-subagents.d.ts

@@ -1,9 +0,0 @@
-#!/usr/bin/env bun
-/**
- * Helper script to test makeSubagentsTool
- *
- * Usage:
- *   bun packages/agent/run-subagents.ts
- *   bun packages/agent/run-subagents.ts "your custom query here"
- */
-export {};

package/dist/run-subagents.js

@@ -1,110 +0,0 @@
-#!/usr/bin/env bun
-/**
- * Helper script to test makeSubagentsTool
- *
- * Usage:
- *   bun packages/agent/run-subagents.ts
- *   bun packages/agent/run-subagents.ts "your custom query here"
- */
-import { makeSubagentsTool } from "./utils";
-import { makeRunnerFromDefinition } from "./runner";
-// Default query if none provided
-const defaultQuery = "Can you search for news about artificial intelligence?";
-const userQuery = process.argv[2] || defaultQuery;
-console.log("🧪 Testing makeSubagentsTool\n");
-console.log("Query:", userQuery);
-console.log("─".repeat(60));
-// Create a simple subagent definition
-const simpleSubagent = {
-    model: "claude-sonnet-4-5-20250929",
-    systemPrompt: "You are a helpful research assistant. Provide concise, informative responses.",
-    tools: [
-        "web_search",
-        "todo_write",
-    ],
-    mcps: [],
-};
-// Create the coordinator agent with subagent tool
-const coordinatorAgent = {
-    model: "claude-sonnet-4-5-20250929",
-    systemPrompt: `You are a coordinator agent that delegates tasks to specialized subagents.
-When you receive a query, evaluate if it needs to be delegated to a subagent.
-If so, use the Task tool with the appropriate subagent.`,
-    tools: [
-        makeSubagentsTool([
-            {
-                agentName: "researcher",
-                description: "Use this agent to research topics, search the web, and gather information",
-                path: import.meta.filename, // Points to itself as a simple test
-            },
-            {
-                agentName: "simple",
-                description: "Use this for general tasks that don't require specialized tools",
-                path: import.meta.filename,
-            },
-        ]),
-    ],
-    mcps: [],
-};
-// Run the test
-async function runTest() {
-    console.log("\n📝 Creating runner...");
-    const runner = makeRunnerFromDefinition(coordinatorAgent);
-    console.log("🚀 Running query...\n");
-    try {
-        const stream = runner.invoke({
-            prompt: [
-                {
-                    type: "text",
-                    text: userQuery,
-                },
-            ],
-            sessionId: "test-session",
-            messageId: "msg-1",
-        });
-        console.log("📊 Stream output:");
-        console.log("─".repeat(60));
-        let messageCount = 0;
-        let toolCallCount = 0;
-        let lastTextContent = "";
-        for await (const chunk of stream) {
-            messageCount++;
-            // Pretty print different types of chunks
-            if (chunk.type === "agent_message_chunk") {
-                const content = chunk.content;
-                if (content.type === "text" && content.text) {
-                    process.stdout.write(content.text);
-                    lastTextContent += content.text;
-                }
-                else if (content.type === "tool_use") {
-                    toolCallCount++;
-                    console.log(`\n\n🔧 Tool Call #${toolCallCount}: ${content.name}`);
-                    console.log("   Input:", JSON.stringify(content.input, null, 2).split('\n').join('\n   '));
-                }
-            }
-            else if (chunk.type === "tool_result") {
-                console.log("\n\n✅ Tool Result:");
-                console.log("   ", JSON.stringify(chunk, null, 2).split('\n').join('\n   '));
-            }
-            else {
-                // Print other chunk types for debugging
-                console.log("\n\n📦 Chunk:", JSON.stringify(chunk, null, 2));
-            }
-        }
-        console.log("\n\n" + "─".repeat(60));
-        console.log("✨ Test completed!");
-        console.log(`   Total chunks: ${messageCount}`);
-        console.log(`   Tool calls: ${toolCallCount}`);
-        console.log("─".repeat(60));
-    }
-    catch (error) {
-        console.error("\n❌ Error during test:");
-        console.error(error);
-        process.exit(1);
-    }
-}
-// Run the test
-runTest().catch((error) => {
-    console.error("Fatal error:", error);
-    process.exit(1);
-});

package/dist/runner/langchain/custom-stream-types.d.ts

@@ -1,36 +0,0 @@
-import { z } from "zod";
-/**
- * Custom stream events emitted by subagent tools via config.writer
- */
-export declare const SubagentToolCallEventSchema: z.ZodObject<{
-    type: z.ZodLiteral<"tool_call">;
-    toolName: z.ZodString;
-}, z.core.$strip>;
-export declare const SubagentMessageEventSchema: z.ZodObject<{
-    type: z.ZodLiteral<"message">;
-    text: z.ZodString;
-}, z.core.$strip>;
-export declare const SubagentEventSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
-    type: z.ZodLiteral<"tool_call">;
-    toolName: z.ZodString;
-}, z.core.$strip>, z.ZodObject<{
-    type: z.ZodLiteral<"message">;
-    text: z.ZodString;
-}, z.core.$strip>], "type">;
-export type SubagentToolCallEvent = z.infer<typeof SubagentToolCallEventSchema>;
-export type SubagentMessageEvent = z.infer<typeof SubagentMessageEventSchema>;
-export type SubagentEvent = z.infer<typeof SubagentEventSchema>;
-/**
- * Wrapper for subagent events that includes the parent tool call ID
- */
-export declare const CustomStreamChunkSchema: z.ZodObject<{
-    parentToolCallId: z.ZodString;
-    event: z.ZodDiscriminatedUnion<[z.ZodObject<{
-        type: z.ZodLiteral<"tool_call">;
-        toolName: z.ZodString;
-    }, z.core.$strip>, z.ZodObject<{
-        type: z.ZodLiteral<"message">;
-        text: z.ZodString;
-    }, z.core.$strip>], "type">;
-}, z.core.$strip>;
-export type CustomStreamChunk = z.infer<typeof CustomStreamChunkSchema>;

package/dist/runner/langchain/custom-stream-types.js

@@ -1,23 +0,0 @@
-import { z } from "zod";
-/**
- * Custom stream events emitted by subagent tools via config.writer
- */
-export const SubagentToolCallEventSchema = z.object({
-    type: z.literal("tool_call"),
-    toolName: z.string(),
-});
-export const SubagentMessageEventSchema = z.object({
-    type: z.literal("message"),
-    text: z.string(),
-});
-export const SubagentEventSchema = z.discriminatedUnion("type", [
-    SubagentToolCallEventSchema,
-    SubagentMessageEventSchema,
-]);
-/**
- * Wrapper for subagent events that includes the parent tool call ID
- */
-export const CustomStreamChunkSchema = z.object({
-    parentToolCallId: z.string(),
-    event: SubagentEventSchema,
-});

package/dist/runner/langchain/tools/bash.d.ts

@@ -1,14 +0,0 @@
-import { z } from "zod";
-export declare function makeBashTool(workingDirectory?: string): import("langchain").DynamicStructuredTool<z.ZodObject<{
-    command: z.ZodString;
-    timeout: z.ZodOptional<z.ZodNumber>;
-    description: z.ZodOptional<z.ZodString>;
-}, z.core.$strip>, {
-    command: string;
-    timeout?: number | undefined;
-    description?: string | undefined;
-}, {
-    command: string;
-    timeout?: number | undefined;
-    description?: string | undefined;
-}, unknown>;