@tyvm/knowhow 0.0.128 → 0.0.129

package/tests/unit/services/MinimalToolsService.test.ts

@@ -0,0 +1,424 @@
+/**
+ * Tests for MinimalToolsService and MinimalToolsMessageProcessor.
+ *
+ * Key scenarios:
+ * 1. Base tools are always in the visible tools array (cache-stable).
+ * 2. Extended tools added via addTools() are NOT in the visible array.
+ * 3. callTool dispatches to any tool in the catalog by name.
+ * 4. inspectTools returns schemas from the full catalog.
+ * 5. MinimalToolsMessageProcessor unwraps callTool(name, args) in the last
+ *    assistant message's tool_calls.
+ * 6. Unwrapping callTool("finalAnswer", ...) → finalAnswer so the agent
+ *    terminates correctly.
+ * 7. Unwrapping callTool("testTool123", ...) → testTool123 so a custom
+ *    required tool is satisfied.
+ * 8. Multiple callTool entries in one message are all unwrapped.
+ * 9. Only the last assistant message is rewritten (older messages untouched).
+ */
+
+// ── Mocks (must come before any imports that touch config/clients) ──────────
+jest.mock("../../../src/config", () => ({
+  getConfig: jest.fn(),
+  getGlobalConfig: jest.fn(),
+  getConfigSync: jest.fn().mockReturnValue({}),
+}));
+
+jest.mock("../../../src/clients", () => ({
+  AIClient: jest.fn(),
+  Clients: jest.fn(),
+}));
+
+jest.mock("../../../src/services", () => ({
+  services: jest.fn().mockReturnValue({
+    Clients: {},
+    Plugins: { listPlugins: jest.fn().mockReturnValue([]) },
+    Agents: {},
+    Tools: {},
+  }),
+}));
+
+// ── Imports ──────────────────────────────────────────────────────────────────
+import { MinimalToolsService } from "../../../src/services/MinimalToolsService";
+import { MinimalToolsMessageProcessor } from "../../../src/processors/MinimalToolsMessageProcessor";
+import { Tool } from "../../../src/clients/types";
+import { Message } from "../../../src/clients/types";
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+function makeTool(name: string): Tool {
+  return {
+    type: "function",
+    function: {
+      name,
+      description: `${name} tool`,
+      parameters: {
+        type: "object",
+        positional: false,
+        properties: {
+          input: { type: "string", description: "input" },
+        },
+        required: [],
+      },
+    },
+  };
+}
+
+function makeCallToolMessage(
+  toolName: string,
+  args: Record<string, any>,
+  id = "tc_1"
+): Message {
+  return {
+    role: "assistant",
+    content: null,
+    tool_calls: [
+      {
+        id,
+        type: "function",
+        function: {
+          name: "callTool",
+          arguments: JSON.stringify({ name: toolName, args }),
+        },
+      },
+    ],
+  } as unknown as Message;
+}
+
+// ── MinimalToolsService ───────────────────────────────────────────────────────
+
+describe("MinimalToolsService", () => {
+  let service: MinimalToolsService;
+
+  beforeEach(() => {
+    service = new MinimalToolsService();
+  });
+
+  test("base tools are always visible", () => {
+    const names = service.getTools().map((t) => t.function.name);
+    expect(names).toContain("finalAnswer");
+    expect(names).toContain("callTool");
+    expect(names).toContain("inspectTools");
+  });
+
+  test("extended tools added via addTools() are NOT in visible tools", () => {
+    const extended = makeTool("readFile");
+    service.addTools([extended]);
+
+    const visibleNames = service.getTools().map((t) => t.function.name);
+    expect(visibleNames).not.toContain("readFile");
+  });
+
+  test("extended tools appear in getAllTools()", () => {
+    const extended = makeTool("readFile");
+    service.addTools([extended]);
+
+    const allNames = service.getAllTools().map((t) => t.function.name);
+    expect(allNames).toContain("readFile");
+    expect(allNames).toContain("finalAnswer");
+  });
+
+  test("visible tools array is stable after adding extended tools", () => {
+    const before = service.getTools().map((t) => t.function.name);
+    service.addTools([makeTool("writeFile"), makeTool("patchFile")]);
+    const after = service.getTools().map((t) => t.function.name);
+    expect(before).toEqual(after);
+  });
+
+  test("duplicate extended tools are not added twice", () => {
+    service.addTools([makeTool("readFile")]);
+    service.addTools([makeTool("readFile")]);
+    const allNames = service.getAllTools().map((t) => t.function.name);
+    const count = allNames.filter((n) => n === "readFile").length;
+    expect(count).toBe(1);
+  });
+
+  test("inspectTools returns schema for all tools when no patterns given", () => {
+    service.addTools([makeTool("readFile"), makeTool("writeFile")]);
+    const result = service.callTool(
+      {
+        id: "tc_inspect",
+        type: "function",
+        function: { name: "inspectTools", arguments: JSON.stringify({}) },
+      },
+      service.getToolNames()
+    );
+    // inspectTools is synchronous inside callTool, check via direct function call
+    const fn = service.getFunction("inspectTools");
+    expect(fn).toBeDefined();
+    const schemas = fn.call(service) as any[];
+    const names = schemas.map((s: any) => s.name);
+    expect(names).toContain("readFile");
+    expect(names).toContain("writeFile");
+    expect(names).toContain("finalAnswer");
+  });
+
+  test("inspectTools filters by glob pattern", () => {
+    service.addTools([makeTool("readFile"), makeTool("writeFile"), makeTool("patchFile")]);
+    const fn = service.getFunction("inspectTools");
+    const schemas = fn.call(service, ["*File"]) as any[];
+    const names = schemas.map((s: any) => s.name);
+    expect(names).toContain("readFile");
+    expect(names).toContain("writeFile");
+    expect(names).toContain("patchFile");
+    expect(names).not.toContain("finalAnswer");
+    expect(names).not.toContain("callTool");
+  });
+
+  test("callToolByName dispatches to an extended tool and returns response", async () => {
+    const mockFn = jest.fn().mockResolvedValue("file contents");
+    service.addTools([makeTool("readFile")]);
+    service.addFunctions({ readFile: mockFn });
+
+    const result = await service.callToolByName(
+      {
+        id: "tc_read",
+        type: "function",
+        function: { name: "readFile", arguments: JSON.stringify({ input: "path/to/file" }) },
+      },
+      "readFile"
+    );
+
+    expect(mockFn).toHaveBeenCalled();
+    expect(result).toBe("file contents");
+  });
+
+  test("callTool() dispatches an extended tool called directly by its real name (not via the callTool meta-tool wrapper)", async () => {
+    // This covers the case where an agent calls a registered tool directly
+    // (toolCall.function.name === "execCommand") instead of routing through
+    // the callTool({name, args}) meta-tool. Previously this threw
+    // "Tool execCommand not found" because extended tools only live in
+    // allToolsCatalog, not in the frozen visible `tools` array that the base
+    // ToolsService.getTool() searches.
+    const mockFn = jest.fn().mockResolvedValue("command output");
+    service.addTools([makeTool("execCommand")]);
+    service.addFunctions({ execCommand: mockFn });
+
+    const result = await service.callTool({
+      id: "tc_direct_exec",
+      type: "function",
+      function: {
+        name: "execCommand",
+        arguments: JSON.stringify({ input: "ls" }),
+      },
+    });
+
+    expect(mockFn).toHaveBeenCalled();
+    expect(result.functionResp).toBe("command output");
+    expect(result.toolMessages[0].content).toBe("command output");
+
+    // Visible tools array must remain stable (cache-stable guarantee) after
+    // dispatching a direct call to an extended tool.
+    const visibleNames = service.getTools().map((t) => t.function.name);
+    expect(visibleNames).not.toContain("execCommand");
+  });
+
+  test("callTool() returns a standard error for an unknown tool name", async () => {
+    const result = await service.callTool({
+      id: "tc_unknown",
+      type: "function",
+      function: { name: "totallyUnknownTool", arguments: JSON.stringify({}) },
+    });
+
+    expect(result.toolMessages[0].name).toBe("error");
+    expect(result.toolMessages[0].content).toMatch(/not found|not enabled/);
+  });
+});
+
+// ── MinimalToolsMessageProcessor ─────────────────────────────────────────────
+
+describe("MinimalToolsMessageProcessor", () => {
+  let processor: MinimalToolsMessageProcessor;
+  let processorFn: ReturnType<MinimalToolsMessageProcessor["createProcessor"]>;
+
+  beforeEach(() => {
+    processor = new MinimalToolsMessageProcessor();
+    processorFn = processor.createProcessor();
+  });
+
+  function applyProcessor(messages: Message[]): Message[] {
+    const original = JSON.parse(JSON.stringify(messages)) as Message[];
+    const modified = JSON.parse(JSON.stringify(messages)) as Message[];
+    processorFn(original, modified);
+    return modified;
+  }
+
+  test("unwraps callTool(name, args) to the real tool call", () => {
+    const messages: Message[] = [
+      makeCallToolMessage("readFile", { filePath: "foo.ts" }, "tc_1"),
+    ];
+
+    const result = applyProcessor(messages);
+    const lastMsg = result[result.length - 1];
+
+    expect(lastMsg.tool_calls).toHaveLength(1);
+    expect(lastMsg.tool_calls[0].function.name).toBe("readFile");
+    expect(JSON.parse(lastMsg.tool_calls[0].function.arguments)).toEqual({
+      filePath: "foo.ts",
+    });
+    // ID preserved
+    expect(lastMsg.tool_calls[0].id).toBe("tc_1");
+  });
+
+  test("unwraps callTool('finalAnswer', ...) so agent terminates", () => {
+    const messages: Message[] = [
+      makeCallToolMessage("finalAnswer", { answer: "All done!" }, "tc_fa"),
+    ];
+
+    const result = applyProcessor(messages);
+    const lastMsg = result[result.length - 1];
+
+    expect(lastMsg.tool_calls[0].function.name).toBe("finalAnswer");
+    expect(JSON.parse(lastMsg.tool_calls[0].function.arguments)).toEqual({
+      answer: "All done!",
+    });
+  });
+
+  test("unwraps callTool('testTool123', ...) so custom required tool is satisfied", () => {
+    const messages: Message[] = [
+      makeCallToolMessage("testTool123", { data: "value" }, "tc_custom"),
+    ];
+
+    const result = applyProcessor(messages);
+    const lastMsg = result[result.length - 1];
+
+    expect(lastMsg.tool_calls[0].function.name).toBe("testTool123");
+    expect(JSON.parse(lastMsg.tool_calls[0].function.arguments)).toEqual({
+      data: "value",
+    });
+  });
+
+  test("unwraps multiple callTool entries in a single assistant message", () => {
+    const messages: Message[] = [
+      {
+        role: "assistant",
+        content: null,
+        tool_calls: [
+          {
+            id: "tc_a",
+            type: "function",
+            function: {
+              name: "callTool",
+              arguments: JSON.stringify({ name: "readFile", args: { filePath: "a.ts" } }),
+            },
+          },
+          {
+            id: "tc_b",
+            type: "function",
+            function: {
+              name: "callTool",
+              arguments: JSON.stringify({ name: "writeFile", args: { filePath: "b.ts", content: "hi" } }),
+            },
+          },
+        ],
+      } as unknown as Message,
+    ];
+
+    const result = applyProcessor(messages);
+    const lastMsg = result[result.length - 1];
+
+    expect(lastMsg.tool_calls).toHaveLength(2);
+    expect(lastMsg.tool_calls[0].function.name).toBe("readFile");
+    expect(lastMsg.tool_calls[1].function.name).toBe("writeFile");
+    expect(JSON.parse(lastMsg.tool_calls[0].function.arguments)).toEqual({ filePath: "a.ts" });
+    expect(JSON.parse(lastMsg.tool_calls[1].function.arguments)).toEqual({
+      filePath: "b.ts",
+      content: "hi",
+    });
+  });
+
+  test("does NOT rewrite older assistant messages (only the last one)", () => {
+    const messages: Message[] = [
+      // older assistant message with callTool — should NOT be rewritten
+      makeCallToolMessage("oldTool", { x: 1 }, "tc_old"),
+      // tool result for that call
+      {
+        role: "tool",
+        tool_call_id: "tc_old",
+        name: "callTool",
+        content: "some result",
+      } as unknown as Message,
+      // newest assistant message — should be rewritten
+      makeCallToolMessage("finalAnswer", { answer: "done" }, "tc_new"),
+    ];
+
+    const result = applyProcessor(messages);
+
+    // First assistant message should be unchanged (still "callTool")
+    const firstAssistant = result[0];
+    expect(firstAssistant.tool_calls[0].function.name).toBe("callTool");
+
+    // Last assistant message should be rewritten
+    const lastAssistant = result[result.length - 1];
+    expect(lastAssistant.tool_calls[0].function.name).toBe("finalAnswer");
+  });
+
+  test("leaves non-callTool tool_calls unchanged", () => {
+    const messages: Message[] = [
+      {
+        role: "assistant",
+        content: null,
+        tool_calls: [
+          {
+            id: "tc_direct",
+            type: "function",
+            function: {
+              name: "inspectTools",
+              arguments: JSON.stringify({ patterns: ["*"] }),
+            },
+          },
+        ],
+      } as unknown as Message,
+    ];
+
+    const result = applyProcessor(messages);
+    const lastMsg = result[result.length - 1];
+
+    expect(lastMsg.tool_calls[0].function.name).toBe("inspectTools");
+  });
+
+  test("skips messages with no tool_calls", () => {
+    const messages: Message[] = [
+      { role: "user", content: "hello" } as Message,
+      { role: "assistant", content: "I will help" } as Message,
+    ];
+
+    const result = applyProcessor(messages);
+    expect(result[1].tool_calls).toBeUndefined();
+  });
+
+  test("mixed: callTool entries alongside direct tool calls are selectively unwrapped", () => {
+    const messages: Message[] = [
+      {
+        role: "assistant",
+        content: null,
+        tool_calls: [
+          {
+            id: "tc_direct",
+            type: "function",
+            function: {
+              name: "inspectTools",
+              arguments: JSON.stringify({}),
+            },
+          },
+          {
+            id: "tc_wrapped",
+            type: "function",
+            function: {
+              name: "callTool",
+              arguments: JSON.stringify({ name: "readFile", args: { filePath: "x.ts" } }),
+            },
+          },
+        ],
+      } as unknown as Message,
+    ];
+
+    const result = applyProcessor(messages);
+    const lastMsg = result[result.length - 1];
+
+    expect(lastMsg.tool_calls).toHaveLength(2);
+    // Direct call preserved
+    expect(lastMsg.tool_calls[0].function.name).toBe("inspectTools");
+    // Wrapped call unwrapped
+    expect(lastMsg.tool_calls[1].function.name).toBe("readFile");
+  });
+});

package/ts_build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@tyvm/knowhow",
-    "version": "0.0.128",
+    "version": "0.0.129",
     "description": "ai cli with plugins and agents",
     "main": "ts_build/src/index.js",
     "bin": {
@@ -22,6 +22,7 @@
         "lint:deps": "depcheck --config=.depcheckrc",
         "lint:all": "npm run lint && npm run lint:deps",
         "check:model-pricing": "ts-node scripts/check-model-pricing.ts",
+        "debug:cache-miss": "ts-node scripts/cache-miss-debug.ts",
         "version:nightly": "bash scripts/publish.sh version:nightly",
         "version:dev": "bash scripts/publish.sh version:dev",
         "publish:nightly": "bash scripts/publish.sh nightly",

package/ts_build/src/agents/base/base.d.ts

@@ -109,13 +109,16 @@ export declare abstract class BaseAgent implements IAgent {
     getTurnCount(): number;
     adjustTotalCostUsd(cost: number): void;
     getTotalCostUsd(): number;
-    adjustTokenUsage(usage: any): void;
+    adjustTokenUsage(usage: any, messages?: Message[]): void;
     getTokenUsage(): {
         totalInputTokens: number;
         totalOutputTokens: number;
         totalCacheReadTokens: number;
         totalCacheWriteTokens: number;
     };
+    protected createAgentCompletion(params: Parameters<GenericClient["createChatCompletion"]>[0], options?: {
+        interruptValue?: CompletionResponse;
+    }): Promise<CompletionResponse>;
     startNewThread(messages: Message[]): void;
     updateCurrentThread(messages: Message[]): void;
     getThreads(): Message[][];

package/ts_build/src/agents/base/base.js

@@ -284,19 +284,22 @@ class BaseAgent {
     getTotalCostUsd() {
         return this.totalCostUsd;
     }
-    adjustTokenUsage(usage) {
+    adjustTokenUsage(usage, messages) {
         if (!usage)
             return;
         const inputTokens = usage.input_tokens ?? usage.prompt_tokens ?? 0;
         const outputTokens = usage.output_tokens ?? usage.completion_tokens ?? 0;
-        const cacheReadTokens = usage.cache_read_input_tokens ?? usage.cache_read_tokens ??
-            usage.prompt_tokens_details?.cached_tokens ?? 0;
+        const cacheReadTokens = usage.cache_read_input_tokens ??
+            usage.cache_read_tokens ??
+            usage.prompt_tokens_details?.cached_tokens ??
+            0;
         const cacheWriteTokens = usage.cache_creation_input_tokens ?? usage.cache_write_tokens ?? 0;
         this.totalInputTokens += inputTokens;
         this.totalOutputTokens += outputTokens;
         this.totalCacheReadTokens += cacheReadTokens;
         this.totalCacheWriteTokens += cacheWriteTokens;
         this.agentEvents.emit(this.eventTypes.tokenUsage, {
+            timestamp: new Date().toISOString(),
             inputTokens,
             outputTokens,
             cacheReadTokens,
@@ -305,6 +308,7 @@ class BaseAgent {
             totalOutputTokens: this.totalOutputTokens,
             totalCacheReadTokens: this.totalCacheReadTokens,
             totalCacheWriteTokens: this.totalCacheWriteTokens,
+            messages,
         });
     }
     getTokenUsage() {
@@ -315,6 +319,16 @@ class BaseAgent {
             totalCacheWriteTokens: this.totalCacheWriteTokens,
         };
     }
+    async createAgentCompletion(params, options = {}) {
+        const { interruptValue } = options;
+        const callPromise = this.getClient().createChatCompletion(params);
+        const response = interruptValue
+            ? await this.makeInterruptible(callPromise, interruptValue)
+            : await callPromise;
+        this.adjustTotalCostUsd(response?.usd_cost);
+        this.adjustTokenUsage(response?.usage, params.messages);
+        return response;
+    }
     startNewThread(messages) {
         this.currentThread++;
         this.agentEvents.emit(this.eventTypes.newThread, messages);
@@ -338,7 +352,14 @@ class BaseAgent {
             toolCallId: toolCall.id,
             functionName: toolCall.function?.name,
             functionArgs: {},
-            toolMessages: [{ role: "tool", tool_call_id: toolCall.id, name: toolCall.function?.name, content: interruptMsg }],
+            toolMessages: [
+                {
+                    role: "tool",
+                    tool_call_id: toolCall.id,
+                    name: toolCall.function?.name,
+                    content: interruptMsg,
+                },
+            ],
         };
         const { functionResp, toolMessages } = await this.makeInterruptible(this.tools.callTool(toolCall, this.getEnabledToolNames()), interruptResult);
         this.agentEvents.emit(this.eventTypes.toolUsed, {
@@ -470,12 +491,14 @@ class BaseAgent {
                 this._interruptResolve = null;
                 resolve(value);
             };
-            promise.then((result) => {
+            promise
+                .then((result) => {
                 if (this._interruptResolve) {
                     this._interruptResolve = null;
                     resolve(result);
                 }
-            }).catch((err) => {
+            })
+                .catch((err) => {
                 if (this._interruptResolve) {
                     this._interruptResolve = null;
                 }
@@ -536,18 +559,25 @@ class BaseAgent {
             const endIndex = messages.length;
             messages = await this.messageProcessor.processMessages(messages, "pre_call");
             const interruptResponse = {
-                choices: [{ message: { role: "assistant", content: "User interrupted this AI completion. Please continue." } }],
+                choices: [
+                    {
+                        message: {
+                            role: "assistant",
+                            content: "User interrupted this AI completion. Please continue.",
+                        },
+                    },
+                ],
                 model,
                 usage: undefined,
                 usd_cost: 0,
             };
-            const response = await this.makeInterruptible(this.getClient().createChatCompletion({
+            const response = await this.createAgentCompletion({
                 model,
                 messages,
                 tools: this.getEnabledTools(),
                 tool_choice: "auto",
                 long_ttl_cache: this.runTime() > 300_000,
-            }), interruptResponse);
+            }, { interruptValue: interruptResponse });
             if (this.status === this.eventTypes.pause) {
                 this.log("Agent was paused after completion, waiting before processing tool calls");
                 await this.unpaused();
@@ -564,17 +594,15 @@ class BaseAgent {
                     throw new Error(`AI response error: ${errMsg}`);
                 }
             }
-            this.adjustTotalCostUsd(response?.usd_cost);
-            this.adjustTokenUsage(response?.usage);
             this.log("agent response cost: " + response?.usd_cost);
             for (const choice of response.choices) {
                 messages.push(choice.message);
                 messages = await this.messageProcessor.processMessages(messages, "post_call");
                 const lastMessage = messages[messages.length - 1];
                 this.logMessages([lastMessage]);
-                const toolCalls = lastMessage.tool_calls;
                 if (lastMessage.tool_calls) {
                     messages = await this.messageProcessor.processMessages(messages, "pre_tools");
+                    const toolCalls = messages[messages.length - 1].tool_calls;
                     this.updateCurrentThread(messages);
                     const truncationWarning = this.detectTruncatedToolCalls(toolCalls, response);
                     if (truncationWarning) {
@@ -695,7 +723,10 @@ class BaseAgent {
                 return this.call(userInput, _messages, retryCount + 1);
             }
             this.log(`Agent failed: ${e}`, "error");
-            if (e != null && typeof e === "object" && "response" in e && "data" in e.response) {
+            if (e != null &&
+                typeof e === "object" &&
+                "response" in e &&
+                "data" in e.response) {
                 this.log(`Error response data: ${JSON.stringify(e.response.data, null, 2)}`, "error");
             }
             this.agentEvents.emit(this.eventTypes.done, e.message);
@@ -817,18 +848,18 @@ class BaseAgent {
     Your output will be used to guide the work of the agent, and determine when we've accomplished the goal
     `;
         const model = this.getModel();
-        const response = await this.getClient().createChatCompletion({
+        const taskBreakdownMessages = [
+            ...messages,
+            {
+                role: "user",
+                content: taskPrompt,
+            },
+        ];
+        const response = await this.createAgentCompletion({
             model,
-            messages: [
-                ...messages,
-                {
-                    role: "user",
-                    content: taskPrompt,
-                },
-            ],
+            messages: taskBreakdownMessages,
             max_tokens: 2000,
         });
-        this.adjustTotalCostUsd(response.usd_cost);
         this.taskBreakdown = response.choices[0].message.content;
         this.log(`task breakdown cost: ${response.usd_cost}`);
         return this.taskBreakdown;
@@ -851,17 +882,17 @@ class BaseAgent {
 
       `;
         const model = this.getModel();
-        const response = await this.getClient().createChatCompletion({
+        const compressMessagesPayload = [
+            ...messages,
+            {
+                role: "user",
+                content: toCompressPrompt,
+            },
+        ];
+        const response = await this.createAgentCompletion({
             model,
-            messages: [
-                ...messages,
-                {
-                    role: "user",
-                    content: toCompressPrompt,
-                },
-            ],
+            messages: compressMessagesPayload,
         });
-        this.adjustTotalCostUsd(response.usd_cost);
         const summaries = response.choices.map((c) => c.message.content);
         this.summaries.push(...summaries);
         const startMessages = [