@tyvm/knowhow 0.0.128 → 0.0.130

package/src/services/modules/types.ts

@@ -13,6 +13,7 @@ import { MediaProcessorService } from "../MediaProcessorService";
 import { TunnelHandler } from "@tyvm/knowhow-tunnel";
 import { EventService } from "../EventService";
 import { ConversionService } from "../conversion/ConversionService";
+import { BehaviorsService } from "../BehaviorsService";
 
 /*
  *
@@ -61,6 +62,7 @@ export interface ModuleContext {
   Conversion?: ConversionService;
   Tunnel?: TunnelHandler;
   Program?: Command;
+  Behaviors?: BehaviorsService
 }
 
 export interface KnowhowModule {

package/src/worker.ts

@@ -225,9 +225,16 @@ export async function worker(options?: {
     console.log(
       "Worker tools configured! Update knowhow.json to adjust which tools are allowed by the worker."
     );
+    console.log(
+      "Tunnel is disabled by default. Set worker.tunnel.enabled = true in knowhow.json to enable it."
+    );
     config.worker = {
       ...config.worker,
       allowedTools: Tools.getToolNames(),
+      tunnel: {
+        enabled: false,
+        ...config.worker?.tunnel,
+      },
     };
     await updateConfig(config);
     return;

package/tests/processors/TokenCompressor.test.ts

@@ -83,6 +83,51 @@ describe("TokenCompressor", () => {
       expect(parsed.smallProp).toBe("small"); // Should remain unchanged
       expect(parsed.largeProp).toContain("[COMPRESSED_JSON_PROPERTY"); // Should be compressed
     });
+
+    it("should compress large content that merely CONTAINS a compression marker deep inside (regression)", () => {
+      // Regression for the false-positive in isAlreadyCompressed(): a large
+      // payload (e.g. a usage.json / message-history dump) whose body contains
+      // the substring "[COMPRESSED_STRING" deep inside its nested data was
+      // incorrectly treated as already-compressed and returned uncompressed,
+      // letting 1MB+ raw tool responses flow into createCompletion untouched.
+      const nestedMarker =
+        '[COMPRESSED_STRING - 10155 tokens in 3 chunks]\\nKey: compressed_abc123';
+      // Build a big JSON array whose entries embed the marker far from the start.
+      const entry = JSON.stringify({
+        role: "tool",
+        name: "readFile",
+        content: nestedMarker,
+        filler: "z".repeat(500),
+      });
+      const bigContent = "[" + new Array(200).fill(entry).join(",") + "]";
+
+      // Sanity: the raw content contains the marker but is NOT at the start.
+      expect(bigContent.includes("[COMPRESSED_STRING")).toBe(true);
+      expect(bigContent.trimStart().startsWith("[COMPRESSED_STRING")).toBe(false);
+
+      const result = tokenCompressor.compressContent(bigContent);
+
+      // It MUST actually get compressed, not returned untouched.
+      expect(result.length).toBeLessThan(bigContent.length);
+    });
+
+    it("should NOT recompress content that genuinely starts with a compression marker", () => {
+      // Idempotency: real compressor output begins with "[COMPRESSED_STRING".
+      const alreadyCompressed =
+        "[COMPRESSED_STRING - 999 tokens in 2 chunks]\nKey: compressed_xyz\nPath: \nPreview: ...";
+      const result = tokenCompressor.compressContent(alreadyCompressed);
+      expect(result).toBe(alreadyCompressed);
+    });
+
+    it("should NOT recompress JSON whose top-level _schema_key marks it compressed", () => {
+      // Idempotency: JSON-compressed output carries a top-level _schema_key.
+      const compressedJson = JSON.stringify({
+        _schema_key: "compressed_schema_1",
+        data: "x".repeat(40000),
+      });
+      const result = tokenCompressor.compressContent(compressedJson);
+      expect(result).toBe(compressedJson);
+    });
   });
 
   describe("compressStringInChunks", () => {

package/tests/unit/services/MinimalToolsService.test.ts

@@ -0,0 +1,424 @@
+/**
+ * Tests for MinimalToolsService and MinimalToolsMessageProcessor.
+ *
+ * Key scenarios:
+ * 1. Base tools are always in the visible tools array (cache-stable).
+ * 2. Extended tools added via addTools() are NOT in the visible array.
+ * 3. callTool dispatches to any tool in the catalog by name.
+ * 4. inspectTools returns schemas from the full catalog.
+ * 5. MinimalToolsMessageProcessor unwraps callTool(name, args) in the last
+ *    assistant message's tool_calls.
+ * 6. Unwrapping callTool("finalAnswer", ...) → finalAnswer so the agent
+ *    terminates correctly.
+ * 7. Unwrapping callTool("testTool123", ...) → testTool123 so a custom
+ *    required tool is satisfied.
+ * 8. Multiple callTool entries in one message are all unwrapped.
+ * 9. Only the last assistant message is rewritten (older messages untouched).
+ */
+
+// ── Mocks (must come before any imports that touch config/clients) ──────────
+jest.mock("../../../src/config", () => ({
+  getConfig: jest.fn(),
+  getGlobalConfig: jest.fn(),
+  getConfigSync: jest.fn().mockReturnValue({}),
+}));
+
+jest.mock("../../../src/clients", () => ({
+  AIClient: jest.fn(),
+  Clients: jest.fn(),
+}));
+
+jest.mock("../../../src/services", () => ({
+  services: jest.fn().mockReturnValue({
+    Clients: {},
+    Plugins: { listPlugins: jest.fn().mockReturnValue([]) },
+    Agents: {},
+    Tools: {},
+  }),
+}));
+
+// ── Imports ──────────────────────────────────────────────────────────────────
+import { MinimalToolsService } from "../../../src/services/MinimalToolsService";
+import { MinimalToolsMessageProcessor } from "../../../src/processors/MinimalToolsMessageProcessor";
+import { Tool } from "../../../src/clients/types";
+import { Message } from "../../../src/clients/types";
+
+// ── Helpers ──────────────────────────────────────────────────────────────────
+
+function makeTool(name: string): Tool {
+  return {
+    type: "function",
+    function: {
+      name,
+      description: `${name} tool`,
+      parameters: {
+        type: "object",
+        positional: false,
+        properties: {
+          input: { type: "string", description: "input" },
+        },
+        required: [],
+      },
+    },
+  };
+}
+
+function makeCallToolMessage(
+  toolName: string,
+  args: Record<string, any>,
+  id = "tc_1"
+): Message {
+  return {
+    role: "assistant",
+    content: null,
+    tool_calls: [
+      {
+        id,
+        type: "function",
+        function: {
+          name: "callTool",
+          arguments: JSON.stringify({ name: toolName, args }),
+        },
+      },
+    ],
+  } as unknown as Message;
+}
+
+// ── MinimalToolsService ───────────────────────────────────────────────────────
+
+describe("MinimalToolsService", () => {
+  let service: MinimalToolsService;
+
+  beforeEach(() => {
+    service = new MinimalToolsService();
+  });
+
+  test("base tools are always visible", () => {
+    const names = service.getTools().map((t) => t.function.name);
+    expect(names).toContain("finalAnswer");
+    expect(names).toContain("callTool");
+    expect(names).toContain("inspectTools");
+  });
+
+  test("extended tools added via addTools() are NOT in visible tools", () => {
+    const extended = makeTool("readFile");
+    service.addTools([extended]);
+
+    const visibleNames = service.getTools().map((t) => t.function.name);
+    expect(visibleNames).not.toContain("readFile");
+  });
+
+  test("extended tools appear in getAllTools()", () => {
+    const extended = makeTool("readFile");
+    service.addTools([extended]);
+
+    const allNames = service.getAllTools().map((t) => t.function.name);
+    expect(allNames).toContain("readFile");
+    expect(allNames).toContain("finalAnswer");
+  });
+
+  test("visible tools array is stable after adding extended tools", () => {
+    const before = service.getTools().map((t) => t.function.name);
+    service.addTools([makeTool("writeFile"), makeTool("patchFile")]);
+    const after = service.getTools().map((t) => t.function.name);
+    expect(before).toEqual(after);
+  });
+
+  test("duplicate extended tools are not added twice", () => {
+    service.addTools([makeTool("readFile")]);
+    service.addTools([makeTool("readFile")]);
+    const allNames = service.getAllTools().map((t) => t.function.name);
+    const count = allNames.filter((n) => n === "readFile").length;
+    expect(count).toBe(1);
+  });
+
+  test("inspectTools returns schema for all tools when no patterns given", () => {
+    service.addTools([makeTool("readFile"), makeTool("writeFile")]);
+    const result = service.callTool(
+      {
+        id: "tc_inspect",
+        type: "function",
+        function: { name: "inspectTools", arguments: JSON.stringify({}) },
+      },
+      service.getToolNames()
+    );
+    // inspectTools is synchronous inside callTool, check via direct function call
+    const fn = service.getFunction("inspectTools");
+    expect(fn).toBeDefined();
+    const schemas = fn.call(service) as any[];
+    const names = schemas.map((s: any) => s.name);
+    expect(names).toContain("readFile");
+    expect(names).toContain("writeFile");
+    expect(names).toContain("finalAnswer");
+  });
+
+  test("inspectTools filters by glob pattern", () => {
+    service.addTools([makeTool("readFile"), makeTool("writeFile"), makeTool("patchFile")]);
+    const fn = service.getFunction("inspectTools");
+    const schemas = fn.call(service, ["*File"]) as any[];
+    const names = schemas.map((s: any) => s.name);
+    expect(names).toContain("readFile");
+    expect(names).toContain("writeFile");
+    expect(names).toContain("patchFile");
+    expect(names).not.toContain("finalAnswer");
+    expect(names).not.toContain("callTool");
+  });
+
+  test("callToolByName dispatches to an extended tool and returns response", async () => {
+    const mockFn = jest.fn().mockResolvedValue("file contents");
+    service.addTools([makeTool("readFile")]);
+    service.addFunctions({ readFile: mockFn });
+
+    const result = await service.callToolByName(
+      {
+        id: "tc_read",
+        type: "function",
+        function: { name: "readFile", arguments: JSON.stringify({ input: "path/to/file" }) },
+      },
+      "readFile"
+    );
+
+    expect(mockFn).toHaveBeenCalled();
+    expect(result).toBe("file contents");
+  });
+
+  test("callTool() dispatches an extended tool called directly by its real name (not via the callTool meta-tool wrapper)", async () => {
+    // This covers the case where an agent calls a registered tool directly
+    // (toolCall.function.name === "execCommand") instead of routing through
+    // the callTool({name, args}) meta-tool. Previously this threw
+    // "Tool execCommand not found" because extended tools only live in
+    // allToolsCatalog, not in the frozen visible `tools` array that the base
+    // ToolsService.getTool() searches.
+    const mockFn = jest.fn().mockResolvedValue("command output");
+    service.addTools([makeTool("execCommand")]);
+    service.addFunctions({ execCommand: mockFn });
+
+    const result = await service.callTool({
+      id: "tc_direct_exec",
+      type: "function",
+      function: {
+        name: "execCommand",
+        arguments: JSON.stringify({ input: "ls" }),
+      },
+    });
+
+    expect(mockFn).toHaveBeenCalled();
+    expect(result.functionResp).toBe("command output");
+    expect(result.toolMessages[0].content).toBe("command output");
+
+    // Visible tools array must remain stable (cache-stable guarantee) after
+    // dispatching a direct call to an extended tool.
+    const visibleNames = service.getTools().map((t) => t.function.name);
+    expect(visibleNames).not.toContain("execCommand");
+  });
+
+  test("callTool() returns a standard error for an unknown tool name", async () => {
+    const result = await service.callTool({
+      id: "tc_unknown",
+      type: "function",
+      function: { name: "totallyUnknownTool", arguments: JSON.stringify({}) },
+    });
+
+    expect(result.toolMessages[0].name).toBe("error");
+    expect(result.toolMessages[0].content).toMatch(/not found|not enabled/);
+  });
+});
+
+// ── MinimalToolsMessageProcessor ─────────────────────────────────────────────
+
+describe("MinimalToolsMessageProcessor", () => {
+  let processor: MinimalToolsMessageProcessor;
+  let processorFn: ReturnType<MinimalToolsMessageProcessor["createProcessor"]>;
+
+  beforeEach(() => {
+    processor = new MinimalToolsMessageProcessor();
+    processorFn = processor.createProcessor();
+  });
+
+  function applyProcessor(messages: Message[]): Message[] {
+    const original = JSON.parse(JSON.stringify(messages)) as Message[];
+    const modified = JSON.parse(JSON.stringify(messages)) as Message[];
+    processorFn(original, modified);
+    return modified;
+  }
+
+  test("unwraps callTool(name, args) to the real tool call", () => {
+    const messages: Message[] = [
+      makeCallToolMessage("readFile", { filePath: "foo.ts" }, "tc_1"),
+    ];
+
+    const result = applyProcessor(messages);
+    const lastMsg = result[result.length - 1];
+
+    expect(lastMsg.tool_calls).toHaveLength(1);
+    expect(lastMsg.tool_calls[0].function.name).toBe("readFile");
+    expect(JSON.parse(lastMsg.tool_calls[0].function.arguments)).toEqual({
+      filePath: "foo.ts",
+    });
+    // ID preserved
+    expect(lastMsg.tool_calls[0].id).toBe("tc_1");
+  });
+
+  test("unwraps callTool('finalAnswer', ...) so agent terminates", () => {
+    const messages: Message[] = [
+      makeCallToolMessage("finalAnswer", { answer: "All done!" }, "tc_fa"),
+    ];
+
+    const result = applyProcessor(messages);
+    const lastMsg = result[result.length - 1];
+
+    expect(lastMsg.tool_calls[0].function.name).toBe("finalAnswer");
+    expect(JSON.parse(lastMsg.tool_calls[0].function.arguments)).toEqual({
+      answer: "All done!",
+    });
+  });
+
+  test("unwraps callTool('testTool123', ...) so custom required tool is satisfied", () => {
+    const messages: Message[] = [
+      makeCallToolMessage("testTool123", { data: "value" }, "tc_custom"),
+    ];
+
+    const result = applyProcessor(messages);
+    const lastMsg = result[result.length - 1];
+
+    expect(lastMsg.tool_calls[0].function.name).toBe("testTool123");
+    expect(JSON.parse(lastMsg.tool_calls[0].function.arguments)).toEqual({
+      data: "value",
+    });
+  });
+
+  test("unwraps multiple callTool entries in a single assistant message", () => {
+    const messages: Message[] = [
+      {
+        role: "assistant",
+        content: null,
+        tool_calls: [
+          {
+            id: "tc_a",
+            type: "function",
+            function: {
+              name: "callTool",
+              arguments: JSON.stringify({ name: "readFile", args: { filePath: "a.ts" } }),
+            },
+          },
+          {
+            id: "tc_b",
+            type: "function",
+            function: {
+              name: "callTool",
+              arguments: JSON.stringify({ name: "writeFile", args: { filePath: "b.ts", content: "hi" } }),
+            },
+          },
+        ],
+      } as unknown as Message,
+    ];
+
+    const result = applyProcessor(messages);
+    const lastMsg = result[result.length - 1];
+
+    expect(lastMsg.tool_calls).toHaveLength(2);
+    expect(lastMsg.tool_calls[0].function.name).toBe("readFile");
+    expect(lastMsg.tool_calls[1].function.name).toBe("writeFile");
+    expect(JSON.parse(lastMsg.tool_calls[0].function.arguments)).toEqual({ filePath: "a.ts" });
+    expect(JSON.parse(lastMsg.tool_calls[1].function.arguments)).toEqual({
+      filePath: "b.ts",
+      content: "hi",
+    });
+  });
+
+  test("does NOT rewrite older assistant messages (only the last one)", () => {
+    const messages: Message[] = [
+      // older assistant message with callTool — should NOT be rewritten
+      makeCallToolMessage("oldTool", { x: 1 }, "tc_old"),
+      // tool result for that call
+      {
+        role: "tool",
+        tool_call_id: "tc_old",
+        name: "callTool",
+        content: "some result",
+      } as unknown as Message,
+      // newest assistant message — should be rewritten
+      makeCallToolMessage("finalAnswer", { answer: "done" }, "tc_new"),
+    ];
+
+    const result = applyProcessor(messages);
+
+    // First assistant message should be unchanged (still "callTool")
+    const firstAssistant = result[0];
+    expect(firstAssistant.tool_calls[0].function.name).toBe("callTool");
+
+    // Last assistant message should be rewritten
+    const lastAssistant = result[result.length - 1];
+    expect(lastAssistant.tool_calls[0].function.name).toBe("finalAnswer");
+  });
+
+  test("leaves non-callTool tool_calls unchanged", () => {
+    const messages: Message[] = [
+      {
+        role: "assistant",
+        content: null,
+        tool_calls: [
+          {
+            id: "tc_direct",
+            type: "function",
+            function: {
+              name: "inspectTools",
+              arguments: JSON.stringify({ patterns: ["*"] }),
+            },
+          },
+        ],
+      } as unknown as Message,
+    ];
+
+    const result = applyProcessor(messages);
+    const lastMsg = result[result.length - 1];
+
+    expect(lastMsg.tool_calls[0].function.name).toBe("inspectTools");
+  });
+
+  test("skips messages with no tool_calls", () => {
+    const messages: Message[] = [
+      { role: "user", content: "hello" } as Message,
+      { role: "assistant", content: "I will help" } as Message,
+    ];
+
+    const result = applyProcessor(messages);
+    expect(result[1].tool_calls).toBeUndefined();
+  });
+
+  test("mixed: callTool entries alongside direct tool calls are selectively unwrapped", () => {
+    const messages: Message[] = [
+      {
+        role: "assistant",
+        content: null,
+        tool_calls: [
+          {
+            id: "tc_direct",
+            type: "function",
+            function: {
+              name: "inspectTools",
+              arguments: JSON.stringify({}),
+            },
+          },
+          {
+            id: "tc_wrapped",
+            type: "function",
+            function: {
+              name: "callTool",
+              arguments: JSON.stringify({ name: "readFile", args: { filePath: "x.ts" } }),
+            },
+          },
+        ],
+      } as unknown as Message,
+    ];
+
+    const result = applyProcessor(messages);
+    const lastMsg = result[result.length - 1];
+
+    expect(lastMsg.tool_calls).toHaveLength(2);
+    // Direct call preserved
+    expect(lastMsg.tool_calls[0].function.name).toBe("inspectTools");
+    // Wrapped call unwrapped
+    expect(lastMsg.tool_calls[1].function.name).toBe("readFile");
+  });
+});

package/ts_build/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@tyvm/knowhow",
-    "version": "0.0.128",
+    "version": "0.0.130",
     "description": "ai cli with plugins and agents",
     "main": "ts_build/src/index.js",
     "bin": {
@@ -22,6 +22,7 @@
         "lint:deps": "depcheck --config=.depcheckrc",
         "lint:all": "npm run lint && npm run lint:deps",
         "check:model-pricing": "ts-node scripts/check-model-pricing.ts",
+        "debug:cache-miss": "ts-node scripts/cache-miss-debug.ts",
         "version:nightly": "bash scripts/publish.sh version:nightly",
         "version:dev": "bash scripts/publish.sh version:dev",
         "publish:nightly": "bash scripts/publish.sh nightly",
@@ -34,7 +35,6 @@
     "license": "MIT",
     "devDependencies": {
         "@babel/preset-typescript": "^7.23.3",
-        "typescript": "^5.0.0",
         "@types/diff": "^5.2.1",
         "@types/express": "^4.17.13",
         "@types/jest": "^29.5.13",
@@ -46,10 +46,11 @@
         "ts-jest": "^29.1.1",
         "ts-node": "^10.7.0",
         "tslint": "^6.1.3",
-        "tslint-config-prettier": "^1.18.0"
+        "tslint-config-prettier": "^1.18.0",
+        "typescript": "^5.0.0"
     },
     "dependencies": {
-        "@anthropic-ai/sdk": "^0.39.0",
+        "@anthropic-ai/sdk": "^0.111.0",
         "@google/genai": "^0.14.1",
         "@inquirer/editor": "^4.2.18",
         "@modelcontextprotocol/sdk": "^1.13.3",

package/ts_build/src/agents/base/base.d.ts

@@ -1,4 +1,4 @@
-import { CompletionResponse, GenericClient, Message, MessageContent, Tool, ToolCall } from "../../clients/types";
+import { CompletionResponse, CompletionOptions, GenericClient, Message, MessageContent, Tool, ToolCall } from "../../clients/types";
 import { IAgent } from "../interface";
 import { ToolsService } from "../../services/Tools";
 import { EventService } from "../../services/EventService";
@@ -8,6 +8,8 @@ export { Message, Tool, ToolCall };
 export interface ModelPreference {
     model: string;
     provider: keyof typeof Clients.clients;
+    reasoning_effort?: CompletionOptions["reasoning_effort"];
+    reasoning_summary?: boolean;
 }
 export interface AgentContext {
     Tools?: ToolsService;
@@ -38,6 +40,8 @@ export declare abstract class BaseAgent implements IAgent {
     protected turnCount: number;
     protected totalCostUsd: number;
     protected currentThread: number;
+    protected reasoningEffort: CompletionOptions["reasoning_effort"] | undefined;
+    protected summarizeReasoning: boolean | undefined;
     protected totalInputTokens: number;
     protected totalOutputTokens: number;
     protected totalCacheReadTokens: number;
@@ -94,6 +98,10 @@ export declare abstract class BaseAgent implements IAgent {
     setModel(value: string): void;
     getProvider(): string;
     setProvider(value: keyof typeof Clients.clients): void;
+    setReasoningEffort(effort: CompletionOptions["reasoning_effort"]): void;
+    getReasoningEffort(): CompletionOptions["reasoning_effort"];
+    setSummarizeReasoning(value: boolean): void;
+    getSummarizeReasoning(): boolean | undefined;
     getClient(): GenericClient;
     setClient(client: GenericClient): void;
     setEasyFinalAnswer(value: boolean): void;
@@ -109,13 +117,16 @@ export declare abstract class BaseAgent implements IAgent {
     getTurnCount(): number;
     adjustTotalCostUsd(cost: number): void;
     getTotalCostUsd(): number;
-    adjustTokenUsage(usage: any): void;
+    adjustTokenUsage(usage: any, messages?: Message[]): void;
     getTokenUsage(): {
         totalInputTokens: number;
         totalOutputTokens: number;
         totalCacheReadTokens: number;
         totalCacheWriteTokens: number;
     };
+    protected createAgentCompletion(params: Parameters<GenericClient["createChatCompletion"]>[0], options?: {
+        interruptValue?: CompletionResponse;
+    }): Promise<CompletionResponse>;
     startNewThread(messages: Message[]): void;
     updateCurrentThread(messages: Message[]): void;
     getThreads(): Message[][];