@tyvm/knowhow 0.0.48 → 0.0.49

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tyvm/knowhow",
-  "version": "0.0.48",
+  "version": "0.0.49",
   "description": "ai cli with plugins and agents",
   "main": "ts_build/src/index.js",
   "bin": {

package/src/agents/base/base.ts

@@ -31,6 +31,11 @@ export interface AgentContext {
   Clients?: AIClient;
 }
 
+export interface ToolCallEvent {
+  toolCall: ToolCall;
+  functionResp: any;
+}
+
 export abstract class BaseAgent implements IAgent {
   abstract name: string;
   abstract description: string;
@@ -62,7 +67,8 @@ export abstract class BaseAgent implements IAgent {
     newThread: "new_thread",
     threadUpdate: "thread_update",
     costUpdate: "cost_update",
-    toolUsed: "tool_used",
+    toolCall: "tool:pre_call",
+    toolUsed: "tool:post_call",
     done: "done",
     pause: "pause",
     kill: "kill",
@@ -294,6 +300,8 @@ export abstract class BaseAgent implements IAgent {
   abstract getInitialMessages(userInput: string): Promise<Message[]>;
 
   async processToolMessages(toolCall: ToolCall) {
+    this.agentEvents.emit(this.eventTypes.toolCall, { toolCall });
+
     const { functionResp, toolMessages } = await this.tools.callTool(
       toolCall,
       this.getEnabledToolNames()

package/src/chat/modules/AgentModule.ts

@@ -20,6 +20,7 @@ import {
 } from "../../processors/index";
 import { TaskInfo, ChatSession } from "../types";
 import { agents } from "../../agents";
+import { ToolCallEvent } from "src/agents/base/base";
 
 export class AgentModule extends BaseChatModule {
   name = "agent";
@@ -719,10 +720,28 @@ ${reason}
       ]);
 
       // Set up event listeners
+      if (!agent.agentEvents.listenerCount(agent.eventTypes.toolCall)) {
+        agent.agentEvents.on(
+          agent.eventTypes.toolCall,
+          (responseMsg: ToolCallEvent) => {
+            console.time(JSON.stringify(responseMsg.toolCall.function.name));
+            console.log(
+              ` 🔨 Tool: ${responseMsg.toolCall.function.name}\n Args: ${responseMsg.toolCall.function.arguments}\n`
+            );
+          }
+        );
+      }
       if (!agent.agentEvents.listenerCount(agent.eventTypes.toolUsed)) {
-        agent.agentEvents.on(agent.eventTypes.toolUsed, (responseMsg) => {
-          console.log(` 🔨 Tool used: ${JSON.stringify(responseMsg, null, 2)}`);
-        });
+        agent.agentEvents.on(
+          agent.eventTypes.toolUsed,
+          (responseMsg: ToolCallEvent) => {
+            console.timeEnd(JSON.stringify(responseMsg.toolCall.function.name));
+            console.log(
+              ` 🔨 Tool Response:
+              ${JSON.stringify(responseMsg.functionResp, null, 2)}`
+            );
+          }
+        );
       }
 
       const taskCompleted = new Promise<string>((resolve) => {

package/src/embeddings.ts

@@ -136,11 +136,10 @@ export async function embedSource(
     inputs = [source.input];
   }
 
-  console.log(`Found ${inputs.length} files`);
+  console.log(`Checking ${inputs.length} files`);
   if (inputs.length > 1000) {
     console.error("Large number of files detected. This may take a while");
   }
-  console.log(inputs);
   const embeddings: Embeddable[] = await loadEmbedding(source.output);
   let batch = [];
   let index = 0;
@@ -231,7 +230,6 @@ export async function embed(
     }
 
     if (alreadyEmbedded) {
-      console.log("Skipping", chunkId);
       continue;
     }
 
@@ -468,7 +466,7 @@ export async function queryEmbedding<E>(
   model = EmbeddingModels.openai.EmbeddingAda2
 ) {
   const providerEmbeddings = await Clients.createEmbedding("", {
-    input: takeFirstNWords(query, 5000),
+    input: takeFirstNWords(query, 5000).slice(0, 16000),
     model,
   });
   const queryVector = providerEmbeddings.data[0].embedding;

package/src/processors/CustomVariables.ts

@@ -101,18 +101,24 @@ export class CustomVariables {
    */
   private listVariables(): string {
     const variableNames = Object.keys(this.variables);
-    
+
     if (variableNames.length === 0) {
       return "No variables are currently stored.";
     }
 
-    const variableList = variableNames.map(name => {
-      const value = this.variables[name];
-      const preview = typeof value === "string" 
-        ? (value.length > 50 ? value.substring(0, 50) + "..." : value)
-        : JSON.stringify(value).substring(0, 50) + (JSON.stringify(value).length > 50 ? "..." : "");
-      return `- ${name}: ${preview}`;
-    }).join("\n");
+    const variableList = variableNames
+      .map((name) => {
+        const value = this.variables[name];
+        const preview =
+          typeof value === "string"
+            ? value.length > 50
+              ? value.substring(0, 50) + "..."
+              : value
+            : JSON.stringify(value).substring(0, 50) +
+              (JSON.stringify(value).length > 50 ? "..." : "");
+        return `- ${name}: ${preview}`;
+      })
+      .join("\n");
 
     return `Currently stored variables (${variableNames.length}):\n${variableList}`;
   }
@@ -146,17 +152,17 @@ export class CustomVariables {
       // Check if ALL variables are undefined - if so, return just the error message for the first one
       const variableMatches = value.match(/\{\{([a-zA-Z0-9_]+)\}\}/g);
       if (variableMatches) {
-        const allUndefined = variableMatches.every(match => {
-          const varName = match.replace(/[{}]/g, '');
+        const allUndefined = variableMatches.every((match) => {
+          const varName = match.replace(/[{}]/g, "");
           return !(varName in this.variables);
         });
-        
+
         if (allUndefined && variableMatches.length > 0) {
-          const firstUndefinedVar = variableMatches[0].replace(/[{}]/g, '');
+          const firstUndefinedVar = variableMatches[0].replace(/[{}]/g, "");
           return `{{ERROR: Variable "${firstUndefinedVar}" is not defined}}`;
         }
       }
-      
+
       // Otherwise, proceed with partial substitution
       return value.replace(/\{\{([a-zA-Z0-9_]+)\}\}/g, (match, varName) => {
         // Prevent infinite recursion
@@ -251,62 +257,34 @@ export class CustomVariables {
    */
   private registerTools(toolsService: ToolsService): void {
     // Register setVariable tool
-    if (!toolsService.getTool(this.setVariableToolName)) {
-      toolsService.addTool(setVariableToolDefinition);
-      toolsService.addFunctions({
-        [this.setVariableToolName]: (name: string, contents: any) => {
-          return this.setVariable(name, contents);
-        },
-      });
-    }
-
-    // Register getVariable tool
-    if (!toolsService.getTool(this.getVariableToolName)) {
-      toolsService.addTool(getVariableToolDefinition);
-      toolsService.addFunctions({
-        [this.getVariableToolName]: (varName: string) => {
-          return this.getVariable(varName);
-        },
-      });
-    }
-
-    // Register storeToolCallToVariable tool
-    if (!toolsService.getTool(this.storeToolCallToolName)) {
-      toolsService.addTool(storeToolCallToVariableDefinition);
-      toolsService.addFunctions({
-        [this.storeToolCallToolName]: async (
-          varName: string,
-          toolName: string,
-          toolArgs: string
-        ) => {
-          return await this.storeToolCallToVariable(
-            varName,
-            toolName,
-            toolArgs
-          );
-        },
-      });
-    }
-
-    // Register listVariables tool
-    if (!toolsService.getTool(this.listVariablesToolName)) {
-      toolsService.addTool(listVariablesToolDefinition);
-      toolsService.addFunctions({
-        [this.listVariablesToolName]: () => {
-          return this.listVariables();
-        },
-      });
-    }
-
-    // Register deleteVariable tool
-    if (!toolsService.getTool(this.deleteVariableToolName)) {
-      toolsService.addTool(deleteVariableToolDefinition);
-      toolsService.addFunctions({
-        [this.deleteVariableToolName]: (varName: string) => {
-          return this.deleteVariable(varName);
-        },
-      });
-    }
+    toolsService.addTools([
+      setVariableToolDefinition,
+      getVariableToolDefinition,
+      storeToolCallToVariableDefinition,
+      listVariablesToolDefinition,
+      deleteVariableToolDefinition,
+    ]);
+    toolsService.addFunctions({
+      [this.setVariableToolName]: (name: string, contents: any) => {
+        return this.setVariable(name, contents);
+      },
+      [this.getVariableToolName]: (varName: string) => {
+        return this.getVariable(varName);
+      },
+      [this.storeToolCallToolName]: async (
+        varName: string,
+        toolName: string,
+        toolArgs: string
+      ) => {
+        return await this.storeToolCallToVariable(varName, toolName, toolArgs);
+      },
+      [this.listVariablesToolName]: () => {
+        return this.listVariables();
+      },
+      [this.deleteVariableToolName]: (varName: string) => {
+        return this.deleteVariable(varName);
+      },
+    });
   }
 
   /**
@@ -434,4 +412,4 @@ export const deleteVariableToolDefinition: Tool = {
       required: ["varName"],
     },
   },
-};
+};

package/src/processors/TokenCompressor.ts

@@ -116,8 +116,9 @@ export class TokenCompressor {
 
     // For nested properties (path !== ""), use maxTokens to avoid recompressing stored data
     // For top-level content (path === ""), use compressionThreshold to determine compression
-    const thresholdToUse = path === "" ? this.compressionThreshold : this.maxTokens;
-    
+    const thresholdToUse =
+      path === "" ? this.compressionThreshold : this.maxTokens;
+
     if (tokens <= thresholdToUse) {
       return content;
     }
@@ -320,8 +321,8 @@ export class TokenCompressor {
   }
 
   registerTool(toolsService?: ToolsService): void {
-    if (toolsService && !toolsService.getTool(this.toolName)) {
-      toolsService.addTool(expandTokensDefinition);
+    if (toolsService) {
+      toolsService.addTools([expandTokensDefinition]);
       toolsService.addFunctions({
         [this.toolName]: (key: string) => {
           const data = this.retrieveString(key);

package/src/processors/ToolResponseCache.ts

@@ -130,7 +130,6 @@ export class ToolResponseCache {
     }
 
     try {
-      
       // First parse the stored string as JSON, then handle nested JSON strings
       const jsonData = this.tryParseJson(data);
       if (!jsonData) {
@@ -207,14 +206,12 @@ export class ToolResponseCache {
    * Registers the jqToolResponse tool with the ToolsService
    */
   registerTool(toolsService: ToolsService): void {
-    if (!toolsService.getTool(this.toolName)) {
-      toolsService.addTool(jqToolResponseDefinition);
-      toolsService.addFunctions({
-        [this.toolName]: async (toolCallId: string, jqQuery: string) => {
-          return await this.queryToolResponse(toolCallId, jqQuery);
-        },
-      });
-    }
+    toolsService.addTools([jqToolResponseDefinition]);
+    toolsService.addFunctions({
+      [this.toolName]: async (toolCallId: string, jqQuery: string) => {
+        return await this.queryToolResponse(toolCallId, jqQuery);
+      },
+    });
   }
 }
 

package/tests/processors/CustomVariables.test.ts

@@ -8,7 +8,7 @@ describe("CustomVariables", () => {
 
   beforeEach(() => {
     mockToolsService = {
-      addTool: jest.fn(),
+      addTools: jest.fn(),
       addFunctions: jest.fn(),
       getTool: jest.fn().mockReturnValue(undefined),
       callTool: jest.fn(),
@@ -23,12 +23,12 @@ describe("CustomVariables", () => {
 
   describe("constructor", () => {
     it("should register all variable tools with ToolsService", () => {
-      expect(mockToolsService.addTool).toHaveBeenCalledTimes(5);
-      expect(mockToolsService.addFunctions).toHaveBeenCalledTimes(5);
+      expect(mockToolsService.addTools).toHaveBeenCalledTimes(1);
+      expect(mockToolsService.addFunctions).toHaveBeenCalledTimes(1);
 
       // Verify all tools are registered
-      const addToolCalls = mockToolsService.addTool.mock.calls;
-      const toolNames = addToolCalls.map(call => call[0].function.name);
+      const addToolCalls = mockToolsService.addTools.mock.calls;
+      const toolNames = addToolCalls[0][0].map((args) => args.function.name);
 
       expect(toolNames).toContain("setVariable");
       expect(toolNames).toContain("getVariable");
@@ -37,12 +37,15 @@ describe("CustomVariables", () => {
       expect(toolNames).toContain("deleteVariable");
     });
 
-    it("should not register tools if they already exist", () => {
-      mockToolsService.getTool.mockReturnValue({ type: "function", function: { name: "setVariable" } } as any);
+    it("should overwrite tools if they already exist", () => {
+      mockToolsService.getTool.mockReturnValue({
+        type: "function",
+        function: { name: "setVariable" },
+      } as any);
       const newCustomVariables = new CustomVariables(mockToolsService);
 
-      // Should still be called only from the first instance (5 times)
-      expect(mockToolsService.addTool).toHaveBeenCalledTimes(5);
+      // Should still be called once per instance creation
+      expect(mockToolsService.addTools).toHaveBeenCalledTimes(2);
     });
   });
 
@@ -51,7 +54,9 @@ describe("CustomVariables", () => {
 
     beforeEach(() => {
       const addFunctionsCalls = mockToolsService.addFunctions.mock.calls;
-      const setVariableCall = addFunctionsCalls.find(call => call[0].setVariable);
+      const setVariableCall = addFunctionsCalls.find(
+        (call) => call[0].setVariable
+      );
       setVariableFunction = setVariableCall[0].setVariable;
     });
 
@@ -86,7 +91,9 @@ describe("CustomVariables", () => {
 
     beforeEach(() => {
       const addFunctionsCalls = mockToolsService.addFunctions.mock.calls;
-      const setVariableCall = addFunctionsCalls.find(call => call[0].setVariable);
+      const setVariableCall = addFunctionsCalls.find(
+        (call) => call[0].setVariable
+      );
       setVariableFunction = setVariableCall[0].setVariable;
     });
 
@@ -122,8 +129,12 @@ describe("CustomVariables", () => {
 
     beforeEach(() => {
       const addFunctionsCalls = mockToolsService.addFunctions.mock.calls;
-      const setVariableCall = addFunctionsCalls.find(call => call[0].setVariable);
-      const getVariableCall = addFunctionsCalls.find(call => call[0].getVariable);
+      const setVariableCall = addFunctionsCalls.find(
+        (call) => call[0].setVariable
+      );
+      const getVariableCall = addFunctionsCalls.find(
+        (call) => call[0].getVariable
+      );
       setVariableFunction = setVariableCall[0].setVariable;
       getVariableFunction = getVariableCall[0].getVariable;
     });
@@ -166,8 +177,12 @@ describe("CustomVariables", () => {
 
     beforeEach(() => {
       const addFunctionsCalls = mockToolsService.addFunctions.mock.calls;
-      const setVariableCall = addFunctionsCalls.find(call => call[0].setVariable);
-      const listVariablesCall = addFunctionsCalls.find(call => call[0].listVariables);
+      const setVariableCall = addFunctionsCalls.find(
+        (call) => call[0].setVariable
+      );
+      const listVariablesCall = addFunctionsCalls.find(
+        (call) => call[0].listVariables
+      );
       setVariableFunction = setVariableCall[0].setVariable;
       listVariablesFunction = listVariablesCall[0].listVariables;
     });
@@ -204,8 +219,12 @@ describe("CustomVariables", () => {
 
     beforeEach(() => {
       const addFunctionsCalls = mockToolsService.addFunctions.mock.calls;
-      const setVariableCall = addFunctionsCalls.find(call => call[0].setVariable);
-      const deleteVariableCall = addFunctionsCalls.find(call => call[0].deleteVariable);
+      const setVariableCall = addFunctionsCalls.find(
+        (call) => call[0].setVariable
+      );
+      const deleteVariableCall = addFunctionsCalls.find(
+        (call) => call[0].deleteVariable
+      );
       setVariableFunction = setVariableCall[0].setVariable;
       deleteVariableFunction = deleteVariableCall[0].deleteVariable;
     });
@@ -213,7 +232,9 @@ describe("CustomVariables", () => {
     it("should delete existing variables", () => {
       setVariableFunction("testVar", "test value");
       const result = deleteVariableFunction("testVar");
-      expect(result).toContain('Variable "testVar" has been deleted successfully');
+      expect(result).toContain(
+        'Variable "testVar" has been deleted successfully'
+      );
       expect(customVariables.getVariableNames()).not.toContain("testVar");
     });
 
@@ -229,11 +250,17 @@ describe("CustomVariables", () => {
   });
 
   describe("storeToolCallToVariable functionality", () => {
-    let storeToolCallFunction: (varName: string, toolName: string, toolArgs: string) => Promise<string>;
+    let storeToolCallFunction: (
+      varName: string,
+      toolName: string,
+      toolArgs: string
+    ) => Promise<string>;
 
     beforeEach(() => {
       const addFunctionsCalls = mockToolsService.addFunctions.mock.calls;
-      const storeToolCallCall = addFunctionsCalls.find(call => call[0].storeToolCallToVariable);
+      const storeToolCallCall = addFunctionsCalls.find(
+        (call) => call[0].storeToolCallToVariable
+      );
       storeToolCallFunction = storeToolCallCall[0].storeToolCallToVariable;
     });
 
@@ -243,38 +270,56 @@ describe("CustomVariables", () => {
         toolCallId: "test-call-id",
         functionName: "testTool",
         functionArgs: { param1: "value1" },
-        functionResp: { success: true, data: "test data" }
+        functionResp: { success: true, data: "test data" },
       };
       mockToolsService.callTool.mockResolvedValue(mockResult);
 
-      const result = await storeToolCallFunction("resultVar", "testTool", '{"param1": "value1"}');
+      const result = await storeToolCallFunction(
+        "resultVar",
+        "testTool",
+        '{"param1": "value1"}'
+      );
 
-      expect(result).toContain('Tool call result for "testTool" has been stored in variable "resultVar"');
+      expect(result).toContain(
+        'Tool call result for "testTool" has been stored in variable "resultVar"'
+      );
       expect(mockToolsService.callTool).toHaveBeenCalledWith({
         id: expect.any(String),
         type: "function",
         function: {
           name: "testTool",
-          arguments: { param1: "value1" }
-        }
+          arguments: { param1: "value1" },
+        },
       });
     });
 
     it("should return error for invalid JSON arguments", async () => {
-      const result = await storeToolCallFunction("resultVar", "testTool", "invalid json");
+      const result = await storeToolCallFunction(
+        "resultVar",
+        "testTool",
+        "invalid json"
+      );
       expect(result).toContain("Error: Invalid JSON in toolArgs parameter");
     });
 
     it("should return error for invalid variable names", async () => {
-      const result = await storeToolCallFunction("invalid-name", "testTool", "{}");
+      const result = await storeToolCallFunction(
+        "invalid-name",
+        "testTool",
+        "{}"
+      );
       expect(result).toContain("Error: Invalid variable name");
     });
 
     it("should handle tool execution errors", async () => {
-      mockToolsService.callTool.mockRejectedValue(new Error("Tool execution failed"));
+      mockToolsService.callTool.mockRejectedValue(
+        new Error("Tool execution failed")
+      );
 
       const result = await storeToolCallFunction("resultVar", "testTool", "{}");
-      expect(result).toContain("Error storing tool call result: Tool execution failed");
+      expect(result).toContain(
+        "Error storing tool call result: Tool execution failed"
+      );
     });
   });
 
@@ -283,7 +328,9 @@ describe("CustomVariables", () => {
 
     beforeEach(() => {
       const addFunctionsCalls = mockToolsService.addFunctions.mock.calls;
-      const setVariableCall = addFunctionsCalls.find(call => call[0].setVariable);
+      const setVariableCall = addFunctionsCalls.find(
+        (call) => call[0].setVariable
+      );
       setVariableFunction = setVariableCall[0].setVariable;
     });
 
@@ -294,14 +341,16 @@ describe("CustomVariables", () => {
       const messages = [
         {
           role: "user" as const,
-          content: "{{greeting}} {{userName}}, how are you today?"
-        }
+          content: "{{greeting}} {{userName}}, how are you today?",
+        },
       ];
 
       const processor = customVariables.createProcessor();
       const modifiedMessages = [...messages];
       await processor(messages, modifiedMessages);
-      expect(modifiedMessages[0].content).toBe("Hello Alice, how are you today?");
+      expect(modifiedMessages[0].content).toBe(
+        "Hello Alice, how are you today?"
+      );
     });
 
     it("should handle multiple substitutions in single message", async () => {
@@ -312,8 +361,8 @@ describe("CustomVariables", () => {
       const messages = [
         {
           role: "user" as const,
-          content: "{{var1}} and {{var2}} and {{var3}}"
-        }
+          content: "{{var1}} and {{var2}} and {{var3}}",
+        },
       ];
 
       const processor = customVariables.createProcessor();
@@ -328,30 +377,34 @@ describe("CustomVariables", () => {
       const messages = [
         {
           role: "user" as const,
-          content: "Configuration: {{config}}"
-        }
+          content: "Configuration: {{config}}",
+        },
       ];
 
       const processor = customVariables.createProcessor();
       const modifiedMessages = [...messages];
       await processor(messages, modifiedMessages);
-      
-      expect(modifiedMessages[0].content).toBe('Configuration: {"api":"v1","timeout":5000}');
+
+      expect(modifiedMessages[0].content).toBe(
+        'Configuration: {"api":"v1","timeout":5000}'
+      );
     });
 
     it("should return error for undefined variables", async () => {
       const messages = [
         {
           role: "user" as const,
-          content: "Hello {{undefinedVar}}"
-        }
+          content: "Hello {{undefinedVar}}",
+        },
       ];
 
       const processor = customVariables.createProcessor();
       const modifiedMessages = [...messages];
       await processor(messages, modifiedMessages);
-      
-      expect(modifiedMessages[0].content).toBe('{{ERROR: Variable "undefinedVar" is not defined}}');
+
+      expect(modifiedMessages[0].content).toBe(
+        '{{ERROR: Variable "undefinedVar" is not defined}}'
+      );
     });
 
     it("should handle partial substitutions with mixed defined/undefined vars", async () => {
@@ -360,15 +413,17 @@ describe("CustomVariables", () => {
       const messages = [
         {
           role: "user" as const,
-          content: "{{defined}} and {{undefined}}"
-        }
+          content: "{{defined}} and {{undefined}}",
+        },
       ];
 
       const processor = customVariables.createProcessor();
       const modifiedMessages = [...messages];
       await processor(messages, modifiedMessages);
-      
-      expect(modifiedMessages[0].content).toBe('value and {{ERROR: Variable "undefined" is not defined}}');
+
+      expect(modifiedMessages[0].content).toBe(
+        'value and {{ERROR: Variable "undefined" is not defined}}'
+      );
     });
 
     it("should preserve message structure while substituting content", async () => {
@@ -378,18 +433,18 @@ describe("CustomVariables", () => {
         {
           role: "assistant" as const,
           content: "Original {{test}} content",
-          metadata: { id: "test-id" }
-        }
+          metadata: { id: "test-id" },
+        },
       ];
 
       const processor = customVariables.createProcessor();
       const modifiedMessages = [...messages];
       await processor(messages, modifiedMessages);
-      
+
       expect(modifiedMessages[0]).toEqual({
         role: "assistant",
         content: "Original replaced content",
-        metadata: { id: "test-id" }
+        metadata: { id: "test-id" },
       });
     });
 
@@ -400,8 +455,8 @@ describe("CustomVariables", () => {
       const messages = [
         {
           role: "user" as const,
-          content: "{{outerVar}} value"
-        }
+          content: "{{outerVar}} value",
+        },
       ];
 
       const processor = customVariables.createProcessor();
@@ -417,8 +472,8 @@ describe("CustomVariables", () => {
       const messages = [
         {
           role: "user" as const,
-          content: "Empty: '{{empty}}' Spaces: '{{spaces}}'"
-        }
+          content: "Empty: '{{empty}}' Spaces: '{{spaces}}'",
+        },
       ];
 
       const processor = customVariables.createProcessor();