@tyvm/knowhow 0.0.48 → 0.0.49

package/tests/processors/TokenCompressor.test.ts

@@ -8,11 +8,11 @@ describe("TokenCompressor", () => {
 
   beforeEach(() => {
     mockToolsService = {
-      addTool: jest.fn(),
+      addTools: jest.fn(),
       addFunctions: jest.fn(),
       getTool: jest.fn(),
     } as any;
-    
+
     tokenCompressor = new TokenCompressor(mockToolsService);
   });
 
@@ -22,17 +22,16 @@ describe("TokenCompressor", () => {
 
   describe("constructor", () => {
     it("should register expandTokens tool with ToolsService", () => {
-      expect(mockToolsService.getTool).toHaveBeenCalledWith("expandTokens");
-      expect(mockToolsService.addTool).toHaveBeenCalled();
+      expect(mockToolsService.addTools).toHaveBeenCalled();
       expect(mockToolsService.addFunctions).toHaveBeenCalled();
     });
 
-    it("should not register tool if already exists", () => {
+    it("should overwrite tool if already exists", () => {
       mockToolsService.getTool.mockReturnValue({ type: "function", function: { name: "expandTokens" } } as any);
       const newCompressor = new TokenCompressor(mockToolsService);
-      
+
       // Should only be called once from the first instance
-      expect(mockToolsService.addTool).toHaveBeenCalledTimes(1);
+      expect(mockToolsService.addTools).toHaveBeenCalledTimes(2);
     });
 
     it("should work without ToolsService", () => {
@@ -50,7 +49,7 @@ describe("TokenCompressor", () => {
     it("should compress long strings using chunking", () => {
       const longContent = "x".repeat(40000); // Well above threshold (40k chars = 10k tokens > 8k maxTokens)
       const result = tokenCompressor.compressContent(longContent);
-      
+
       expect(result).toContain("[COMPRESSED_STRING");
       expect(result).toContain("Key:");
       expect(result).toContain("expandTokens");
@@ -64,7 +63,7 @@ describe("TokenCompressor", () => {
       };
       const content = JSON.stringify(largeJson);
       const result = tokenCompressor.compressContent(content);
-      
+
       // Should be parsed and compressed as JSON
       const parsed = JSON.parse(result);
       expect(typeof parsed.data).toBe("string");
@@ -79,7 +78,7 @@ describe("TokenCompressor", () => {
       };
       const content = JSON.stringify(complexJson);
       const result = tokenCompressor.compressContent(content);
-      
+
       const parsed = JSON.parse(result);
       expect(parsed.smallProp).toBe("small"); // Should remain unchanged
       expect(parsed.largeProp).toContain("[COMPRESSED_JSON_PROPERTY"); // Should be compressed
@@ -96,11 +95,11 @@ describe("TokenCompressor", () => {
     it("should create chunked compression for large strings", () => {
       const content = "x".repeat(20000);
       const result = tokenCompressor.compressStringInChunks(content);
-      
+
       expect(result).toContain("[COMPRESSED_STRING");
       expect(result).toContain("chunks");
       expect(result).toContain("Key:");
-      
+
       // Should have stored chunks in storage
       expect(tokenCompressor.getStorageSize()).toBeGreaterThan(0);
     });
@@ -108,10 +107,10 @@ describe("TokenCompressor", () => {
     it("should create chain of NEXT_CHUNK_KEY references", () => {
       const content = "a".repeat(20000);
       tokenCompressor.compressStringInChunks(content);
-      
+
       const keys = tokenCompressor.getStorageKeys();
       expect(keys.length).toBeGreaterThan(1);
-      
+
       // Check that chunks are linked
       const firstChunk = tokenCompressor.retrieveString(keys[0]);
       expect(firstChunk).toContain("NEXT_CHUNK_KEY");
@@ -123,7 +122,7 @@ describe("TokenCompressor", () => {
       // Create array that will definitely create chunks > 4000 chars (50 elements × 200 chars = 10k+ chars per potential chunk)
       const largeArray = new Array(15).fill("x".repeat(2200));
       const result = tokenCompressor.compressJsonProperties(largeArray);
-      
+
       // Should contain compression markers
       expect(JSON.stringify(result)).toContain("[COMPRESSED_JSON_ARRAY_CHUNK");
     });
@@ -136,7 +135,7 @@ describe("TokenCompressor", () => {
           }
         }
       };
-      
+
       const result = tokenCompressor.compressJsonProperties(nestedObj);
       expect(JSON.stringify(result)).toContain("[COMPRESSED_JSON");
     });
@@ -148,7 +147,7 @@ describe("TokenCompressor", () => {
         { data: "y".repeat(5000) }, // Large object
         "another small item"
       ];
-      
+
       const result = tokenCompressor.compressJsonProperties(mixedArray);
       expect(Array.isArray(result)).toBe(true);
     });
@@ -166,9 +165,9 @@ describe("TokenCompressor", () => {
         role: "user",
         content: "x".repeat(20000)
       };
-      
+
       await tokenCompressor.compressMessage(message);
-      
+
       expect(message.content).toContain("[COMPRESSED_STRING");
     });
 
@@ -180,12 +179,12 @@ describe("TokenCompressor", () => {
           { type: "image_url", image_url: { url: "http://example.com/img.jpg" } }
         ]
       };
-      
+
       await tokenCompressor.compressMessage(message);
-      
+
       const textContent = message.content[0] as { type: string; text: string };
       expect(textContent.text).toContain("[COMPRESSED_STRING");
-      
+
       // Non-text content should remain unchanged
       const imageContent = message.content[1] as { type: string; image_url: { url: string } };
       expect(imageContent.image_url.url).toBe("http://example.com/img.jpg");
@@ -204,9 +203,9 @@ describe("TokenCompressor", () => {
         { role: "user", content: "x".repeat(20000) },
         { role: "assistant", content: "short response" }
       ];
-      
+
       await processor([], messages);
-      
+
       expect(messages[0].content).toContain("[COMPRESSED_STRING");
       expect(messages[1].content).toBe("short response"); // Should remain short
     });
@@ -219,9 +218,9 @@ describe("TokenCompressor", () => {
         { role: "user", content: "x".repeat(20000) },
         { role: "assistant", content: "x".repeat(20000) }
       ];
-      
+
       await processor([], messages);
-      
+
       expect(messages[0].content).toContain("[COMPRESSED_STRING");
       expect(messages[1].content).not.toContain("[COMPRESSED_STRING");
     });
@@ -231,10 +230,10 @@ describe("TokenCompressor", () => {
     it("should store and retrieve strings", () => {
       const key = "test_key";
       const value = "test_value";
-      
+
       tokenCompressor.storeString(key, value);
       const retrieved = tokenCompressor.retrieveString(key);
-      
+
       expect(retrieved).toBe(value);
     });
 
@@ -246,9 +245,9 @@ describe("TokenCompressor", () => {
     it("should clear all storage", () => {
       tokenCompressor.storeString("key1", "value1");
       tokenCompressor.storeString("key2", "value2");
-      
+
       expect(tokenCompressor.getStorageSize()).toBe(2);
-      
+
       tokenCompressor.clearStorage();
       expect(tokenCompressor.getStorageSize()).toBe(0);
     });
@@ -256,7 +255,7 @@ describe("TokenCompressor", () => {
     it("should return storage keys", () => {
       tokenCompressor.storeString("key1", "value1");
       tokenCompressor.storeString("key2", "value2");
-      
+
       const keys = tokenCompressor.getStorageKeys();
       expect(keys).toContain("key1");
       expect(keys).toContain("key2");
@@ -266,23 +265,23 @@ describe("TokenCompressor", () => {
   describe("configuration", () => {
     it("should allow setting compression threshold", () => {
       tokenCompressor.setCompressionThreshold(1000);
-      
+
       const content = "x".repeat(5000); // Above new threshold (5000 chars = 1250 tokens > 1000)
       const result = tokenCompressor.compressContent(content);
-      
+
       expect(result).toContain("[COMPRESSED_STRING");
     });
 
     it("should adjust character limit with threshold", () => {
       const originalThreshold = 4000;
       const newThreshold = 2000;
-      
+
       tokenCompressor.setCompressionThreshold(newThreshold);
-      
+
       // Should compress at lower threshold
       const content = "x".repeat(9000); // Above new threshold (9000 chars = 2250 tokens > 2000)
       const result = tokenCompressor.compressContent(content);
-      
+
       expect(result).toContain("[COMPRESSED_STRING");
     });
   });
@@ -291,7 +290,7 @@ describe("TokenCompressor", () => {
     it("should register expandTokens function correctly", () => {
       const toolsServiceCalls = mockToolsService.addFunctions.mock.calls;
       expect(toolsServiceCalls.length).toBe(1);
-      
+
       const functions = toolsServiceCalls[0][0];
       expect(functions.expandTokens).toBeDefined();
       expect(typeof functions.expandTokens).toBe("function");
@@ -301,11 +300,11 @@ describe("TokenCompressor", () => {
       const key = "test_key";
       const value = "test_value";
       tokenCompressor.storeString(key, value);
-      
+
       const toolsServiceCalls = mockToolsService.addFunctions.mock.calls;
       const functions = toolsServiceCalls[0][0];
       const result = functions.expandTokens(key);
-      
+
       expect(result).toBe(value);
     });
 
@@ -313,7 +312,7 @@ describe("TokenCompressor", () => {
       const toolsServiceCalls = mockToolsService.addFunctions.mock.calls;
       const functions = toolsServiceCalls[0][0];
       const result = functions.expandTokens("non_existent");
-      
+
       expect(result).toContain("Error: No data found for key");
       expect(result).toContain("Available keys:");
     });
@@ -328,7 +327,7 @@ describe("TokenCompressor", () => {
     it("should handle malformed JSON gracefully", () => {
       const malformedJson = '{"incomplete": "json"'; // Missing closing brace
       const result = tokenCompressor.compressContent(malformedJson);
-      
+
       // Should treat as string, not JSON
       if (result.length > malformedJson.length) {
         expect(result).toContain("[COMPRESSED_STRING");
@@ -340,13 +339,13 @@ describe("TokenCompressor", () => {
     it("should handle very large objects without stack overflow", () => {
       const deepObject = { level: 0 } as any;
       let current = deepObject;
-      
+
       // Create deeply nested object
       for (let i = 1; i < 100; i++) {
         current.next = { level: i, data: "x".repeat(100) };
         current = current.next;
       }
-      
+
       expect(() => {
         tokenCompressor.compressJsonProperties(deepObject);
       }).not.toThrow();
@@ -355,11 +354,11 @@ describe("TokenCompressor", () => {
     it("should handle circular references in JSON", () => {
       const obj: any = { name: "test" };
       obj.self = obj; // Create circular reference
-      
+
       expect(() => {
         JSON.stringify(obj); // This should throw
       }).toThrow();
-      
+
       // Our compressor should handle this gracefully by not receiving circular JSON
       const safeObj = { name: "test", data: "x".repeat(10000) };
       expect(() => {
@@ -372,9 +371,9 @@ describe("TokenCompressor", () => {
     it("should handle large arrays efficiently", () => {
       const largeArray = new Array(1000).fill("x".repeat(100));
       const startTime = Date.now();
-      
+
       tokenCompressor.compressJsonProperties(largeArray);
-      
+
       const duration = Date.now() - startTime;
       expect(duration).toBeLessThan(5000); // Should complete within 5 seconds
     });
@@ -383,9 +382,9 @@ describe("TokenCompressor", () => {
       const content = "small content";
       const result1 = tokenCompressor.compressContent(content);
       const result2 = tokenCompressor.compressContent(result1);
-      
+
       expect(result1).toBe(content);
       expect(result2).toBe(content);
     });
   });
-});
+});