@tyvm/knowhow 0.0.109 → 0.0.110

package/tests/processors/Base64ImageDetector.test.ts

@@ -594,3 +594,163 @@ describe("Base64ImageDetector", () => {
     });
   });
 });
+
+describe("Base64ImageDetector - image path hint detection", () => {
+  let detector: Base64ImageProcessor;
+  let processor: ReturnType<Base64ImageProcessor["createProcessor"]>;
+
+  beforeEach(() => {
+    detector = new Base64ImageProcessor();
+    processor = detector.createProcessor();
+  });
+
+  /**
+   * Simulates the actual output from the Playwright MCP browser_take_screenshot tool.
+   * When an agent calls browser_take_screenshot, the tool message content looks like:
+   *
+   *   ### Result
+   *   - [Screenshot of viewport](./hackernews-screenshot.png)
+   *   ### Ran Playwright code
+   *   ...
+   *
+   * The Base64ImageDetector should detect the .png path in this text and add a hint
+   * telling the model it can call loadImageAsBase64 to actually view the image.
+   */
+  describe("browser screenshot tool response", () => {
+    it("should detect image path from a browser screenshot tool message and add hint", () => {
+      // This is the exact format returned by the browser MCP take_screenshot tool
+      const screenshotToolResponse =
+        "### Result\n- [Screenshot of viewport](./hackernews-screenshot.png)\n### Ran Playwright code\n```js\nawait page.screenshot({ path: './hackernews-screenshot.png', scale: 'css', type: 'png' });\n```";
+
+      const originalMessages: Message[] = [];
+      const modifiedMessages: Message[] = [
+        {
+          role: "tool",
+          content: screenshotToolResponse,
+          tool_call_id: "call_abc123",
+        },
+      ];
+
+      processor(originalMessages, modifiedMessages);
+
+      const content = modifiedMessages[0].content as string;
+      expect(typeof content).toBe("string");
+      // Should still contain the original text
+      expect(content).toContain("Screenshot of viewport");
+      expect(content).toContain("./hackernews-screenshot.png");
+      // Should contain the hint
+      expect(content).toContain("[TIP:");
+      expect(content).toContain("loadImageAsBase64");
+      expect(content).toContain("./hackernews-screenshot.png");
+    });
+
+    it("should include the exact file path in the hint", () => {
+      const screenshotPath = "./hackernews-screenshot.png";
+      const toolResponse = `### Result\n- [Screenshot of viewport](${screenshotPath})\n`;
+
+      const originalMessages: Message[] = [];
+      const modifiedMessages: Message[] = [
+        {
+          role: "tool",
+          content: toolResponse,
+          tool_call_id: "call_xyz789",
+        },
+      ];
+
+      processor(originalMessages, modifiedMessages);
+
+      const content = modifiedMessages[0].content as string;
+      // The hint should reference the exact path
+      expect(content).toContain(`loadImageAsBase64("${screenshotPath}")`);
+    });
+
+    it("should not add hint when message contains no image paths", () => {
+      const toolResponse = "The page has been loaded successfully.";
+
+      const originalMessages: Message[] = [];
+      const modifiedMessages: Message[] = [
+        {
+          role: "tool",
+          content: toolResponse,
+          tool_call_id: "call_noimages",
+        },
+      ];
+
+      processor(originalMessages, modifiedMessages);
+
+      const content = modifiedMessages[0].content as string;
+      expect(content).toBe(toolResponse);
+      expect(content).not.toContain("[TIP:");
+    });
+
+    it("should detect absolute paths like /tmp/page-123.png", () => {
+      const toolResponse =
+        "Screenshot saved to /tmp/page-2026-01-01T12-00-00.png for review.";
+
+      const originalMessages: Message[] = [];
+      const modifiedMessages: Message[] = [
+        {
+          role: "tool",
+          content: toolResponse,
+          tool_call_id: "call_abs",
+        },
+      ];
+
+      processor(originalMessages, modifiedMessages);
+
+      const content = modifiedMessages[0].content as string;
+      expect(content).toContain("[TIP:");
+      expect(content).toContain("loadImageAsBase64");
+      expect(content).toContain("/tmp/page-2026-01-01T12-00-00.png");
+    });
+
+    it("should detect multiple image paths and hint about all of them", () => {
+      const toolResponse =
+        "Before: ./before.png\nAfter: ./after.jpg\nDiff: ./diff.png";
+
+      const originalMessages: Message[] = [];
+      const modifiedMessages: Message[] = [
+        {
+          role: "tool",
+          content: toolResponse,
+          tool_call_id: "call_multi",
+        },
+      ];
+
+      processor(originalMessages, modifiedMessages);
+
+      const content = modifiedMessages[0].content as string;
+      expect(content).toContain("[TIP:");
+      expect(content).toContain("./before.png");
+      expect(content).toContain("./after.jpg");
+      expect(content).toContain("./diff.png");
+    });
+
+    it("should not add hint to messages that are actual base64 image data (already converted)", () => {
+      const validPngBase64 =
+        "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==";
+
+      const originalMessages: Message[] = [];
+      const modifiedMessages: Message[] = [
+        {
+          role: "tool",
+          content: validPngBase64,
+          tool_call_id: "call_base64",
+        },
+      ];
+
+      processor(originalMessages, modifiedMessages);
+
+      // Should be converted to image array, not get a text hint
+      const content = modifiedMessages[0].content;
+      if (Array.isArray(content)) {
+        // Good - was converted to image content, no hint needed
+        expect(content[0]).toHaveProperty("type", "image_url");
+      } else {
+        // If kept as string, the hint should NOT be about a file path
+        // because base64 data URLs don't contain file paths
+        expect(content as string).not.toMatch(/loadImageAsBase64\("data:/);
+      }
+    });
+  });
+});

package/tests/unit/clients/AIClient.test.ts

@@ -0,0 +1,446 @@
+/**
+ * Integration tests for AIClient — verifies that retry, timeout, and
+ * AbortSignal options flow correctly through AIClient into the underlying
+ * GenericClient mock.
+ *
+ * We bypass all real provider initialisation by calling:
+ *   aiClient.registerClient(provider, mockClient)
+ *   aiClient.registerModels(provider, [model])
+ */
+
+// Prevent real _initDefaultProviders from firing (it reads env vars / files)
+jest.mock("../../../src/config", () => ({
+  getConfig: jest.fn().mockResolvedValue({ modules: [] }),
+  getGlobalConfig: jest.fn().mockResolvedValue({ modules: [] }),
+  getConfigSync: jest.fn().mockReturnValue({}),
+}));
+jest.mock("../../../src/services/KnowhowClient", () => ({
+  loadKnowhowJwt: jest.fn().mockReturnValue(null),
+  KNOWHOW_API_URL: "https://mock.local",
+}));
+
+import { AIClient } from "../../../src/clients/index";
+import type { GenericClient } from "../../../src/clients/types";
+
+// ─── Helpers ────────────────────────────────────────────────────────────────
+
+/** Build a minimal mock CompletionResponse */
+const mockCompletion = () => ({
+  choices: [{ message: { role: "assistant" as const, content: "hello" } }],
+  model: "mock-model",
+  usage: { prompt_tokens: 10, completion_tokens: 5 },
+});
+
+/** Build a minimal mock ImageGenerationResponse */
+const mockImage = () => ({
+  created: Date.now(),
+  data: [{ url: "https://mock.local/image.png" }],
+});
+
+/** Build a minimal mock EmbeddingResponse */
+const mockEmbedding = () => ({
+  data: [{ object: "embedding", embedding: [0.1, 0.2], index: 0 }],
+  model: "mock-embed",
+  usage: { prompt_tokens: 5, total_tokens: 5 },
+});
+
+/** Build a minimal mock AudioGenerationResponse */
+const mockAudio = () => ({
+  audio: Buffer.from("fake-audio"),
+  format: "mp3",
+});
+
+/**
+ * Create an AIClient with a registered mock provider.
+ * Returns the AIClient and the mocked GenericClient.
+ */
+function setupClient(overrides: Partial<GenericClient> = {}) {
+  const mockGenericClient: GenericClient = {
+    setKey: jest.fn(),
+    createChatCompletion: jest
+      .fn()
+      .mockResolvedValue(mockCompletion()),
+    createEmbedding: jest.fn().mockResolvedValue(mockEmbedding()),
+    createImageGeneration: jest.fn().mockResolvedValue(mockImage()),
+    createAudioGeneration: jest.fn().mockResolvedValue(mockAudio()),
+    createAudioTranscription: jest
+      .fn()
+      .mockResolvedValue({ text: "transcribed" }),
+    createVideoGeneration: jest.fn().mockResolvedValue({
+      created: Date.now(),
+      data: [{ url: "https://mock.local/video.mp4" }],
+    }),
+    getModels: jest.fn().mockResolvedValue([]),
+    ...overrides,
+  };
+
+  const aiClient = new AIClient();
+  // Register our mock bypassing all env/network checks
+  aiClient.registerClient("mock", mockGenericClient);
+  aiClient.registerModels("mock", ["mock-model", "mock-embed"]);
+
+  return { aiClient, mockGenericClient };
+}
+
+// ─── Tests ──────────────────────────────────────────────────────────────────
+
+describe("AIClient — retry / timeout / AbortSignal", () => {
+  afterEach(() => {
+    jest.useRealTimers();
+  });
+
+  // ── createCompletion ──────────────────────────────────────────────────────
+
+  describe("createCompletion", () => {
+    it("returns a completion on success", async () => {
+      const { aiClient } = setupClient();
+      const result = await aiClient.createCompletion("mock", {
+        model: "mock-model",
+        messages: [{ role: "user", content: "hi" }],
+      });
+      expect(result.choices[0].message.content).toBe("hello");
+    });
+
+    it("forwards the AbortSignal to createChatCompletion", async () => {
+      const { aiClient, mockGenericClient } = setupClient();
+      const controller = new AbortController();
+
+      await aiClient.createCompletion("mock", {
+        model: "mock-model",
+        messages: [],
+        signal: controller.signal,
+      });
+
+      const callArgs = (mockGenericClient.createChatCompletion as jest.Mock)
+        .mock.calls[0][0];
+      expect(callArgs.signal).toBeInstanceOf(AbortSignal);
+    });
+
+    it("retries on 429 and succeeds", async () => {
+      jest.useFakeTimers();
+      const { aiClient, mockGenericClient } = setupClient({
+        createChatCompletion: jest
+          .fn()
+          .mockRejectedValueOnce(new Error("429 rate limited"))
+          .mockResolvedValueOnce(mockCompletion()),
+      });
+
+      const promise = aiClient.createCompletion("mock", {
+        model: "mock-model",
+        messages: [],
+        maxRetries: 2,
+        backoffMs: 50,
+      });
+      await jest.runAllTimersAsync();
+      const result = await promise;
+
+      expect(result.choices[0].message.content).toBe("hello");
+      expect(mockGenericClient.createChatCompletion).toHaveBeenCalledTimes(2);
+    });
+
+    it("aborts immediately when external signal is pre-aborted", async () => {
+      const controller = new AbortController();
+      controller.abort();
+
+      const { aiClient, mockGenericClient } = setupClient();
+      await expect(
+        aiClient.createCompletion("mock", {
+          model: "mock-model",
+          messages: [],
+          signal: controller.signal,
+        })
+      ).rejects.toMatchObject({ name: "AbortError" });
+
+      expect(mockGenericClient.createChatCompletion).not.toHaveBeenCalled();
+    });
+
+    it("cancels in-flight request when external signal is aborted", async () => {
+      const controller = new AbortController();
+      const { aiClient, mockGenericClient } = setupClient({
+        createChatCompletion: jest.fn().mockImplementation((opts: any) => {
+          return new Promise((_, reject) => {
+            opts.signal?.addEventListener("abort", () =>
+              reject(opts.signal.reason)
+            );
+          });
+        }),
+      });
+
+      const promise = aiClient.createCompletion("mock", {
+        model: "mock-model",
+        messages: [],
+        signal: controller.signal,
+      });
+
+      setImmediate(() =>
+        controller.abort(new DOMException("User cancelled", "AbortError"))
+      );
+
+      await expect(promise).rejects.toMatchObject({ name: "AbortError" });
+      expect(mockGenericClient.createChatCompletion).toHaveBeenCalledTimes(1);
+    });
+
+    it("times out per-attempt and retries", async () => {
+      jest.useFakeTimers();
+      const { aiClient, mockGenericClient } = setupClient({
+        createChatCompletion: jest
+          .fn()
+          .mockImplementationOnce((opts: any) => {
+            return new Promise((_, reject) => {
+              opts.signal?.addEventListener("abort", () =>
+                reject(opts.signal.reason)
+              );
+            });
+          })
+          .mockResolvedValueOnce(mockCompletion()),
+      });
+
+      const promise = aiClient.createCompletion("mock", {
+        model: "mock-model",
+        messages: [],
+        timeout: 1000,
+        maxRetries: 2,
+        backoffMs: 10,
+      });
+      await jest.runAllTimersAsync();
+      const result = await promise;
+
+      expect(result.choices[0].message.content).toBe("hello");
+      expect(mockGenericClient.createChatCompletion).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  // ── createEmbedding ───────────────────────────────────────────────────────
+
+  describe("createEmbedding", () => {
+    it("forwards the AbortSignal to createEmbedding on the client", async () => {
+      const { aiClient, mockGenericClient } = setupClient();
+      const controller = new AbortController();
+
+      await aiClient.createEmbedding("mock", {
+        input: "test text",
+        model: "mock-embed",
+        signal: controller.signal,
+      });
+
+      const callArgs = (mockGenericClient.createEmbedding as jest.Mock).mock
+        .calls[0][0];
+      expect(callArgs.signal).toBeInstanceOf(AbortSignal);
+    });
+
+    it("retries on 500 and succeeds", async () => {
+      jest.useFakeTimers();
+      const { aiClient, mockGenericClient } = setupClient({
+        createEmbedding: jest
+          .fn()
+          .mockRejectedValueOnce(new Error("500 Internal Server Error"))
+          .mockResolvedValueOnce(mockEmbedding()),
+      });
+
+      const promise = aiClient.createEmbedding("mock", {
+        input: "test",
+        model: "mock-embed",
+        maxRetries: 2,
+        backoffMs: 50,
+      });
+      await jest.runAllTimersAsync();
+      const result = await promise;
+
+      expect(result.data[0].embedding).toEqual([0.1, 0.2]);
+      expect(mockGenericClient.createEmbedding).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  // ── createImageGeneration ─────────────────────────────────────────────────
+
+  describe("createImageGeneration", () => {
+    it("forwards the AbortSignal to createImageGeneration on the client", async () => {
+      const { aiClient, mockGenericClient } = setupClient();
+      const controller = new AbortController();
+
+      await aiClient.createImageGeneration("mock", {
+        model: "mock-model",
+        prompt: "a cat",
+        signal: controller.signal,
+      });
+
+      const callArgs = (mockGenericClient.createImageGeneration as jest.Mock)
+        .mock.calls[0][0];
+      expect(callArgs.signal).toBeInstanceOf(AbortSignal);
+    });
+
+    it("retries on 429 and succeeds", async () => {
+      jest.useFakeTimers();
+      const { aiClient, mockGenericClient } = setupClient({
+        createImageGeneration: jest
+          .fn()
+          .mockRejectedValueOnce(new Error("429 Too Many Requests"))
+          .mockResolvedValueOnce(mockImage()),
+      });
+
+      const promise = aiClient.createImageGeneration("mock", {
+        model: "mock-model",
+        prompt: "a cat",
+        maxRetries: 2,
+        backoffMs: 50,
+      });
+      await jest.runAllTimersAsync();
+      const result = await promise;
+
+      expect(result.data[0].url).toBe("https://mock.local/image.png");
+      expect(mockGenericClient.createImageGeneration).toHaveBeenCalledTimes(2);
+    });
+
+    it("aborts when external signal fires mid-request", async () => {
+      const controller = new AbortController();
+      const { aiClient, mockGenericClient } = setupClient({
+        createImageGeneration: jest.fn().mockImplementation((opts: any) => {
+          return new Promise((_, reject) => {
+            opts.signal?.addEventListener("abort", () =>
+              reject(opts.signal.reason)
+            );
+          });
+        }),
+      });
+
+      const promise = aiClient.createImageGeneration("mock", {
+        model: "mock-model",
+        prompt: "a cat",
+        signal: controller.signal,
+      });
+      setImmediate(() =>
+        controller.abort(new DOMException("User cancelled", "AbortError"))
+      );
+
+      await expect(promise).rejects.toMatchObject({ name: "AbortError" });
+      expect(mockGenericClient.createImageGeneration).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  // ── createAudioGeneration ─────────────────────────────────────────────────
+
+  describe("createAudioGeneration", () => {
+    it("forwards the AbortSignal to createAudioGeneration on the client", async () => {
+      const { aiClient, mockGenericClient } = setupClient();
+      const controller = new AbortController();
+
+      await aiClient.createAudioGeneration("mock", {
+        model: "mock-model",
+        input: "Hello world",
+        voice: "alloy",
+        signal: controller.signal,
+      });
+
+      const callArgs = (mockGenericClient.createAudioGeneration as jest.Mock)
+        .mock.calls[0][0];
+      expect(callArgs.signal).toBeInstanceOf(AbortSignal);
+    });
+
+    it("retries on ECONNRESET and succeeds", async () => {
+      jest.useFakeTimers();
+      const { aiClient, mockGenericClient } = setupClient({
+        createAudioGeneration: jest
+          .fn()
+          .mockRejectedValueOnce(new Error("ECONNRESET"))
+          .mockResolvedValueOnce(mockAudio()),
+      });
+
+      const promise = aiClient.createAudioGeneration("mock", {
+        model: "mock-model",
+        input: "Hello",
+        voice: "alloy",
+        maxRetries: 2,
+        backoffMs: 50,
+      });
+      await jest.runAllTimersAsync();
+      const result = await promise;
+
+      expect(result.format).toBe("mp3");
+      expect(mockGenericClient.createAudioGeneration).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  // ── createVideoGeneration ─────────────────────────────────────────────────
+
+  describe("createVideoGeneration", () => {
+    it("forwards the AbortSignal to createVideoGeneration on the client", async () => {
+      const { aiClient, mockGenericClient } = setupClient();
+      const controller = new AbortController();
+
+      await aiClient.createVideoGeneration("mock", {
+        model: "mock-model",
+        prompt: "a sunset",
+        signal: controller.signal,
+      });
+
+      const callArgs = (mockGenericClient.createVideoGeneration as jest.Mock)
+        .mock.calls[0][0];
+      expect(callArgs.signal).toBeInstanceOf(AbortSignal);
+    });
+
+    it("retries on 503 and succeeds", async () => {
+      jest.useFakeTimers();
+      const { aiClient, mockGenericClient } = setupClient({
+        createVideoGeneration: jest
+          .fn()
+          .mockRejectedValueOnce(new Error("503 Service Unavailable"))
+          .mockResolvedValueOnce({
+            created: Date.now(),
+            data: [{ url: "https://mock.local/video.mp4" }],
+          }),
+      });
+
+      const promise = aiClient.createVideoGeneration("mock", {
+        model: "mock-model",
+        prompt: "a sunset",
+        maxRetries: 2,
+        backoffMs: 50,
+      });
+      await jest.runAllTimersAsync();
+      const result = await promise;
+
+      expect(result.data[0].url).toBe("https://mock.local/video.mp4");
+      expect(mockGenericClient.createVideoGeneration).toHaveBeenCalledTimes(2);
+    });
+  });
+
+  // ── createAudioTranscription ──────────────────────────────────────────────
+
+  describe("createAudioTranscription", () => {
+    it("forwards the AbortSignal to createAudioTranscription on the client", async () => {
+      const { aiClient, mockGenericClient } = setupClient();
+      const controller = new AbortController();
+
+      await aiClient.createAudioTranscription("mock", {
+        file: Buffer.from("fake-audio"),
+        signal: controller.signal,
+      });
+
+      const callArgs = (
+        mockGenericClient.createAudioTranscription as jest.Mock
+      ).mock.calls[0][0];
+      expect(callArgs.signal).toBeInstanceOf(AbortSignal);
+    });
+
+    it("retries on timeout error and succeeds", async () => {
+      jest.useFakeTimers();
+      const { aiClient, mockGenericClient } = setupClient({
+        createAudioTranscription: jest
+          .fn()
+          .mockRejectedValueOnce(new Error("timeout"))
+          .mockResolvedValueOnce({ text: "hello world" }),
+      });
+
+      const promise = aiClient.createAudioTranscription("mock", {
+        file: Buffer.from("fake-audio"),
+        maxRetries: 2,
+        backoffMs: 50,
+      });
+      await jest.runAllTimersAsync();
+      const result = await promise;
+
+      expect(result.text).toBe("hello world");
+      expect(mockGenericClient.createAudioTranscription).toHaveBeenCalledTimes(2);
+    });
+  });
+});