@tyvm/knowhow 0.0.158 → 0.0.160

package/tests/clients/AIClient.test.ts

@@ -1,668 +0,0 @@
-import { AIClient } from "../../src/clients";
-
-// Prevent real HTTP calls during provider initialisation — resolveClient
-// returns null so all DEFAULT_PROVIDERS are skipped by registerModelProviders.
-jest.spyOn(AIClient.prototype as any, "resolveClient").mockReturnValue(null);
-
-import {
-  GenericClient,
-  CompletionOptions,
-  CompletionResponse,
-  EmbeddingOptions,
-  EmbeddingResponse,
-} from "../../src/clients/types";
-
-class FakeClient implements GenericClient {
-  private apiKey: string = "";
-  private models: { id: string }[] = [
-    { id: "fake-model-1" },
-    { id: "fake-model-2" },
-    { id: "fake-embed-model" },
-  ];
-
-  constructor(modelIds?: string[]) {
-    if (modelIds) {
-      this.models = modelIds.map((id) => ({ id }));
-    }
-  }
-
-  setKey(key: string): void {
-    this.apiKey = key;
-  }
-
-  setModels(models: { id: string }[]): void {
-    this.models = models;
-  }
-
-  async createChatCompletion(
-    options: CompletionOptions
-  ): Promise<CompletionResponse> {
-    return {
-      choices: [
-        {
-          message: {
-            role: "assistant",
-            content: `Fake response for model: ${options.model}`,
-          },
-        },
-      ],
-      model: options.model,
-      usage: { prompt_tokens: 50, completion_tokens: 50, total_tokens: 100 },
-    };
-  }
-
-  async createEmbedding(options: EmbeddingOptions): Promise<EmbeddingResponse> {
-    return {
-      data: [
-        {
-          object: "embedding",
-          embedding: [0.1, 0.2, 0.3],
-          index: 0,
-        },
-      ],
-      model: options.model || "fake-embed-model",
-      usage: {
-        prompt_tokens: 10,
-        total_tokens: 10,
-      },
-    };
-  }
-
-  async getModels(): Promise<{ id: string }[]> {
-    return this.models;
-  }
-}
-
-describe("AIClient", () => {
-  let aiClient: AIClient;
-  let fakeClient: FakeClient;
-
-  beforeEach(() => {
-    aiClient = new AIClient();
-    fakeClient = new FakeClient();
-  });
-
-  describe("registerClient and getClient", () => {
-    it("should register a fake client and retrieve it", () => {
-      aiClient.registerClient("fake", fakeClient);
-      aiClient.registerModels("fake", ["fake-model-1", "fake-model-2"]);
-
-      const result = aiClient.getClient("fake");
-
-      expect(result.client).toBe(fakeClient);
-      expect(result.provider).toBe("fake");
-      expect(result.model).toBeUndefined();
-    });
-
-    it("should register a fake client and retrieve it with model", () => {
-      aiClient.registerClient("fake", fakeClient);
-      aiClient.registerModels("fake", ["fake-model-1", "fake-model-2"]);
-
-      const result = aiClient.getClient("fake", "fake-model-1");
-
-      expect(result.client).toBe(fakeClient);
-      expect(result.provider).toBe("fake");
-      expect(result.model).toBe("fake-model-1");
-    });
-
-    it("should return null client when provider is not registered", () => {
-      const result = aiClient.getClient("unregistered");
-      expect(result.client).toBeUndefined();
-      expect(result.provider).toBe("unregistered");
-      expect(result.model).toBeUndefined();
-    });
-
-    it("should throw error when model is not found", () => {
-      aiClient.registerClient("fake", fakeClient);
-      aiClient.registerModels("fake", ["fake-model-1"]);
-
-      // getClient now warns instead of throws for unregistered models,
-      // so the API itself can accept/reject the model (e.g. newly-released models)
-      const result = aiClient.getClient("fake", "non-existent-model");
-      expect(result.provider).toBe("fake");
-      expect(result.model).toBe("non-existent-model");
-    });
-  });
-  describe("detectProviderModel", () => {
-    beforeEach(() => {
-      aiClient.registerClient("fake", fakeClient);
-      aiClient.registerModels("fake", [
-        "fake-model-1",
-        "fake-model-2",
-        "fake-embed-model",
-      ]);
-
-      aiClient.registerClient("another", new FakeClient());
-      aiClient.registerModels("another", ["another-model-1", "gpt-4"]);
-    });
-
-    it("should detect exact provider and model match", () => {
-      const result = aiClient.detectProviderModel("fake", "fake-model-1");
-      expect(result.provider).toBe("fake");
-      expect(result.model).toBe("fake-model-1");
-    });
-
-    it("should detect model from slash-separated format (provider/model)", () => {
-      const result = aiClient.detectProviderModel("", "fake/fake-model-1");
-      expect(result.provider).toBe("fake");
-      expect(result.model).toBe("fake-model-1");
-    });
-
-    it("should detect model from nested slash format (provider/subprovider/model)", () => {
-      aiClient.registerClient("knowhow", new FakeClient());
-      aiClient.registerModels("knowhow", [
-        "openai/gpt-4",
-        "anthropic/claude-3",
-      ]);
-
-      const result = aiClient.detectProviderModel("", "knowhow/openai/gpt-4");
-      expect(result.provider).toBe("knowhow"); // AIClient returns the first part as provider
-      expect(result.model).toBe("openai/gpt-4"); // Rest becomes model
-    });
-
-    it("should find model by detection in registered providers", () => {
-      // Register an explicit myopenai provider with a gpt-4 prefix model
-      aiClient.registerClient("myopenai", new FakeClient());
-      aiClient.registerModels("myopenai", ["gpt-4-turbo", "gpt-4-vision"]);
-
-      // "another" provider (registered in beforeEach) has exact "gpt-4" match
-      // "myopenai" has prefix-matches "gpt-4-turbo" and "gpt-4-vision"
-      // findModel finds the first provider with a model matching the prefix "gpt-4"
-      const result = aiClient.detectProviderModel("", "gpt-4");
-      // Some provider should be found that has a gpt-4 model
-      expect(result.provider).toBeTruthy();
-      // The found model should start with "gpt-4"
-      expect(result.model).toMatch(/^gpt-4/);
-    });
-
-    it("should handle model with provider prefix when provider is empty", () => {
-      const result = aiClient.detectProviderModel("", "another/gpt-4");
-      expect(result.provider).toBe("another"); // Provider from prefix
-      expect(result.model).toBe("gpt-4");
-    });
-
-    it("should return original values when no match found", () => {
-      const result = aiClient.detectProviderModel("unknown", "unknown-model");
-      expect(result.provider).toBe("unknown");
-      expect(result.model).toBe("unknown-model");
-    });
-
-    // ── New tests: explicit provider + slashed model prefix stripping ────────
-
-    it("knowhow provider: resolves 'anthropic/claude-sonnet-4-6' to model stored as 'anthropic/claude-sonnet-4-6'", () => {
-      aiClient.registerClient("knowhow", new FakeClient());
-      aiClient.registerModels("knowhow", [
-        "anthropic/claude-sonnet-4-6",
-        "openai/gpt-4",
-        "google/gemini-pro",
-      ]);
-
-      // Exact match: provider=knowhow, model=anthropic/claude-sonnet-4-6
-      const result = aiClient.detectProviderModel("knowhow", "anthropic/claude-sonnet-4-6");
-      expect(result.provider).toBe("knowhow");
-      expect(result.model).toBe("anthropic/claude-sonnet-4-6");
-    });
-
-    it("knowhow provider: resolves bare 'claude-sonnet-4-6' to 'anthropic/claude-sonnet-4-6' stored on knowhow", () => {
-      aiClient.registerClient("knowhow", new FakeClient());
-      aiClient.registerModels("knowhow", [
-        "anthropic/claude-sonnet-4-6",
-        "openai/gpt-4",
-      ]);
-
-      // Bare model name — should resolve by stripping the provider prefix from
-      // the registered model and matching on the trailing segment
-      const result = aiClient.detectProviderModel("knowhow", "claude-sonnet-4-6");
-      expect(result.provider).toBe("knowhow");
-      expect(result.model).toBe("anthropic/claude-sonnet-4-6");
-    });
-
-    it("fireworks provider: resolves 'kimi-k3' to 'accounts/fireworks/models/kimi-k3'", () => {
-      aiClient.registerClient("fireworks", new FakeClient());
-      aiClient.registerModels("fireworks", [
-        "accounts/fireworks/models/kimi-k3",
-        "accounts/fireworks/models/kimi-k2p7-code",
-        "accounts/fireworks/models/minimax-m3",
-        "accounts/fireworks/models/deepseek-v3-2",
-      ]);
-
-      const result = aiClient.detectProviderModel("fireworks", "kimi-k3");
-      expect(result.provider).toBe("fireworks");
-      expect(result.model).toBe("accounts/fireworks/models/kimi-k3");
-    });
-
-    it("fireworks provider: resolves 'accounts/fireworks/models/kimi-k3' (full path) correctly", () => {
-      aiClient.registerClient("fireworks", new FakeClient());
-      aiClient.registerModels("fireworks", [
-        "accounts/fireworks/models/kimi-k3",
-        "accounts/fireworks/models/deepseek-v4-pro",
-      ]);
-
-      // Full path exact match
-      const result = aiClient.detectProviderModel("fireworks", "accounts/fireworks/models/kimi-k3");
-      expect(result.provider).toBe("fireworks");
-      expect(result.model).toBe("accounts/fireworks/models/kimi-k3");
-    });
-
-    it("explicit provider: does NOT leak to a different provider when stripping slashes", () => {
-      aiClient.registerClient("knowhow", new FakeClient());
-      aiClient.registerModels("knowhow", ["anthropic/claude-sonnet-4-6"]);
-
-      // 'fake' provider doesn't have this model — should not find it on knowhow
-      const result = aiClient.detectProviderModel("fake", "claude-sonnet-4-6");
-      // Falls back: provider=fake, model=claude-sonnet-4-6 (no cross-provider leak)
-      expect(result.provider).toBe("fake");
-      expect(result.model).toBe("claude-sonnet-4-6");
-    });
-
-    it("empty provider: finds 'claude-sonnet-4-6' cross-provider on knowhow (stored as 'anthropic/claude-sonnet-4-6')", () => {
-      aiClient.registerClient("knowhow", new FakeClient());
-      aiClient.registerModels("knowhow", [
-        "anthropic/claude-sonnet-4-6",
-        "openai/gpt-4",
-      ]);
-
-      const result = aiClient.detectProviderModel("", "claude-sonnet-4-6");
-      expect(result.provider).toBe("knowhow");
-      expect(result.model).toBe("anthropic/claude-sonnet-4-6");
-    });
-
-    it("empty provider: finds 'kimi-k3' cross-provider on fireworks (stored as 'accounts/fireworks/models/kimi-k3')", () => {
-      aiClient.registerClient("fireworks", new FakeClient());
-      aiClient.registerModels("fireworks", [
-        "accounts/fireworks/models/kimi-k3",
-        "accounts/fireworks/models/deepseek-v4-pro",
-      ]);
-
-      const result = aiClient.detectProviderModel("", "kimi-k3");
-      expect(result.provider).toBe("fireworks");
-      expect(result.model).toBe("accounts/fireworks/models/kimi-k3");
-    });
-
-    it("should detect real provider when model exists", () => {
-      // Register an anthropic provider explicitly so we don't rely on env vars
-      aiClient.registerClient("anthropic", new FakeClient());
-      aiClient.registerModels("anthropic", ["claude-3-opus-20240229"]);
-
-      // Test with provider prefix that gets stripped
-      const result = aiClient.detectProviderModel(
-        "",
-        "anthropic/claude-3-opus-20240229"
-      );
-      expect(result.provider).toBe("anthropic"); // anthropic provider found
-      expect(result.model).toBe("claude-3-opus-20240229");
-    });
-  });
-  describe("Model Listing Functionality", () => {
-    beforeEach(() => {
-      aiClient.registerClient("fake", fakeClient);
-      aiClient.registerModels("fake", [
-        "fake-model-1",
-        "fake-model-2",
-        "fake-embed-model",
-      ]);
-
-      aiClient.registerClient("another", new FakeClient());
-      aiClient.registerModels("another", [
-        "another-model-1",
-        "gpt-4",
-        "claude-3",
-      ]);
-    });
-
-    describe("listAllModels", () => {
-      it("should return all registered models from all providers", () => {
-        const allModels = aiClient.listAllModels();
-        expect(typeof allModels).toBe("object");
-        // Verify our registered test providers are present
-        expect(Object.keys(allModels).length).toBeGreaterThan(0);
-        // The test providers registered in beforeEach should be present
-        const providers = Object.keys(allModels);
-        expect(
-          providers.some((p) =>
-            ["fake", "another"].includes(p)
-          )
-        ).toBe(true);
-      });
-
-      it("should return empty array when no clients registered", () => {
-        const freshClient = new AIClient();
-        const allModels = freshClient.listAllModels();
-        // Note: AIClient starts with real providers from environment, so this will not be empty
-        expect(typeof allModels).toBe("object"); // Should return object with real providers
-      });
-    });
-
-    describe("getRegisteredModels", () => {
-      it("should return models for specific provider", () => {
-        const fakeModels = aiClient.getRegisteredModels("fake");
-        expect(fakeModels).toEqual([
-          "fake-model-1",
-          "fake-model-2",
-          "fake-embed-model",
-        ]);
-
-        const anotherModels = aiClient.getRegisteredModels("another");
-        expect(anotherModels).toEqual(["another-model-1", "gpt-4", "claude-3"]);
-      });
-
-      it("should return empty array for unregistered provider", () => {
-        const models = aiClient.getRegisteredModels("unregistered");
-        expect(models).toEqual([]);
-      });
-    });
-
-    describe("Model registration from client.getModels()", () => {
-      it("should register models from fake client getModels() method", async () => {
-        const clientWithModels = new FakeClient();
-        clientWithModels.setModels([
-          { id: "dynamic-model-1" },
-          { id: "dynamic-model-2" },
-        ]);
-
-        aiClient.registerClient("dynamic", clientWithModels);
-
-        // Register models from the client's getModels method
-        const models = await clientWithModels.getModels();
-        const modelIds = models.map((m) => m.id);
-        aiClient.registerModels("dynamic", modelIds);
-
-        const registeredModels = aiClient.getRegisteredModels("dynamic");
-        expect(registeredModels).toEqual([
-          "dynamic-model-1",
-          "dynamic-model-2",
-        ]);
-      });
-    });
-  });
-  describe("Various Model Format Support", () => {
-    beforeEach(() => {
-      aiClient.registerClient("openai", new FakeClient());
-      aiClient.registerModels("openai", ["gpt-4", "gpt-3.5-turbo"]);
-
-      aiClient.registerClient("knowhow", new FakeClient());
-      aiClient.registerModels("knowhow", [
-        "openai/gpt-4",
-        "anthropic/claude-3",
-        "google/gemini-pro",
-      ]);
-    });
-
-    it("should support format: provider='openai', model='gpt-4'", () => {
-      const client = aiClient.getClient("openai", "gpt-4");
-      expect(client).toBeDefined();
-    });
-
-    it("should support format: provider='knowhow', model='openai/gpt-4'", () => {
-      const client = aiClient.getClient("knowhow", "openai/gpt-4");
-      expect(client).toBeDefined();
-    });
-
-    it("should support format: provider='', model='openai/gpt-4' (auto-detect)", () => {
-      const client = aiClient.getClient("", "openai/gpt-4");
-      expect(client).toBeDefined();
-    });
-
-    it("should support format: provider='', model='knowhow/openai/gpt-4' (nested)", () => {
-      const client = aiClient.getClient("", "knowhow/openai/gpt-4");
-      expect(client).toBeDefined();
-    });
-
-    it("should support model detection with complex nested paths", () => {
-      aiClient.registerClient("complex", new FakeClient());
-      aiClient.registerModels("complex", [
-        "provider/subprovider/model-name",
-        "another/path/to/model",
-      ]);
-
-      const client1 = aiClient.getClient(
-        "",
-        "complex/provider/subprovider/model-name"
-      );
-      expect(client1).toBeDefined();
-
-      const client2 = aiClient.getClient("", "complex/another/path/to/model");
-      expect(client2).toBeDefined();
-    });
-
-    it("should handle model detection with provider stripping", () => {
-      // Register a model without provider prefix
-      aiClient.registerClient("stripped", new FakeClient());
-      aiClient.registerModels("stripped", ["claude-3-opus"]);
-
-      // Should find it even when requested with provider prefix
-      const client = aiClient.getClient("", "anthropic/claude-3-opus");
-      expect(client).toBeDefined();
-    });
-  });
-  describe("Integration Tests", () => {
-    it("should handle end-to-end flow: register client → register models → detect → retrieve", () => {
-      // Register client and models
-      const fakeClient = new FakeClient();
-      aiClient.registerClient("integration", fakeClient);
-      aiClient.registerModels("integration", [
-        "model-1",
-        "model-2",
-        "provider/model-3",
-      ]);
-
-      // Test detection
-      const detection1 = aiClient.detectProviderModel(
-        "",
-        "integration/model-1"
-      );
-      expect(detection1).toEqual({ provider: "integration", model: "model-1" });
-
-      const detection2 = aiClient.detectProviderModel(
-        "",
-        "integration/provider/model-3"
-      );
-      expect(detection2).toEqual({
-        provider: "integration",
-        model: "provider/model-3",
-      });
-
-      // Test retrieval
-      const result1 = aiClient.getClient("integration", "model-1");
-      expect(result1.client).toBe(fakeClient);
-      expect(result1.provider).toBe("integration");
-      expect(result1.model).toBe("model-1");
-
-      const result2 = aiClient.getClient("", "integration/model-1");
-      expect(result2.client).toBe(fakeClient);
-      expect(result2.provider).toBe("integration");
-    });
-
-    it("should register models from client.getModels() and make them available", async () => {
-      const fakeClient = new FakeClient(["auto-model-1", "auto-model-2"]);
-      aiClient.registerClient("auto", fakeClient);
-
-      // Get models from client
-      const models = await fakeClient.getModels();
-      aiClient.registerModels(
-        "auto",
-        models.map((m) => m.id)
-      );
-
-      // Should be able to retrieve client using these models
-      const result1 = aiClient.getClient("auto", "auto-model-1");
-      expect(result1.client).toBe(fakeClient);
-      expect(result1.provider).toBe("auto");
-
-      const result2 = aiClient.getClient("", "auto/auto-model-1");
-      expect(result2.client).toBe(fakeClient);
-      expect(result2.provider).toBe("auto");
-
-      // Models should appear in listings
-      const allModels = aiClient.listAllModels() as any;
-      expect(allModels.auto).toContain("auto-model-1");
-      expect(allModels.auto).toContain("auto-model-2");
-    });
-
-    it("should handle multiple providers with overlapping model names", () => {
-      // Register multiple providers with same model names
-      aiClient.registerClient("provider1", new FakeClient());
-      aiClient.registerModels("provider1", ["common-model", "unique-model-1"]);
-
-      aiClient.registerClient("provider2", new FakeClient());
-      aiClient.registerModels("provider2", ["common-model", "unique-model-2"]);
-
-      // Should be able to get specific provider's model
-      const client1 = aiClient.getClient("provider1", "common-model");
-      const client2 = aiClient.getClient("provider2", "common-model");
-
-      expect(client1.client).toBeDefined();
-      expect(client2.client).toBeDefined();
-      expect(client1.client).not.toBe(client2.client);
-
-      // Auto-detection should work with full paths
-      const autoClient1 = aiClient.getClient("", "provider1/common-model");
-      const autoClient2 = aiClient.getClient("", "provider2/common-model");
-
-      expect(autoClient1.client).toBe(client1.client);
-      expect(autoClient2.client).toBe(client2.client);
-    });
-  });
-  describe("Edge Case Testing", () => {
-    beforeEach(() => {
-      aiClient.registerClient("edge", new FakeClient());
-      aiClient.registerModels("edge", [
-        "normal-model",
-        "model-with-dashes",
-        "model_with_underscores",
-      ]);
-    });
-
-    it("should handle empty provider and model strings", () => {
-      // Empty provider + empty model: detectProviderModel returns the fallback (empty strings)
-      // getClient("", "") throws because "" is not a registered provider
-      expect(() => aiClient.getClient("", "")).toThrow();
-
-      const detection = aiClient.detectProviderModel("", "");
-      // Falls back to original values — both empty
-      expect(detection?.provider).toBe("");
-      expect(detection?.model).toBe("");
-    });
-
-    it("should handle malformed model formats", () => {
-      // Test various malformed formats
-      const malformedInputs = [
-        "//model",
-        "provider//model",
-        "/provider/model",
-        "provider/",
-        "///",
-        "provider/model/",
-      ];
-
-      malformedInputs.forEach((input) => {
-        const detection = aiClient.detectProviderModel("", input);
-        // Should either find a valid match or return fallback values, not throw
-        expect(detection).toBeDefined();
-        expect(detection?.provider).toBeDefined();
-        expect(detection?.model).toBeDefined();
-      });
-    });
-
-    it("should handle provider stripping with complex model names", () => {
-      // Register providers explicitly so we don't rely on env vars
-      aiClient.registerClient("anthropic", new FakeClient());
-      aiClient.registerModels("anthropic", ["claude-3-opus-20240229"]);
-
-      const detection1 = aiClient.detectProviderModel(
-        "",
-        "anthropic/claude-3-opus-20240229"
-      );
-      expect(detection1?.provider).toBe("anthropic");
-      expect(detection1?.model).toBe("claude-3-opus-20240229");
-
-      // For models that don't exist in the registered providers, AIClient falls back
-      const detection2 = aiClient.detectProviderModel(
-        "",
-        "openai/non-existent-model"
-      );
-      // Should return original values when no match found
-      expect(detection2).toBeDefined();
-      expect(detection2?.provider).toBe("");
-      expect(detection2?.model).toBe("openai/non-existent-model");
-    });
-
-    it("should handle model prefix matching edge cases", () => {
-      aiClient.registerClient("prefix", new FakeClient());
-      aiClient.registerModels("prefix", [
-        "test-model",
-        "test-model-turbo",
-        "test-model-vision",
-      ]);
-
-      // Should match exact model first
-      const detection1 = aiClient.detectProviderModel("", "prefix/test-model");
-      expect(detection1?.model).toBe("test-model");
-
-      // Custom providers don't do prefix matching - should return empty provider
-      const detection2 = aiClient.detectProviderModel(
-        "",
-        "prefix/test-model-unknown"
-      );
-      expect(detection2?.provider).toBe("");
-      // Should return the full model name since no match found
-      expect(detection2?.model).toBe("prefix/test-model-unknown");
-    });
-
-    it("should handle special characters in model names", () => {
-      aiClient.registerClient("special", new FakeClient());
-      aiClient.registerModels("special", [
-        "model-with-dashes",
-        "model_with_underscores",
-        "model.with.dots",
-        "model@with@symbols",
-      ]);
-
-      const testCases = [
-        "special/model-with-dashes",
-        "special/model_with_underscores",
-        "special/model.with.dots",
-        "special/model@with@symbols",
-      ];
-
-      testCases.forEach((testCase) => {
-        const detection = aiClient.detectProviderModel("", testCase);
-        expect(detection).toBeDefined();
-        expect(detection?.provider).toBe("special");
-      });
-    });
-
-    it("should handle case sensitivity correctly", () => {
-      aiClient.registerClient("CaseTest", new FakeClient());
-      aiClient.registerModels("CaseTest", ["Model-Name", "UPPERCASE-MODEL"]);
-
-      // Test exact case matches
-      let detection = aiClient.detectProviderModel("", "CaseTest/Model-Name");
-      expect(detection?.model).toBe("Model-Name");
-
-      detection = aiClient.detectProviderModel("", "CaseTest/UPPERCASE-MODEL");
-      expect(detection?.model).toBe("UPPERCASE-MODEL");
-
-      // Test case mismatches - AIClient is case sensitive for providers
-      detection = aiClient.detectProviderModel("", "casetest/Model-Name");
-      // AIClient actually finds the provider despite case mismatch
-      expect(detection?.provider).toBe("CaseTest");
-      expect(detection?.model).toBe("Model-Name");
-    });
-
-    it("should handle very long model paths", () => {
-      const longProvider = "very-long-provider-name-with-many-segments";
-      const longModel =
-        "extremely/long/nested/model/path/with/many/segments/final-model-name";
-
-      aiClient.registerClient(longProvider, new FakeClient());
-      aiClient.registerModels(longProvider, [longModel]);
-
-      const fullPath = `${longProvider}/${longModel}`;
-      const detection = aiClient.detectProviderModel("", fullPath);
-
-      expect(detection?.provider).toBe(longProvider);
-      expect(detection?.model).toBe(longModel);
-    });
-  });
-});