@node-llm/core 1.4.0 → 1.4.1

package/packages/core/test/unit/core/errors.test.ts

@@ -0,0 +1,116 @@
+import { describe, it, expect, vi } from "vitest";
+import { 
+  LLMError, 
+  APIError, 
+  RateLimitError, 
+  AuthenticationError, 
+  BadRequestError,
+  ServerError
+} from "../../../src/errors/index.js";
+import { Executor } from "../../../src/executor/Executor.js";
+import { handleOpenAIError } from "../../../src/providers/openai/Errors.js";
+
+describe("Error System", () => {
+  describe("Hierarchy", () => {
+    it("should maintain proper inheritance", () => {
+      const error = new RateLimitError("Too many requests", { detail: "limit exceeded" }, "openai", "gpt-4");
+      
+      expect(error).toBeInstanceOf(RateLimitError);
+      expect(error).toBeInstanceOf(APIError);
+      expect(error).toBeInstanceOf(LLMError);
+      expect(error).toBeInstanceOf(Error);
+      expect(error.status).toBe(429);
+      expect(error.provider).toBe("openai");
+    });
+  });
+
+  describe("Executor Retry Logic", () => {
+    it("should retry on RateLimitError", async () => {
+      let attempts = 0;
+      const mockProvider = {
+        chat: vi.fn().mockImplementation(async () => {
+          attempts++;
+          if (attempts < 2) {
+            throw new RateLimitError("Rate limited", {});
+          }
+          return { content: "Success" };
+        })
+      };
+
+      const executor = new Executor(mockProvider as any, { attempts: 3, delayMs: 0 });
+      const result = await executor.executeChat({ model: "test", messages: [] });
+
+      expect(result.content).toBe("Success");
+      expect(attempts).toBe(2);
+      expect(mockProvider.chat).toHaveBeenCalledTimes(2);
+    });
+
+    it("should NOT retry on AuthenticationError", async () => {
+      let attempts = 0;
+      const mockProvider = {
+        chat: vi.fn().mockImplementation(async () => {
+          attempts++;
+          throw new AuthenticationError("Invalid key", 401, {});
+        })
+      };
+
+      const executor = new Executor(mockProvider as any, { attempts: 3, delayMs: 0 });
+      
+      await expect(executor.executeChat({ model: "test", messages: [] }))
+        .rejects.toThrow(AuthenticationError);
+        
+      expect(attempts).toBe(1); // Should fail fast
+    });
+
+    it("should NOT retry on BadRequestError", async () => {
+      let attempts = 0;
+      const mockProvider = {
+        chat: vi.fn().mockImplementation(async () => {
+          attempts++;
+          throw new BadRequestError("Invalid params", {});
+        })
+      };
+
+      const executor = new Executor(mockProvider as any, { attempts: 3, delayMs: 0 });
+      
+      await expect(executor.executeChat({ model: "test", messages: [] }))
+        .rejects.toThrow(BadRequestError);
+        
+      expect(attempts).toBe(1);
+    });
+  });
+
+  describe("OpenAI Error Mapper", () => {
+    const mockResponse = (status: number, body: any) => ({
+      status,
+      ok: false,
+      json: async () => body,
+      text: async () => JSON.stringify(body)
+    } as Response);
+
+    it("should map 401 to AuthenticationError", async () => {
+      const response = mockResponse(401, { error: { message: "Invalid API Key" } });
+      await expect(handleOpenAIError(response)).rejects.toThrow(AuthenticationError);
+    });
+
+    it("should map 429 to RateLimitError", async () => {
+      const response = mockResponse(429, { error: { message: "Quota exceeded" } });
+      await expect(handleOpenAIError(response)).rejects.toThrow(RateLimitError);
+    });
+
+    it("should map 500 to ServerError", async () => {
+      const response = mockResponse(500, { error: { message: "Internal crash" } });
+      await expect(handleOpenAIError(response)).rejects.toThrow(ServerError);
+    });
+
+    it("should extract message from OpenAI error body", async () => {
+      const response = mockResponse(400, { error: { message: "Custom OpenAI message" } });
+      try {
+        await handleOpenAIError(response);
+      } catch (e: any) {
+        expect(e.message).toBe("Custom OpenAI message");
+        expect(e).toBeInstanceOf(BadRequestError);
+      }
+    });
+  });
+});

package/packages/core/test/unit/core/llm.test.ts

@@ -0,0 +1,118 @@
+import { describe, it, expect } from "vitest";
+import { NodeLLM } from "../../../src/llm.js";
+import { providerRegistry } from "../../../src/providers/registry.js";
+import { FakeProvider } from "../../fake-provider.js";
+
+describe("NodeLLM.configure", () => {
+  it("resolves provider by name using registry", async () => {
+    providerRegistry.register("fake", () => {
+      return new FakeProvider(["fake reply"]);
+    });
+
+    NodeLLM.configure({ provider: "fake" });
+
+    const chat = NodeLLM.chat("test-model");
+    const reply = await chat.ask("Hello");
+
+    expect(String(reply)).toBe("fake reply");
+  });
+
+  it("throws error when provider is not configured", () => {
+    const freshLLM = new (NodeLLM.constructor as any)();
+
+    expect(() => {
+      freshLLM.chat("test-model");
+    }).toThrow("LLM provider not configured");
+  });
+});
+
+describe("NodeLLM Default Model Resolution", () => {
+  it("uses provider default when no model is specified", () => {
+    const provider = new FakeProvider();
+    NodeLLM.configure({ provider });
+
+    const chat = NodeLLM.chat();
+    expect(chat.modelId).toBe("fake-default-model");
+  });
+
+  it("prioritizes global configuration default over provider default", () => {
+    const provider = new FakeProvider();
+    NodeLLM.configure({ 
+      provider,
+      defaultChatModel: "custom-global-default"
+    });
+
+    const chat = NodeLLM.chat();
+    expect(chat.modelId).toBe("custom-global-default");
+  });
+
+  it("prioritizes explicit argument over all defaults", () => {
+    const provider = new FakeProvider();
+    NodeLLM.configure({ 
+      provider,
+      defaultChatModel: "custom-global-default"
+    });
+
+    const chat = NodeLLM.chat("explicit-model");
+    expect(chat.modelId).toBe("explicit-model");
+  });
+});
+
+class MockCapabilitiesProvider extends FakeProvider {
+  // Explicitly deny support for everything
+  capabilities = {
+    supportsVision: (_m: string) => false,
+    supportsTools: (_m: string) => false,
+    supportsStructuredOutput: (_m: string) => false,
+    supportsEmbeddings: (_m: string) => false,
+    supportsImageGeneration: (_m: string) => false,
+    supportsTranscription: (_m: string) => false,
+    supportsModeration: (_m: string) => false,
+    supportsReasoning: (_m: string) => false,
+    getContextWindow: (_m: string) => 0
+  } as any;
+
+  async paint(_req: any): Promise<any> { return { url: "ok" }; }
+  async transcribe(_req: any): Promise<any> { return { text: "ok" }; }
+  async embed(_req: any): Promise<any> { return { vectors: [[0.1]], model: "test", dimensions: 1 }; }
+  async moderate(_req: any): Promise<any> { return { flagged: false }; }
+}
+
+describe("NodeLLM assumeModelExists", () => {
+  it("bypasses capability checks when assumeModelExists is true", async () => {
+    const provider = new MockCapabilitiesProvider();
+    NodeLLM.configure({ provider });
+
+    // 1. Image Generation (paint)
+    // Should fail without flag
+    await expect(NodeLLM.paint("test", { model: "unsupported-model" }))
+      .rejects.toThrow("does not support image generation");
+
+    // Should succeed with flag
+    await expect(NodeLLM.paint("test", { model: "unsupported-model", assumeModelExists: true }))
+      .resolves.toBeDefined();
+
+    // 2. Transcription
+    // Should fail without flag
+    await expect(NodeLLM.transcribe("audio.mp3", { model: "unsupported-model" }))
+      .rejects.toThrow("does not support transcription");
+      
+    // Should succeed with flag
+    await expect(NodeLLM.transcribe("audio.mp3", { model: "unsupported-model", assumeModelExists: true }))
+      .resolves.toBeDefined();
+
+    // 3. Embeddings
+    await expect(NodeLLM.embed("text", { model: "unsupported-model" }))
+      .rejects.toThrow("does not support embeddings");
+
+    await expect(NodeLLM.embed("text", { model: "unsupported-model", assumeModelExists: true }))
+      .resolves.toBeDefined();
+      
+    // 4. Moderation
+    await expect(NodeLLM.moderate("text", { model: "unsupported-model" }))
+      .rejects.toThrow("does not support moderation");
+
+    await expect(NodeLLM.moderate("text", { model: "unsupported-model", assumeModelExists: true }))
+      .resolves.toBeDefined();
+  });
+});

package/packages/core/test/unit/core/model-aliases-export.test.ts

@@ -0,0 +1,55 @@
+import { describe, it, expect } from "vitest";
+import { MODEL_ALIASES, resolveModelAlias } from "../../../src/index.js";
+
+describe("MODEL_ALIASES Export", () => {
+  it("should export MODEL_ALIASES object", () => {
+    expect(MODEL_ALIASES).toBeDefined();
+    expect(typeof MODEL_ALIASES).toBe("object");
+  });
+
+  it("should contain known aliases", () => {
+    expect(MODEL_ALIASES["claude-3-5-haiku"]).toBeDefined();
+    expect(MODEL_ALIASES["claude-3-5-haiku"]["anthropic"]).toBe("claude-3-5-haiku-20241022");
+    
+    expect(MODEL_ALIASES["gpt-4o"]).toBeDefined();
+    expect(MODEL_ALIASES["gpt-4o"]["openai"]).toBe("gpt-4o");
+  });
+
+  it("should allow programmatic alias validation", () => {
+    const userInput = "claude-3-5-haiku";
+    const isValidAlias = userInput in MODEL_ALIASES;
+    
+    expect(isValidAlias).toBe(true);
+  });
+
+  it("should allow checking provider support", () => {
+    const alias = "claude-3-5-haiku";
+    const provider = "anthropic";
+    
+    const hasProviderMapping = MODEL_ALIASES[alias]?.[provider] !== undefined;
+    
+    expect(hasProviderMapping).toBe(true);
+  });
+
+  it("should export resolveModelAlias function", () => {
+    expect(typeof resolveModelAlias).toBe("function");
+    
+    const result = resolveModelAlias("claude-3-5-haiku", "anthropic");
+    expect(result).toBe("claude-3-5-haiku-20241022");
+  });
+
+  it("should allow listing all aliases", () => {
+    const allAliases = Object.keys(MODEL_ALIASES);
+    
+    expect(allAliases.length).toBeGreaterThan(0);
+    expect(allAliases).toContain("claude-3-5-haiku");
+    expect(allAliases).toContain("gpt-4o");
+  });
+
+  it("should allow listing providers for an alias", () => {
+    const providers = Object.keys(MODEL_ALIASES["claude-3-5-haiku"]);
+    
+    expect(providers).toContain("anthropic");
+    expect(providers).toContain("openrouter");
+  });
+});

package/packages/core/test/unit/core/model_alias.test.ts

@@ -0,0 +1,130 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { NodeLLM } from "../../../src/llm.js";
+import { providerRegistry } from "../../../src/providers/registry.js";
+import { FakeProvider } from "../../fake-provider.js";
+import { Provider } from "../../../src/providers/Provider.js";
+
+// Mock aliases
+vi.mock("../../../src/aliases.js", () => ({
+  default: {
+    "test-alias": {
+      "fake": "resolved-model-id"
+    },
+    // Real-world examples
+    "gpt-4o": {
+      "openai": "gpt-4o",
+      "openrouter": "openai/gpt-4o"
+    },
+    "claude-3-5-sonnet": {
+      "anthropic": "claude-3-5-sonnet-20241022",
+      "bedrock": "anthropic.claude-3-5-sonnet-20241022-v2:0"
+    }
+  }
+}));
+
+class MockProvider extends FakeProvider implements Provider {
+    id = "fake";
+    capabilities = {
+      supportsVision: ((m: string) => true) as any,
+      supportsTools: ((m: string) => true) as any,
+      supportsStructuredOutput: ((m: string) => true) as any,
+      supportsEmbeddings: ((m: string) => true) as any,
+      supportsImageGeneration: ((m: string) => true) as any,
+      supportsTranscription: ((m: string) => true) as any,
+      supportsModeration: ((m: string) => true) as any,
+      getContextWindow: ((m: string) => 1000) as any
+    } as any;
+
+    async paint(req: any) { return { url: "ok", b64_json: "" }; }
+    async transcribe(req: any) { return { text: "ok", segments: [], model: "resolved-model-id" }; }
+    async moderate(req: any) { return { flagged: false, results: [], id: "mod-1", model: "resolved-model-id" }; }
+    async embed(req: any) { return { vectors: [], model: req.model, dimensions: 0, input_tokens: 10 }; }
+    async listModels() { return []; }
+}
+
+describe("Model Alias Resolution", () => {
+  let provider: MockProvider;
+
+  beforeEach(() => {
+    provider = new MockProvider();
+    
+    // Spy on methods 
+    vi.spyOn(provider, "paint");
+    vi.spyOn(provider, "transcribe");
+    vi.spyOn(provider, "moderate");
+    vi.spyOn(provider, "embed");
+
+    providerRegistry.register("fake", () => provider);
+    NodeLLM.configure({ provider: "fake" });
+  });
+
+  it("resolves alias in paint()", async () => {
+    await NodeLLM.paint("prompt", { model: "test-alias" });
+    
+    expect(provider.paint).toHaveBeenCalledWith(expect.objectContaining({
+      model: "resolved-model-id"
+    }));
+  });
+
+  it("resolves alias in transcribe()", async () => {
+    await NodeLLM.transcribe("file.mp3", { model: "test-alias" });
+
+    expect(provider.transcribe).toHaveBeenCalledWith(expect.objectContaining({
+      model: "resolved-model-id"
+    }));
+  });
+
+  it("resolves alias in moderate()", async () => {
+    await NodeLLM.moderate("text", { model: "test-alias" });
+
+    expect(provider.moderate).toHaveBeenCalledWith(expect.objectContaining({
+      model: "resolved-model-id"
+    }));
+  });
+
+describe("Real-world Alias Examples", () => {
+  let openaiProvider: MockProvider;
+  let anthropicProvider: MockProvider;
+
+  beforeEach(() => {
+    openaiProvider = new MockProvider();
+    openaiProvider.id = "openai";
+    
+    anthropicProvider = new MockProvider();
+    anthropicProvider.id = "anthropic";
+
+    vi.spyOn(openaiProvider, "paint");
+    vi.spyOn(anthropicProvider, "transcribe");
+
+    providerRegistry.register("openai", () => openaiProvider);
+    providerRegistry.register("anthropic", () => anthropicProvider);
+  });
+
+  it("resolves gpt-4o for openai provider", async () => {
+    NodeLLM.configure({ provider: "openai" });
+    await NodeLLM.paint("image prompt", { model: "gpt-4o" });
+
+    expect(openaiProvider.paint).toHaveBeenCalledWith(expect.objectContaining({
+      model: "gpt-4o"
+    }));
+  });
+
+  it("resolves claude-3-5-sonnet for anthropic provider", async () => {
+    NodeLLM.configure({ provider: "anthropic" });
+    // Using transcribe as a generic method call for testing
+    await NodeLLM.transcribe("audio.mp3", { model: "claude-3-5-sonnet" });
+
+    expect(anthropicProvider.transcribe).toHaveBeenCalledWith(expect.objectContaining({
+      model: "claude-3-5-sonnet-20241022"
+    }));
+  });
+});
+
+  it("resolves alias in embed()", async () => {
+    await NodeLLM.embed("text", { model: "test-alias" });
+
+    expect(provider.embed).toHaveBeenCalledWith(expect.objectContaining({
+      model: "resolved-model-id"
+    }));
+  });
+});

package/packages/core/test/unit/core/withProvider.test.ts

@@ -0,0 +1,83 @@
+import { describe, it, expect, beforeEach } from "vitest";
+import { NodeLLM } from "../../../src/llm.js";
+import { config } from "../../../src/config.js";
+
+describe("NodeLLM.withProvider()", () => {
+  beforeEach(() => {
+    // Reset config to known state
+    config.openaiApiKey = "test-openai-key";
+    config.anthropicApiKey = "test-anthropic-key";
+  });
+
+  it("should create a scoped instance with the specified provider", () => {
+    const scoped = NodeLLM.withProvider("openai");
+    
+    // Scoped instance should be a different object
+    expect(scoped).not.toBe(NodeLLM);
+    
+    // Should have access to the same methods
+    expect(typeof scoped.chat).toBe("function");
+    expect(typeof scoped.configure).toBe("function");
+  });
+
+  it("should inherit global config by default", () => {
+    config.openaiApiKey = "global-openai-key";
+    
+    const scoped = NodeLLM.withProvider("openai");
+    
+    // Should have inherited the global config
+    expect(scoped.config.openaiApiKey).toBe("global-openai-key");
+  });
+
+  it("should accept scoped configuration overrides", () => {
+    config.anthropicApiKey = "global-anthropic-key";
+    
+    const scoped = NodeLLM.withProvider("anthropic", {
+      anthropicApiKey: "scoped-anthropic-key"
+    });
+    
+    // Scoped instance should use the override
+    expect(scoped.config.anthropicApiKey).toBe("scoped-anthropic-key");
+    
+    // Global config should remain unchanged
+    expect(config.anthropicApiKey).toBe("global-anthropic-key");
+  });
+
+  it("should support multiple scoped instances with different configs", () => {
+    const scoped1 = NodeLLM.withProvider("anthropic", {
+      anthropicApiKey: "key-1"
+    });
+    
+    const scoped2 = NodeLLM.withProvider("anthropic", {
+      anthropicApiKey: "key-2"
+    });
+    
+    // Each scope should have its own config
+    expect(scoped1.config.anthropicApiKey).toBe("key-1");
+    expect(scoped2.config.anthropicApiKey).toBe("key-2");
+    
+    // They should be different instances
+    expect(scoped1).not.toBe(scoped2);
+  });
+
+  it("should allow partial config overrides", () => {
+    config.openaiApiKey = "global-openai";
+    config.anthropicApiKey = "global-anthropic";
+    
+    const scoped = NodeLLM.withProvider("openai", {
+      openaiApiKey: "scoped-openai"
+      // anthropicApiKey not overridden
+    });
+    
+    expect(scoped.config.openaiApiKey).toBe("scoped-openai");
+    expect(scoped.config.anthropicApiKey).toBe("global-anthropic");
+  });
+
+  it("should support custom base URLs in scoped config", () => {
+    const scoped = NodeLLM.withProvider("anthropic", {
+      anthropicApiBase: "https://custom-proxy.example.com"
+    });
+    
+    expect(scoped.config.anthropicApiBase).toBe("https://custom-proxy.example.com");
+  });
+});

package/packages/core/test/unit/discovery/embedding.test.ts

@@ -0,0 +1,108 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+import { NodeLLM } from "../../../src/llm.js";
+import { EmbeddingRequest, EmbeddingResponse } from "../../../src/providers/Embedding.js";
+
+// Mock Provider
+const mockEmbed = vi.fn();
+const mockSupportsEmbeddings = vi.fn().mockReturnValue(true);
+
+const mockProvider = {
+  embed: mockEmbed,
+  capabilities: {
+    supportsEmbeddings: mockSupportsEmbeddings,
+  },
+};
+
+describe("NodeLLM Embeddings", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    NodeLLM.configure({ 
+      provider: mockProvider as any,
+      defaultEmbeddingModel: "text-embedding-3-small"
+    });
+  });
+
+  it("should call provider.embed with correct arguments", async () => {
+    mockEmbed.mockResolvedValue({
+      vectors: [[0.1, 0.2]],
+      model: "text-embedding-3-small",
+      input_tokens: 10,
+      dimensions: 2,
+    });
+
+    const input = "test input";
+    const response = await NodeLLM.embed(input);
+
+    expect(mockEmbed).toHaveBeenCalledWith({
+      input,
+      model: "text-embedding-3-small", // Default
+      dimensions: undefined,
+    });
+
+    expect(response.vectors).toEqual([[0.1, 0.2]]);
+    expect(response.input_tokens).toBe(10);
+    expect(response.dimensions).toBe(2);
+    expect(response.model).toBe("text-embedding-3-small");
+  });
+
+  it("should support batch input", async () => {
+    mockEmbed.mockResolvedValue({
+      vectors: [[0.1], [0.2]],
+      model: "text-embedding-3-small",
+      input_tokens: 20,
+      dimensions: 1,
+    });
+
+    const input = ["test1", "test2"];
+    await NodeLLM.embed(input);
+
+    expect(mockEmbed).toHaveBeenCalledWith(expect.objectContaining({
+      input,
+    }));
+  });
+
+  it("should allow overriding model and dimensions", async () => {
+    mockEmbed.mockResolvedValue({});
+
+    await NodeLLM.embed("test", { model: "custom-model", dimensions: 128 });
+
+    expect(mockEmbed).toHaveBeenCalledWith({
+      input: "test",
+      model: "custom-model",
+      dimensions: 128,
+    });
+  });
+
+  it("should use defaultEmbeddingModel if configured", async () => {
+    NodeLLM.configure({
+      provider: mockProvider as any,
+      defaultEmbeddingModel: "configured-default",
+    });
+
+    mockEmbed.mockResolvedValue({});
+    await NodeLLM.embed("test");
+
+    expect(mockEmbed).toHaveBeenCalledWith({
+      input: "test",
+      model: "configured-default",
+      dimensions: undefined,
+    });
+  });
+
+  it("should throw if provider not configured", async () => {
+    // @ts-ignore - hacking private property for test
+    NodeLLM["provider"] = undefined;
+    await expect(NodeLLM.embed("test")).rejects.toThrow("LLM provider not configured");
+  });
+
+  it("should throw if provider does not support embed", async () => {
+    NodeLLM.configure({ provider: {} as any });
+    await expect(NodeLLM.embed("test")).rejects.toThrow("Provider does not support embed");
+  });
+
+  it("should throw if model does not support embeddings", async () => {
+    mockSupportsEmbeddings.mockReturnValueOnce(false);
+    await expect(NodeLLM.embed("test", { model: "chat-model" }))
+      .rejects.toThrow("Model chat-model does not support embeddings");
+  });
+});

package/packages/core/test/unit/models/ModelRegistry.test.ts

@@ -0,0 +1,49 @@
+import { describe, it, expect } from "vitest";
+import { ModelRegistry } from "../../../src/models/ModelRegistry.js";
+
+describe("ModelRegistry - Filtered Sync Verification", () => {
+  it("should have loaded the filtered modelsData", () => {
+    const allModels = ModelRegistry.all();
+    expect(allModels.length).toBeGreaterThan(500);
+    
+    // Check that we only have supported providers
+    const providers = new Set(allModels.map(m => m.provider));
+    const supported = ['openai', 'anthropic', 'gemini', 'deepseek', 'openrouter', 'ollama'];
+    
+    // Convert Set to Array for easier comparison
+    const providerArray = Array.from(providers);
+    providerArray.forEach(p => {
+      expect(supported).toContain(p);
+    });
+  });
+
+  it("should find the new models like Claude 3.7", () => {
+    const claude37 = ModelRegistry.find("claude-3-7-sonnet-20250219");
+    expect(claude37).toBeDefined();
+    expect(claude37?.name).toBe("Claude Sonnet 3.7");
+    expect(claude37?.context_window).toBe(200000);
+  });
+
+  it("should find the new OpenAI models like gpt-4o", () => {
+    const gpt4o = ModelRegistry.find("gpt-4o");
+    expect(gpt4o).toBeDefined();
+    expect(gpt4o?.provider).toBe("openai");
+  });
+
+  it("should calculate costs using the new pricing data", () => {
+    // GPT-4o pricing: $2.50 / $10.00
+    const usage = { input_tokens: 1_000_000, output_tokens: 1_000_000, total_tokens: 2_000_000 };
+    const cost = ModelRegistry.calculateCost(usage, "gpt-4o", "openai");
+    
+    expect(cost.cost).toBe(12.5); // 2.5 + 10
+  });
+
+  it("should still have Ollama models", () => {
+    const llama = ModelRegistry.find("llama3");
+    expect(llama).toBeDefined();
+    expect(llama?.provider).toBe("ollama");
+  });
+});
+ bitumen: 102
+ bitumen: 102
+ bitumen: 102