@node-llm/testing 0.1.0

package/test/cassettes/testunitdxtestts-vcr-feature-5-6-dx-sugar-auto-naming-automatically-names-and-records-cassettes.json

@@ -0,0 +1,33 @@
+{
+  "name": "test/unit/dx.test.ts > VCR Feature 5 & 6: DX Sugar & Auto-Naming > Automatically names and records cassettes",
+  "version": "1.0",
+  "metadata": {
+    "recordedAt": "2026-01-26T10:42:20.369Z",
+    "duration": 7
+  },
+  "interactions": [
+    {
+      "method": "chat",
+      "request": {
+        "model": "mock-model",
+        "messages": [
+          {
+            "role": "user",
+            "content": "Sugar test"
+          }
+        ],
+        "max_tokens": 4096,
+        "headers": {},
+        "requestTimeout": 30000
+      },
+      "response": {
+        "content": "Response to Sugar test",
+        "usage": {
+          "input_tokens": 10,
+          "output_tokens": 10,
+          "total_tokens": 20
+        }
+      }
+    }
+  ]
+}

package/test/cassettes/vcr-feature-5-6-dx-sugar-auto-naming-automatically-names-and-records-cassettes.json

@@ -0,0 +1,28 @@
+{
+  "name": "VCR Feature 5 & 6: DX Sugar & Auto-Naming > Automatically names and records cassettes",
+  "interactions": [
+    {
+      "method": "chat",
+      "request": {
+        "model": "mock-model",
+        "messages": [
+          {
+            "role": "user",
+            "content": "Sugar test"
+          }
+        ],
+        "max_tokens": 4096,
+        "headers": {},
+        "requestTimeout": 30000
+      },
+      "response": {
+        "content": "Response to Sugar test",
+        "usage": {
+          "input_tokens": 10,
+          "output_tokens": 10,
+          "total_tokens": 20
+        }
+      }
+    }
+  ]
+}

package/test/cassettes/vcr-streaming.json

@@ -0,0 +1,17 @@
+{
+  "name": "vcr-streaming",
+  "version": "1.0",
+  "interactions": [
+    {
+      "method": "chat",
+      "request": {
+        "messages": [{ "role": "user", "content": "Tell me a short story" }]
+      },
+      "response": {
+        "content": "Once upon a time, there was a small robot who learned to dream.",
+        "tool_calls": [],
+        "usage": { "input_tokens": 10, "output_tokens": 15, "total_tokens": 25 }
+      }
+    }
+  ]
+}

package/test/helpers/MockProvider.ts

@@ -0,0 +1,75 @@
+import { vi, type Mock } from "vitest";
+import {
+  BaseProvider,
+  ChatRequest,
+  ChatResponse,
+  ProviderCapabilities,
+  EmbeddingResponse,
+  ImageResponse,
+  TranscriptionResponse,
+  ModerationResponse,
+  EmbeddingRequest,
+  ImageRequest,
+  TranscriptionRequest,
+  ModerationRequest
+} from "@node-llm/core";
+
+export class MockProvider extends BaseProvider {
+  get id() {
+    return "mock-provider";
+  }
+
+  apiBase() {
+    return "http://mock";
+  }
+  headers() {
+    return {};
+  }
+  protected providerName() {
+    return "mock-provider";
+  }
+
+  defaultModel() {
+    return "mock-model";
+  }
+
+  capabilities: ProviderCapabilities = {
+    supportsVision: () => true,
+    supportsTools: () => true,
+    supportsStructuredOutput: () => true,
+    supportsEmbeddings: () => true,
+    supportsImageGeneration: () => true,
+    supportsTranscription: () => true,
+    supportsModeration: () => true,
+    supportsReasoning: () => true,
+    supportsDeveloperRole: () => true,
+    getContextWindow: () => 128000
+  };
+
+  chat: Mock<(req: ChatRequest) => Promise<ChatResponse>> = vi.fn(
+    async (req: ChatRequest): Promise<ChatResponse> => {
+      const lastMsg = req.messages[req.messages.length - 1];
+      let contentStr = "nothing";
+
+      if (lastMsg && lastMsg.content) {
+        if (typeof lastMsg.content === "string") {
+          contentStr = lastMsg.content;
+        } else if (Array.isArray(lastMsg.content)) {
+          contentStr = lastMsg.content.map((p) => (p.type === "text" ? p.text : "")).join("");
+        } else {
+          contentStr = String(lastMsg.content);
+        }
+      }
+
+      return {
+        content: `Response to ${contentStr}`,
+        usage: { input_tokens: 10, output_tokens: 10, total_tokens: 20 }
+      };
+    }
+  );
+
+  embed: Mock<(req: EmbeddingRequest) => Promise<EmbeddingResponse>> = vi.fn();
+  paint: Mock<(req: ImageRequest) => Promise<ImageResponse>> = vi.fn();
+  transcribe: Mock<(req: TranscriptionRequest) => Promise<TranscriptionResponse>> = vi.fn();
+  moderate: Mock<(req: ModerationRequest) => Promise<ModerationResponse>> = vi.fn();
+}

package/test/unit/ci.test.ts

@@ -0,0 +1,36 @@
+import { test, expect, describe, beforeEach, afterEach } from "vitest";
+import { setupVCR } from "../../src/vcr.js";
+import fs from "node:fs";
+import path from "node:path";
+
+describe("VCR Feature 8: CI-Safe Replay", () => {
+  const CASSETTE_NAME = "ci-test-cassette";
+  const CASSETTE_DIR = path.join(__dirname, "../cassettes");
+  const CASSETTE_PATH = path.join(CASSETTE_DIR, `${CASSETTE_NAME}.json`);
+  const originalCI = process.env.CI;
+
+  beforeEach(() => {
+    if (fs.existsSync(CASSETTE_PATH)) fs.unlinkSync(CASSETTE_PATH);
+  });
+
+  afterEach(() => {
+    process.env.CI = originalCI;
+  });
+
+  test("Throws error if cassette is missing in CI", () => {
+    process.env.CI = "true";
+
+    // Using auto mode when file doesn't exist should throw in CI
+    expect(() => setupVCR(CASSETTE_NAME, { mode: "auto", cassettesDir: CASSETTE_DIR })).toThrow(
+      /Cassette missing in CI/
+    );
+  });
+
+  test("Throws error if record mode is explicitly set in CI", () => {
+    process.env.CI = "true";
+
+    expect(() => setupVCR(CASSETTE_NAME, { mode: "record", cassettesDir: CASSETTE_DIR })).toThrow(
+      /Recording cassettes is not allowed in CI/
+    );
+  });
+});

package/test/unit/dx.test.ts

@@ -0,0 +1,86 @@
+import { test, expect, describe, beforeEach, afterEach } from "vitest";
+import { withVCR } from "../../src/vcr.js";
+import { NodeLLM, providerRegistry } from "@node-llm/core";
+import fs from "node:fs";
+import path from "node:path";
+import { MockProvider } from "../helpers/MockProvider.js";
+
+describe("VCR Feature 5 & 6: DX Sugar & Auto-Naming", () => {
+  let mock: MockProvider;
+  const CASSETTE_DIR = path.join(__dirname, "../cassettes");
+
+  beforeEach(() => {
+    mock = new MockProvider();
+    providerRegistry.register("mock-provider", () => mock);
+  });
+
+  afterEach(() => {
+    providerRegistry.setInterceptor(undefined);
+  });
+
+  test(
+    "Automatically names and records cassettes",
+    withVCR({ cassettesDir: CASSETTE_DIR }, async () => {
+      const llm = NodeLLM.withProvider("mock-provider");
+      await llm.chat().ask("Sugar test");
+
+      // The cassette name should be derived from the test title
+    })
+  );
+
+  test("Verifying the file exists after withVCR", async () => {
+    // Note: Since withVCR above runs, we check it here
+    const files = fs.readdirSync(CASSETTE_DIR);
+    const found = files.some((f) => f.includes("automatically-names-and-records-cassettes"));
+    expect(found).toBe(true);
+  });
+
+  test(
+    "Allows explicit naming",
+    withVCR("explicit-sugar-test", { cassettesDir: CASSETTE_DIR }, async () => {
+      const llm = NodeLLM.withProvider("mock-provider");
+      await llm.chat().ask("Explicit test");
+    })
+  );
+
+  test("Verifying explicit file existence", async () => {
+    const filePath = path.join(CASSETTE_DIR, "explicit-sugar-test.json");
+    expect(fs.existsSync(filePath)).toBe(true);
+  });
+
+  test("Respects global configuration", async () => {
+    // 1. Set global config
+    const { configureVCR } = await import("../../src/vcr.js");
+    configureVCR({ sensitiveKeys: ["global_secret"] });
+
+    // 2. Run a test that relies on this global config merging
+    await withVCR("global-config-test", { cassettesDir: CASSETTE_DIR }, async () => {
+      const llm = NodeLLM.withProvider("mock-provider");
+      await llm.chat().ask("global test");
+    })();
+
+    // Cleanup
+    configureVCR({});
+  });
+
+  test("Respects global cassettesDir configuration", async () => {
+    // 1. Set global cassettesDir
+    const { configureVCR } = await import("../../src/vcr.js");
+    const GLOBAL_DIR = path.join(CASSETTE_DIR, "global-custom-dir");
+    configureVCR({ cassettesDir: GLOBAL_DIR });
+
+    // 2. Run a test relying on global dir (explicit record mode to create cassette)
+    await withVCR("global-dir-test", { mode: "record" }, async () => {
+      const llm = NodeLLM.withProvider("mock-provider");
+      await llm.chat().ask("global dir test");
+    })();
+
+    // 3. Verify file exists in global dir
+    const filePath = path.join(GLOBAL_DIR, "global-dir-test.json");
+    expect(fs.existsSync(filePath)).toBe(true);
+
+    // Cleanup
+    if (fs.existsSync(GLOBAL_DIR)) fs.rmSync(GLOBAL_DIR, { recursive: true, force: true });
+    configureVCR({});
+  });
+});

package/test/unit/mocker-debug.test.ts

@@ -0,0 +1,68 @@
+import { test, expect, describe, beforeEach, afterEach } from "vitest";
+import { mockLLM } from "../../src/Mocker.js";
+import { NodeLLM, providerRegistry } from "@node-llm/core";
+import { MockProvider } from "../helpers/MockProvider.js";
+
+describe("Mocker: Debug Information", () => {
+  let mocker: ReturnType<typeof mockLLM>;
+
+  beforeEach(() => {
+    providerRegistry.register("mock-provider", () => new MockProvider());
+    mocker = mockLLM();
+  });
+
+  afterEach(() => {
+    mocker.clear();
+  });
+
+  test("getDebugInfo returns total mock count", () => {
+    mocker.chat("Hello").respond("Hi");
+    mocker.chat("Bye").respond("Goodbye");
+    mocker.embed("test").respond({ vectors: [[0.1]] });
+
+    const debug = mocker.getDebugInfo();
+
+    expect(debug.totalMocks).toBe(3);
+    expect(debug.methods).toContain("chat");
+    expect(debug.methods).toContain("embed");
+  });
+
+  test("getDebugInfo lists unique methods", () => {
+    mocker.chat("First").respond("1");
+    mocker.chat("Second").respond("2");
+    mocker.chat("Third").respond("3");
+    mocker.embed("input").respond({ vectors: [[0.1]] });
+    mocker.paint("prompt").respond({ url: "http://test.com/img.png" });
+
+    const debug = mocker.getDebugInfo();
+
+    expect(debug.totalMocks).toBe(5);
+    expect(debug.methods).toEqual(expect.arrayContaining(["chat", "embed", "paint"]));
+    expect(debug.methods.length).toBe(3); // Unique methods only
+  });
+
+  test("getDebugInfo with no mocks", () => {
+    const debug = mocker.getDebugInfo();
+
+    expect(debug.totalMocks).toBe(0);
+    expect(debug.methods).toEqual([]);
+  });
+
+  test("getDebugInfo works with all mock types", () => {
+    mocker.chat("test").respond("chat response");
+    mocker.embed("test").respond({ vectors: [[0.1]] });
+    mocker.paint("test").respond({ url: "http://test.com/img.png" });
+    mocker.transcribe("test.mp3").respond("transcript");
+    mocker.moderate("test").respond({ results: [] });
+
+    const debug = mocker.getDebugInfo();
+
+    const methods = debug.methods;
+    expect(methods).toContain("chat");
+    expect(methods).toContain("embed");
+    expect(methods).toContain("paint");
+    expect(methods).toContain("transcribe");
+    expect(methods).toContain("moderate");
+    expect(debug.totalMocks).toBe(5);
+  });
+});

package/test/unit/mocker.test.ts

@@ -0,0 +1,46 @@
+import { test, expect, describe, beforeEach, afterEach } from "vitest";
+import { mockLLM } from "../../src/Mocker.js";
+import { NodeLLM, providerRegistry } from "@node-llm/core";
+import { MockProvider } from "../helpers/MockProvider.js";
+
+describe("Mocker Feature 10: Declarative Mocks", () => {
+  let mocker: ReturnType<typeof mockLLM>;
+
+  beforeEach(() => {
+    // Register a real provider to verify pass-through vs mock
+    providerRegistry.register("mock-provider", () => new MockProvider());
+    mocker = mockLLM();
+  });
+
+  afterEach(() => {
+    mocker.clear();
+  });
+
+  test("Mocks exact chat queries", async () => {
+    mocker.chat("Ping").respond("Pong");
+
+    const llm = NodeLLM.withProvider("mock-provider");
+    const res = await llm.chat().ask("Ping");
+
+    expect(res.content).toBe("Pong");
+  });
+
+  test("Mocks using Regex patterns", async () => {
+    mocker.chat(/hello/i).respond("Greetings!");
+
+    const llm = NodeLLM.withProvider("mock-provider");
+    const res = await llm.chat().ask("HHeelllooo"); // Should fail
+    const res2 = await llm.chat().ask("Hello there");
+
+    expect(res2.content).toBe("Greetings!");
+  });
+
+  test("Simulates Provider Errors", async () => {
+    mocker.chat("Fail me").respond({
+      error: new Error("Rate limit exceeded")
+    });
+
+    const llm = NodeLLM.withProvider("mock-provider");
+    await expect(llm.chat().ask("Fail me")).rejects.toThrow("Rate limit exceeded");
+  });
+});

package/test/unit/multimodal.test.ts

@@ -0,0 +1,46 @@
+import { test, expect, describe, beforeEach, afterEach } from "vitest";
+import { mockLLM } from "../../src/Mocker.js";
+import { NodeLLM, providerRegistry } from "@node-llm/core";
+import { MockProvider } from "../helpers/MockProvider.js";
+
+describe("Mocker Feature: Explicit Multimodal API", () => {
+  let mocker: ReturnType<typeof mockLLM>;
+
+  beforeEach(() => {
+    providerRegistry.register("mock-provider", () => new MockProvider());
+    mocker = mockLLM();
+  });
+
+  afterEach(() => {
+    mocker.clear();
+  });
+
+  test("Uses explicit .paint() mock", async () => {
+    mocker.paint(/scary cat/i).respond({ url: "http://mock.com/scary-cat.png" });
+
+    const llm = NodeLLM.withProvider("mock-provider");
+    const res = await (llm.provider as any).paint({ prompt: "A very scary cat" });
+
+    expect(res.url).toBe("http://mock.com/scary-cat.png");
+  });
+
+  test("Uses explicit .transcribe() mock", async () => {
+    mocker.transcribe("audio.mp3").respond("This is the transcript");
+
+    const llm = NodeLLM.withProvider("mock-provider");
+    const res = await (llm.provider as any).transcribe({ file: "audio.mp3" });
+
+    expect(res.text).toBe("This is the transcript");
+  });
+
+  test("Uses explicit .moderate() mock", async () => {
+    mocker.moderate(/hate/i).respond({
+      results: [{ flagged: true, categories: { hate: true }, category_scores: { hate: 0.99 } }]
+    });
+
+    const llm = NodeLLM.withProvider("mock-provider");
+    const res = await (llm.provider as any).moderate({ input: "I hate mushrooms" });
+
+    expect(res.results[0].flagged).toBe(true);
+  });
+});

package/test/unit/scoping.test.ts

@@ -0,0 +1,54 @@
+import { test, expect, describe, beforeEach, afterEach } from "vitest";
+import { withVCR, describeVCR } from "../../src/vcr.js";
+import { NodeLLM, providerRegistry } from "@node-llm/core";
+import { MockProvider } from "../helpers/MockProvider.js";
+import fs from "node:fs";
+import path from "node:path";
+
+describe("VCR Feature 12: Hierarchical Scoping", () => {
+  // Use absolute path relative to this test file
+  const CUSTOM_DIR = path.join(__dirname, "../cassettes/custom-scope-test");
+  const LEVEL_1 = "Authentication";
+  const LEVEL_2 = "Login Flow";
+  const TEST_NAME = "Successful Login";
+
+  beforeEach(() => {
+    if (fs.existsSync(CUSTOM_DIR)) fs.rmSync(CUSTOM_DIR, { recursive: true, force: true });
+    providerRegistry.register("mock-provider", () => new MockProvider());
+  });
+
+  afterEach(() => {
+    // Clean up after test
+    if (fs.existsSync(CUSTOM_DIR)) fs.rmSync(CUSTOM_DIR, { recursive: true, force: true });
+  });
+
+  test("Organizes cassettes into nested subfolders", async () => {
+    // We use process.env to set the base dir for this test
+    process.env.VCR_CASSETTE_DIR = CUSTOM_DIR;
+
+    await describeVCR(LEVEL_1, () => {
+      return describeVCR(LEVEL_2, async () => {
+        // Use explicit mode: "record" to test cassette creation in temp dir
+        const testFn = withVCR(TEST_NAME, { mode: "record" }, async () => {
+          const llm = NodeLLM.withProvider("mock-provider");
+          await llm.chat().ask("Trigger record");
+        });
+
+        await testFn();
+      });
+    });
+
+    // Verify the path exists
+    const expectedPath = path.join(
+      CUSTOM_DIR,
+      "authentication",
+      "login-flow",
+      "successful-login.json"
+    );
+
+    expect(fs.existsSync(expectedPath)).toBe(true);
+
+    // Clean up env
+    delete process.env.VCR_CASSETTE_DIR;
+  });
+});

package/test/unit/scrubbing.test.ts

@@ -0,0 +1,110 @@
+import { test, expect, describe, beforeEach, afterEach } from "vitest";
+import { setupVCR } from "../../src/vcr.js";
+import { NodeLLM, providerRegistry } from "@node-llm/core";
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import { MockProvider } from "../helpers/MockProvider.js";
+
+describe("VCR Feature 4: Automatic Scrubbing", () => {
+  const CASSETTE_NAME = "scrub-test";
+  // Use temp directory for scrubbing tests to avoid re-recording committed cassettes
+  let CASSETTE_DIR: string;
+  let CASSETTE_PATH: string;
+  let mock: MockProvider;
+
+  beforeEach(() => {
+    CASSETTE_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "vcr-scrub-test-"));
+    CASSETTE_PATH = path.join(CASSETTE_DIR, `${CASSETTE_NAME}.json`);
+    mock = new MockProvider();
+    providerRegistry.register("mock-provider", () => mock);
+  });
+
+  afterEach(() => {
+    providerRegistry.setInterceptor(undefined);
+    // Clean up temp directory
+    if (fs.existsSync(CASSETTE_DIR)) {
+      fs.rmSync(CASSETTE_DIR, { recursive: true, force: true });
+    }
+  });
+
+  test("Automatically scrubs API keys and sensitive JSON keys", async () => {
+    const vcr = setupVCR(CASSETTE_NAME, { mode: "record", cassettesDir: CASSETTE_DIR });
+    const llm = NodeLLM.withProvider("mock-provider");
+
+    // 1. Trigger request with secrets
+    await llm.chat().ask("My key is sk-abcdef1234567890abcdef1234567890");
+
+    await vcr.stop();
+
+    // 2. Read back from disk and verify redaction
+    const raw = fs.readFileSync(CASSETTE_PATH, "utf-8");
+    expect(raw).not.toContain("sk-abcdef");
+    expect(raw).toContain("[REDACTED]");
+
+    // 3. Verify that we DID NOT redact token counts (because they are numbers)
+    expect(raw).toContain('"total_tokens": 20');
+  });
+
+  test("Allows custom scrubbing hooks", async () => {
+    const vcr = setupVCR(CASSETTE_NAME, {
+      mode: "record",
+      cassettesDir: CASSETTE_DIR,
+      scrub: (data: unknown) => {
+        // Deep string replacement on the whole interaction object
+        return JSON.parse(JSON.stringify(data).replace(/sensitive-info/g, "XXXX"));
+      }
+    });
+
+    const llm = NodeLLM.withProvider("mock-provider");
+    await llm.chat().ask("This contains sensitive-info");
+    await vcr.stop();
+
+    const raw = fs.readFileSync(CASSETTE_PATH, "utf-8");
+    expect(raw).toContain("XXXX");
+    expect(raw).not.toContain("sensitive-info");
+  });
+
+  test("Supports declarative custom keys and patterns via config", async () => {
+    const vcr = setupVCR("custom-scrub-config", {
+      mode: "record",
+      cassettesDir: CASSETTE_DIR,
+      sensitiveKeys: ["user_email", "internal_id"],
+      sensitivePatterns: [/secret-project-[a-z]+/g]
+    });
+
+    const llm = NodeLLM.withProvider("mock-provider");
+    await llm.chat().ask("status of secret-project-omega");
+
+    await vcr.stop();
+
+    const cassettePath = path.join(CASSETTE_DIR, "custom-scrub-config.json");
+    const raw = fs.readFileSync(cassettePath, "utf-8");
+
+    // Verify Pattern Scrubbing
+    expect(raw).not.toContain("secret-project-omega");
+    expect(raw).toContain("[REDACTED]");
+  });
+
+  test("Retains default scrubbing when custom config is provided", async () => {
+    const vcr = setupVCR("defaults-plus-custom", {
+      mode: "record",
+      cassettesDir: CASSETTE_DIR,
+      sensitiveKeys: ["custom_field"]
+    });
+
+    const llm = NodeLLM.withProvider("mock-provider");
+
+    // We send a standard fake key (matches default pattern) AND a custom field
+    await llm.chat().ask("key sk-123456789012345678901234567890 plus custom_field");
+
+    await vcr.stop();
+
+    const path = `${CASSETTE_DIR}/defaults-plus-custom.json`;
+    const raw = fs.readFileSync(path, "utf-8");
+
+    // Default pattern should still work
+    expect(raw).not.toContain("sk-1234567890");
+    expect(raw).toContain("[REDACTED]");
+  });
+});

package/test/unit/streaming.test.ts

@@ -0,0 +1,51 @@
+import { test, expect, describe, beforeEach, afterEach } from "vitest";
+import { mockLLM } from "../../src/Mocker.js";
+import { NodeLLM, providerRegistry } from "@node-llm/core";
+import { MockProvider } from "../helpers/MockProvider.js";
+
+describe("Mocker Feature 11: Streaming Mocks", () => {
+  let mocker: ReturnType<typeof mockLLM>;
+
+  beforeEach(() => {
+    providerRegistry.register("mock-provider", () => new MockProvider());
+    mocker = mockLLM();
+  });
+
+  afterEach(() => {
+    mocker.clear();
+  });
+
+  test("Mocks a streaming response token by token", async () => {
+    const chunks = ["Once ", "upon ", "a ", "time."];
+    mocker.chat("Tell me a story").stream(chunks);
+
+    const llm = NodeLLM.withProvider("mock-provider");
+    const stream = llm.chat("mock-model").stream("Tell me a story");
+
+    const received: string[] = [];
+    for await (const chunk of stream) {
+      received.push(chunk.content);
+    }
+
+    expect(received).toEqual(chunks);
+  });
+
+  test("Mocks a streaming response with manual ChatChunk objects", async () => {
+    const chunks = [
+      { content: "Hello", done: false },
+      { content: " world", done: true }
+    ];
+    mocker.chat("Greeting").stream(chunks);
+
+    const llm = NodeLLM.withProvider("mock-provider");
+    const stream = llm.chat("mock-model").stream("Greeting");
+
+    const received: any[] = [];
+    for await (const chunk of stream) {
+      received.push(chunk);
+    }
+
+    expect(received[0].content).toBe("Hello");
+    expect(received[1].done).toBe(true);
+  });
+});