@node-llm/testing 0.1.0

package/dist/vcr.js

@@ -0,0 +1,291 @@
+import { providerRegistry } from "@node-llm/core";
+import fs from "node:fs";
+import path from "node:path";
+import { Scrubber } from "./Scrubber.js";
+// Internal state for nested scoping (Feature 12)
+const currentVCRScopes = [];
+// Try to import Vitest's expect to get test state
+let vitestExpect;
+try {
+    // @ts-ignore
+    import("vitest").then((m) => {
+        vitestExpect = m.expect;
+    });
+}
+catch {
+    // Not in vitest env
+}
+export class VCR {
+    cassette;
+    interactionIndex = 0;
+    mode;
+    filePath;
+    scrubber;
+    recordStartTime = 0;
+    constructor(name, options = {}) {
+        // 1. Merge Global Defaults
+        // Explicitly merge arrays to avoid overwriting global sensitive keys/patterns
+        const mergedOptions = {
+            ...globalVCROptions,
+            ...options,
+            sensitivePatterns: [
+                ...(globalVCROptions.sensitivePatterns || []),
+                ...(options.sensitivePatterns || [])
+            ],
+            sensitiveKeys: [...(globalVCROptions.sensitiveKeys || []), ...(options.sensitiveKeys || [])]
+        };
+        // 2. Resolve Base Directory (Env -> Option -> Default)
+        // Rails-inspired organization: cassettes belong inside the test folder
+        const baseDir = mergedOptions.cassettesDir || process.env.VCR_CASSETTE_DIR || "test/cassettes";
+        // 2. Resolve Hierarchical Scopes
+        const scopes = [];
+        if (Array.isArray(mergedOptions.scope)) {
+            scopes.push(...mergedOptions.scope);
+        }
+        else if (mergedOptions.scope) {
+            scopes.push(mergedOptions.scope);
+        }
+        else {
+            scopes.push(...currentVCRScopes);
+        }
+        // 3. Construct Final Directory Path
+        const targetDir = path.join(baseDir, ...scopes.map((s) => this.slugify(s)));
+        // Robust path resolution: Never join CWD if the target is already absolute
+        if (path.isAbsolute(targetDir)) {
+            this.filePath = path.join(targetDir, `${this.slugify(name)}.json`);
+        }
+        else {
+            this.filePath = path.join(process.cwd(), targetDir, `${this.slugify(name)}.json`);
+        }
+        const initialMode = mergedOptions.mode || process.env.VCR_MODE || "auto";
+        const isCI = !!process.env.CI;
+        const exists = fs.existsSync(this.filePath);
+        // CI Enforcement
+        if (isCI) {
+            if (initialMode === "record") {
+                throw new Error(`VCR[${name}]: Recording cassettes is not allowed in CI.`);
+            }
+            if (initialMode === "auto" && !exists) {
+                throw new Error(`VCR[${name}]: Cassette missing in CI. Run locally to generate ${this.filePath}`);
+            }
+        }
+        // Mode Resolution:
+        // - "record": Always record (overwrites existing cassette)
+        // - "replay": Always replay (fails if cassette missing)
+        // - "auto": Replay if exists, otherwise FAIL (requires explicit record)
+        // - "passthrough": No VCR, make real calls
+        if (initialMode === "auto") {
+            if (exists) {
+                this.mode = "replay";
+            }
+            else {
+                throw new Error(`VCR[${name}]: Cassette not found at ${this.filePath}. ` +
+                    `Use mode: "record" to create it, then switch back to "auto" or "replay".`);
+            }
+        }
+        else {
+            this.mode = initialMode;
+        }
+        this.scrubber = new Scrubber({
+            customScrubber: mergedOptions.scrub,
+            sensitivePatterns: mergedOptions.sensitivePatterns,
+            sensitiveKeys: mergedOptions.sensitiveKeys
+        });
+        if (this.mode === "replay") {
+            if (!exists) {
+                throw new Error(`VCR[${name}]: Cassette not found at ${this.filePath}`);
+            }
+            this.cassette = JSON.parse(fs.readFileSync(this.filePath, "utf-8"));
+        }
+        else {
+            this.cassette = {
+                name,
+                version: "1.0",
+                metadata: {
+                    recordedAt: new Date().toISOString()
+                },
+                interactions: []
+            };
+            this.recordStartTime = Date.now();
+        }
+    }
+    get currentMode() {
+        return this.mode;
+    }
+    async stop() {
+        if (this.mode === "record" && this.interactionsCount > 0) {
+            const dir = path.dirname(this.filePath);
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir, { recursive: true });
+            }
+            // Update metadata with duration
+            const duration = Date.now() - this.recordStartTime;
+            if (this.cassette.metadata) {
+                this.cassette.metadata.duration = duration;
+            }
+            fs.writeFileSync(this.filePath, JSON.stringify(this.cassette, null, 2));
+        }
+        providerRegistry.setInterceptor(undefined);
+    }
+    get interactionsCount() {
+        return this.cassette.interactions.length;
+    }
+    async execute(method, originalMethod, request) {
+        if (this.mode === "replay") {
+            const interaction = this.cassette.interactions[this.interactionIndex++];
+            if (!interaction) {
+                throw new Error(`VCR[${this.cassette.name}]: No more interactions for ${method}`);
+            }
+            return interaction.response;
+        }
+        const response = await originalMethod(request);
+        if (this.mode === "record") {
+            const interaction = this.scrubber.scrub({
+                method,
+                request: this.clone(request),
+                response: this.clone(response)
+            });
+            this.cassette.interactions.push(interaction);
+        }
+        return response;
+    }
+    async *executeStream(method, originalMethod, request) {
+        if (this.mode === "replay") {
+            const interaction = this.cassette.interactions[this.interactionIndex++];
+            if (!interaction || !interaction.chunks) {
+                throw new Error(`VCR[${this.cassette.name}]: No streaming interactions found`);
+            }
+            for (const chunk of interaction.chunks) {
+                yield chunk;
+            }
+            return;
+        }
+        const stream = originalMethod(request);
+        const chunks = [];
+        for await (const chunk of stream) {
+            if (this.mode === "record")
+                chunks.push(this.clone(chunk));
+            yield chunk;
+        }
+        if (this.mode === "record") {
+            const interaction = this.scrubber.scrub({
+                method,
+                request: this.clone(request),
+                response: null,
+                chunks: chunks.map((c) => this.clone(c))
+            });
+            this.cassette.interactions.push(interaction);
+        }
+    }
+    clone(obj) {
+        try {
+            return JSON.parse(JSON.stringify(obj));
+        }
+        catch {
+            return obj;
+        }
+    }
+    slugify(text) {
+        return text
+            .toString()
+            .toLowerCase()
+            .trim()
+            .replace(/\s+/g, "-")
+            .replace(/[^\w-]+/g, "")
+            .replace(/--+/g, "-");
+    }
+}
+const EXECUTION_METHODS = ["chat", "stream", "paint", "transcribe", "moderate", "embed"];
+export function setupVCR(name, options = {}) {
+    const vcr = new VCR(name, options);
+    providerRegistry.setInterceptor((provider) => {
+        return new Proxy(provider, {
+            get(target, prop, receiver) {
+                const originalValue = Reflect.get(target, prop, receiver);
+                const method = prop.toString();
+                if (typeof originalValue === "function" && EXECUTION_METHODS.includes(method)) {
+                    return function (...args) {
+                        if (method === "stream") {
+                            return vcr.executeStream(method, originalValue.bind(target), args[0]);
+                        }
+                        return vcr.execute(method, originalValue.bind(target), args[0]);
+                    };
+                }
+                return originalValue;
+            }
+        });
+    });
+    return vcr;
+}
+export function withVCR(...args) {
+    // Capture scopes at initialization time
+    const capturedScopes = [...currentVCRScopes];
+    return async function () {
+        let name;
+        let options = {};
+        let fn;
+        if (typeof args[0] === "function") {
+            fn = args[0];
+        }
+        else if (typeof args[0] === "string") {
+            name = args[0];
+            if (typeof args[1] === "function") {
+                fn = args[1];
+            }
+            else {
+                options = args[1] || {};
+                fn = args[2];
+            }
+        }
+        else {
+            options = args[0] || {};
+            fn = args[1];
+        }
+        // Pass captured inherited scopes if not explicitly overridden
+        if (capturedScopes.length > 0 && !options.scope) {
+            options.scope = capturedScopes;
+        }
+        if (!name && vitestExpect) {
+            const state = vitestExpect.getState();
+            name = state.currentTestName || "unnamed-test";
+        }
+        if (!name)
+            throw new Error("VCR: Could not determine cassette name.");
+        const vcr = setupVCR(name, options);
+        try {
+            await fn();
+        }
+        finally {
+            await vcr.stop();
+        }
+    };
+}
+// Global configuration for VCR
+let globalVCROptions = {};
+export function configureVCR(options) {
+    globalVCROptions = { ...globalVCROptions, ...options };
+}
+export function resetVCRConfig() {
+    globalVCROptions = {};
+}
+/**
+ * Organizes cassettes by hierarchical subdirectories.
+ */
+export function describeVCR(name, fn) {
+    currentVCRScopes.push(name);
+    const finish = () => {
+        currentVCRScopes.pop();
+    };
+    try {
+        const result = fn();
+        if (result instanceof Promise) {
+            return result.finally(finish);
+        }
+        finish();
+        return result;
+    }
+    catch (err) {
+        finish();
+        throw err;
+    }
+}

package/package.json

@@ -0,0 +1,19 @@
+{
+  "name": "@node-llm/testing",
+  "version": "0.01.0",
+  "description": "Deterministic testing for NodeLLM powered AI systems",
+  "main": "dist/index.js",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "test": "vitest run",
+    "test:record": "VCR_MODE=record vitest run",
+    "test:vitest": "vitest run"
+  },
+  "dependencies": {},
+  "devDependencies": {
+    "@node-llm/core": "workspace:*",
+    "vitest": "^1.0.0",
+    "typescript": "^5.0.0"
+  }
+}

package/src/Mocker.ts

@@ -0,0 +1,311 @@
+import {
+  Provider,
+  providerRegistry,
+  ChatResponse,
+  EmbeddingResponse,
+  ChatRequest,
+  EmbeddingRequest,
+  ImageRequest,
+  TranscriptionRequest,
+  ModerationRequest,
+  ImageResponse,
+  TranscriptionResponse,
+  ModerationResponse,
+  ChatChunk,
+  ToolCall,
+  ModerationResult,
+  MessageContent
+} from "@node-llm/core";
+
+export interface MockResponse {
+  content?: string | null;
+  tool_calls?: ToolCall[];
+  usage?: {
+    input_tokens: number;
+    output_tokens: number;
+    total_tokens: number;
+  };
+  error?: Error;
+  finish_reason?: string | null;
+  chunks?: string[] | ChatChunk[];
+  vectors?: number[][];
+  url?: string;
+  data?: string;
+  text?: string;
+  results?: ModerationResult[];
+  revised_prompt?: string;
+  id?: string;
+}
+
+export type MockMatcher = (request: unknown) => boolean;
+
+export interface MockDefinition {
+  method: string;
+  match: MockMatcher;
+  response: MockResponse | ((request: unknown) => MockResponse);
+}
+
+/**
+ * Debug information about defined mocks.
+ */
+export interface MockerDebugInfo {
+  totalMocks: number;
+  methods: string[];
+}
+
+const EXECUTION_METHODS = ["chat", "stream", "paint", "transcribe", "moderate", "embed"];
+
+export class Mocker {
+  private mocks: MockDefinition[] = [];
+  public strict = false;
+
+  constructor() {
+    this.setupInterceptor();
+  }
+
+  public chat(query?: string | RegExp): this {
+    return this.addMock("chat", (req: unknown) => {
+      const chatReq = req as ChatRequest;
+      if (!query) return true;
+      const lastMessage = [...chatReq.messages].reverse().find((m) => m.role === "user");
+      if (!lastMessage) return false;
+
+      const content = this.getContentString(lastMessage.content);
+      if (typeof query === "string") return content === query;
+      if (query instanceof RegExp) return typeof content === "string" && query.test(content);
+      return false;
+    });
+  }
+
+  public stream(chunks: string[] | ChatChunk[]): this {
+    const lastMock = this.mocks[this.mocks.length - 1];
+    if (!lastMock || (lastMock.method !== "chat" && lastMock.method !== "stream")) {
+      throw new Error("Mocker: .stream() must follow a .chat() or .addMock('stream') definition.");
+    }
+    lastMock.method = "stream";
+    lastMock.response = { chunks };
+    return this;
+  }
+
+  public placeholder(query: string | RegExp): this {
+    return this.addMock("chat", (req: unknown) => {
+      const chatReq = req as ChatRequest;
+      return chatReq.messages.some((m) => {
+        const content = this.getContentString(m.content);
+        if (typeof query === "string") return content === query;
+        return typeof content === "string" && query.test(content);
+      });
+    });
+  }
+
+  public callsTool(name: string, args: Record<string, unknown> = {}): this {
+    const lastMock = this.mocks[this.mocks.length - 1];
+    if (!lastMock || lastMock.method !== "chat") {
+      throw new Error("Mocker: .callsTool() must follow a .chat() definition.");
+    }
+    lastMock.response = {
+      content: null,
+      tool_calls: [
+        {
+          id: `call_${Math.random().toString(36).slice(2, 9)}`,
+          type: "function",
+          function: { name, arguments: JSON.stringify(args) }
+        }
+      ],
+      finish_reason: "tool_calls"
+    };
+    return this;
+  }
+
+  public embed(input?: string | string[]): this {
+    return this.addMock("embed", (req: unknown) => {
+      const embReq = req as EmbeddingRequest;
+      if (!input) return true;
+      return JSON.stringify(embReq.input) === JSON.stringify(input);
+    });
+  }
+
+  public paint(prompt?: string | RegExp): this {
+    return this.addMock("paint", (req: unknown) => {
+      const paintReq = req as ImageRequest;
+      if (!prompt) return true;
+      if (typeof prompt === "string") return paintReq.prompt === prompt;
+      return prompt.test(paintReq.prompt);
+    });
+  }
+
+  public transcribe(file?: string | RegExp): this {
+    return this.addMock("transcribe", (req: unknown) => {
+      const transReq = req as TranscriptionRequest;
+      if (!file) return true;
+      if (typeof file === "string") return transReq.file === file;
+      return file.test(transReq.file);
+    });
+  }
+
+  public moderate(input?: string | string[] | RegExp): this {
+    return this.addMock("moderate", (req: unknown) => {
+      const modReq = req as ModerationRequest;
+      if (!input) return true;
+      const content = Array.isArray(modReq.input) ? modReq.input.join(" ") : modReq.input;
+      if (input instanceof RegExp) return input.test(content);
+      return JSON.stringify(modReq.input) === JSON.stringify(input);
+    });
+  }
+
+  public respond(response: string | MockResponse | ((req: unknown) => MockResponse)): this {
+    const lastMock = this.mocks[this.mocks.length - 1];
+    if (!lastMock) throw new Error("Mocker: No mock definition started.");
+    if (typeof response === "string") {
+      lastMock.response = { content: response, text: response };
+    } else {
+      lastMock.response = response;
+    }
+    return this;
+  }
+
+  /**
+   * Returns debug information about defined mocks.
+   * Useful for troubleshooting what mocks are defined.
+   */
+  public getDebugInfo(): MockerDebugInfo {
+    const methods = this.mocks.map((m) => m.method);
+    return {
+      totalMocks: this.mocks.length,
+      methods: [...new Set(methods)]
+    };
+  }
+
+  public clear(): void {
+    this.mocks = [];
+    providerRegistry.setInterceptor(undefined);
+  }
+
+  private addMock(method: string, matcher: MockMatcher): this {
+    this.mocks.push({ method, match: matcher, response: { content: "Mock response" } });
+    return this;
+  }
+
+  private getContentString(content: MessageContent | null | undefined | unknown): string | null {
+    if (content === null || content === undefined) return null;
+    if (typeof content === "string") return content;
+    if (Array.isArray(content)) {
+      return content.map((part) => (part.type === "text" ? part.text : "")).join("");
+    }
+    return String(content);
+  }
+
+  private setupInterceptor(): void {
+    providerRegistry.setInterceptor((provider: Provider) => {
+      return new Proxy(provider, {
+        get: (target, prop) => {
+          const originalValue = Reflect.get(target, prop);
+          const methodName = prop.toString();
+
+          if (methodName === "id") return target.id;
+
+          if (EXECUTION_METHODS.includes(methodName)) {
+            if (methodName === "stream") {
+              return async function* (this: Mocker, request: ChatRequest) {
+                const matchingMocks = this.mocks.filter(
+                  (m) => m.method === methodName && m.match(request)
+                );
+                const mock = matchingMocks[matchingMocks.length - 1];
+                if (mock) {
+                  const res =
+                    typeof mock.response === "function" ? mock.response(request) : mock.response;
+                  if (res.error) throw res.error;
+                  const chunks = res.chunks || [];
+                  for (let i = 0; i < chunks.length; i++) {
+                    const chunk = chunks[i];
+                    yield typeof chunk === "string"
+                      ? { content: chunk, done: i === chunks.length - 1 }
+                      : chunk;
+                  }
+                  return;
+                }
+                if (this.strict) throw new Error("Mocker: Unexpected LLM call to 'stream'");
+                const original = originalValue
+                  ? (originalValue as any).apply(target, [request])
+                  : undefined;
+                if (original) yield* original;
+              }.bind(this);
+            }
+
+            // Promise-based methods
+            return (async (request: unknown) => {
+              const matchingMocks = this.mocks.filter(
+                (m) => m.method === methodName && m.match(request)
+              );
+              const mock = matchingMocks[matchingMocks.length - 1];
+
+              if (mock) {
+                const res =
+                  typeof mock.response === "function" ? mock.response(request) : mock.response;
+                if (res.error) throw res.error;
+
+                switch (methodName) {
+                  case "chat": {
+                    return {
+                      content:
+                        res.content !== undefined && res.content !== null
+                          ? String(res.content)
+                          : null,
+                      tool_calls: res.tool_calls || [],
+                      usage: res.usage || { input_tokens: 10, output_tokens: 10, total_tokens: 20 },
+                      finish_reason:
+                        res.finish_reason || (res.tool_calls?.length ? "tool_calls" : "stop")
+                    } as ChatResponse;
+                  }
+                  case "embed": {
+                    const embReq = request as EmbeddingRequest;
+                    return {
+                      vectors: res.vectors || [[0.1, 0.2, 0.3]],
+                      model: embReq.model || "mock-embed",
+                      input_tokens: 10,
+                      dimensions: res.vectors?.[0]?.length || 3
+                    } as EmbeddingResponse;
+                  }
+                  case "paint": {
+                    const paintReq = request as ImageRequest;
+                    return {
+                      url: res.url || "http://mock.com/image.png",
+                      revised_prompt: res.revised_prompt || paintReq.prompt
+                    } as ImageResponse;
+                  }
+                  case "transcribe": {
+                    const transReq = request as TranscriptionRequest;
+                    return {
+                      text: res.text || "Mock transcript",
+                      model: transReq.model || "mock-whisper"
+                    } as TranscriptionResponse;
+                  }
+                  case "moderate": {
+                    const modReq = request as ModerationRequest;
+                    return {
+                      id: res.id || "mod-123",
+                      model: modReq.model || "mock-mod",
+                      results: res.results || [
+                        { flagged: false, categories: {}, category_scores: {} }
+                      ]
+                    } as ModerationResponse;
+                  }
+                  default:
+                    return res;
+                }
+              }
+
+              if (this.strict) throw new Error(`Mocker: Unexpected LLM call to '${methodName}'`);
+              return originalValue ? (originalValue as any).apply(target, [request]) : undefined;
+            }).bind(this);
+          }
+          return originalValue;
+        }
+      });
+    });
+  }
+}
+
+export function mockLLM() {
+  return new Mocker();
+}

package/src/Scrubber.ts

@@ -0,0 +1,85 @@
+export const DEFAULT_SECRET_PATTERNS = [
+  /sk-[a-zA-Z0-9]{20,}/g, // OpenAI/Anthropic likely patterns
+  /x-[a-zA-Z0-9]{20,}/g, // Generic API keys
+  /[a-zA-Z0-9]{32,}/g // Long hashes/keys
+];
+
+export interface ScrubberOptions {
+  customScrubber?: (data: unknown) => unknown;
+  sensitivePatterns?: RegExp[];
+  sensitiveKeys?: string[];
+}
+
+export class Scrubber {
+  private customScrubber?: (data: unknown) => unknown;
+  private sensitivePatterns: RegExp[];
+  private sensitiveKeys: Set<string>;
+
+  constructor(options: ScrubberOptions = {}) {
+    this.customScrubber = options.customScrubber;
+    this.sensitivePatterns = [...DEFAULT_SECRET_PATTERNS, ...(options.sensitivePatterns || [])];
+    this.sensitiveKeys = new Set([
+      "key",
+      "api_key",
+      "token",
+      "auth",
+      "authorization",
+      ...(options.sensitiveKeys || []).map((k) => k.toLowerCase())
+    ]);
+  }
+
+  /**
+   * Applies scrubbing to the data using default patterns and custom logic.
+   */
+  public scrub(data: unknown): unknown {
+    // 1. Perform deep regex scrubbing and key-based scrubbing
+    let result = this.deepScrub(data);
+
+    // 2. Run custom hook on the scrubbed data if provided
+    if (this.customScrubber) {
+      result = this.customScrubber(result);
+    }
+
+    return result;
+  }
+
+  private deepScrub(val: unknown): unknown {
+    if (typeof val === "string") {
+      let scrubbed = val;
+      for (const pattern of this.sensitivePatterns) {
+        scrubbed = scrubbed.replace(pattern, "[REDACTED]");
+      }
+      return scrubbed;
+    }
+
+    if (Array.isArray(val)) {
+      return val.map((v) => this.deepScrub(v));
+    }
+
+    if (val !== null && typeof val === "object") {
+      const obj = val as Record<string, unknown>;
+      const newObj: Record<string, unknown> = {};
+
+      for (const key in obj) {
+        if (!Object.prototype.hasOwnProperty.call(obj, key)) continue;
+
+        const lowerKey = key.toLowerCase();
+        const value = obj[key];
+
+        // SENSITIVE KEY LOGIC:
+        // Only redact if the key looks like a credential AND the value is a string.
+        // We don't want to redact 'total_tokens' or 'input_tokens' which are numbers.
+        const isCredentialKey = this.sensitiveKeys.has(lowerKey);
+
+        if (isCredentialKey && typeof value === "string") {
+          newObj[key] = "[REDACTED]";
+        } else {
+          newObj[key] = this.deepScrub(value);
+        }
+      }
+      return newObj;
+    }
+
+    return val;
+  }
+}