llm-mock-server 1.0.0

package/src/history.ts

@@ -0,0 +1,66 @@
+import type { MockRequest } from "./types.js";
+
+/** A recorded request with the rule that matched and when it happened. */
+export interface RecordedRequest {
+  readonly request: MockRequest;
+  /** The rule that matched, or `undefined` if the fallback was used. */
+  readonly rule: string | undefined;
+  readonly timestamp: number;
+}
+
+/**
+ * Records every request the server handles.
+ * Iterable and has fluent query methods for test assertions.
+ *
+ * @example
+ * ```ts
+ * expect(server.history.count()).toBe(3);
+ * expect(server.history.last()?.request.lastMessage).toBe("hello");
+ * const matched = server.history.where(r => r.rule !== undefined);
+ * ```
+ */
+export class RequestHistory {
+  private readonly entries: RecordedRequest[] = [];
+
+  record(request: MockRequest, rule: string | undefined): void {
+    this.entries.push({ request, rule, timestamp: Date.now() });
+  }
+
+  /** Number of recorded requests. */
+  count(): number {
+    return this.entries.length;
+  }
+
+  /** First recorded request, or `undefined` if empty. */
+  first(): RecordedRequest | undefined {
+    return this.entries[0];
+  }
+
+  /** Most recent recorded request, or `undefined` if empty. */
+  last(): RecordedRequest | undefined {
+    return this.entries.at(-1);
+  }
+
+  /** Get the entry at a specific index. Supports negative indices. */
+  at(index: number): RecordedRequest | undefined {
+    return this.entries.at(index);
+  }
+
+  /** Filter entries by a predicate. */
+  where(predicate: (entry: RecordedRequest) => boolean): RecordedRequest[] {
+    return this.entries.filter(predicate);
+  }
+
+  /** All entries as a readonly array. */
+  get all(): readonly RecordedRequest[] {
+    return this.entries;
+  }
+
+  clear(): void {
+    this.entries.length = 0;
+  }
+
+  [Symbol.iterator](): Iterator<RecordedRequest> {
+    return this.entries[Symbol.iterator]();
+  }
+}

package/src/index.ts

@@ -0,0 +1,44 @@
+export { MockServer } from "./mock-server.js";
+export type { MockServerOptions } from "./mock-server.js";
+export type { LogLevel } from "./logger.js";
+export { RequestHistory } from "./history.js";
+export type { RecordedRequest } from "./history.js";
+export type {
+  FormatName,
+  MockRequest,
+  Message,
+  ToolDef,
+  Reply,
+  ReplyObject,
+  ToolCall,
+  Resolver,
+  Match,
+  MatchObject,
+  ReplyOptions,
+  ErrorReply,
+  PendingRule,
+  RuleHandle,
+  RuleSummary,
+  SequenceEntry,
+  Handler,
+} from "./types.js";
+
+import { MockServer } from "./mock-server.js";
+import type { MockServerOptions } from "./mock-server.js";
+
+/**
+ * Create a server and start it in one go.
+ *
+ * @example
+ * ```ts
+ * const server = await createMock({ port: 0, logLevel: "info" });
+ * server.when("hello").reply("Hi!");
+ * // run your tests
+ * await server.stop();
+ * ```
+ */
+export async function createMock(options: MockServerOptions = {}): Promise<MockServer> {
+  const server = new MockServer(options);
+  await server.start(options.port ?? 0);
+  return server;
+}

package/src/loader.ts

@@ -0,0 +1,213 @@
+import { readFile, readdir, stat } from "node:fs/promises";
+import { join, extname } from "node:path";
+import JSON5 from "json5";
+import { z } from "zod";
+import type { Handler, Match, MatchObject, Reply } from "./types.js";
+import type { RuleEngine } from "./rule-engine.js";
+
+export interface LoadContext {
+  engine: RuleEngine;
+  setFallback?: (reply: Reply) => void;
+}
+
+const json5MatchSchema = z.union([
+  z.string(),
+  z.object({
+    message: z.string().optional(),
+    model: z.string().optional(),
+    system: z.string().optional(),
+    format: z.enum(["openai", "anthropic", "responses"]).optional(),
+  }),
+]);
+
+const json5ReplySchema = z.union([
+  z.string(),
+  z.object({
+    text: z.string().optional(),
+    reasoning: z.string().optional(),
+    tools: z.array(z.object({ name: z.string(), args: z.record(z.string(), z.unknown()) })).optional(),
+  }),
+]);
+
+const json5ReplyRef = z.union([json5ReplySchema, z.string().startsWith("$")]);
+
+const json5SequenceEntrySchema = z.union([
+  json5ReplyRef,
+  z.object({
+    reply: json5ReplyRef,
+    latency: z.int().nonnegative().optional(),
+    chunkSize: z.int().nonnegative().optional(),
+  }),
+]);
+
+const json5RuleSchema = z.union([
+  z.object({
+    when: json5MatchSchema,
+    reply: json5ReplyRef,
+    times: z.int().positive().optional(),
+  }),
+  z.object({
+    when: json5MatchSchema,
+    replies: z.array(json5SequenceEntrySchema).min(1),
+  }),
+]);
+
+const json5FileSchema = z.union([
+  z.array(json5RuleSchema),
+  z.object({
+    templates: z.record(z.string(), json5ReplySchema).optional(),
+    fallback: json5ReplySchema.optional(),
+    rules: z.array(json5RuleSchema),
+  }),
+]);
+
+function parseRegexString(s: string): RegExp | string {
+  const match = /^\/(.+)\/([dgimsuyv]*)$/.exec(s);
+  if (match) {
+    return new RegExp(match[1]!, match[2]);
+  }
+  return s;
+}
+
+type Json5ReplyRef = z.infer<typeof json5ReplyRef>;
+type Templates = Record<string, z.infer<typeof json5ReplySchema>> | undefined;
+
+function compileMatch(when: z.infer<typeof json5MatchSchema>): Match {
+  if (typeof when === "string") {
+    return parseRegexString(when);
+  }
+  const obj: MatchObject = {
+    ...(when.message !== undefined && { message: parseRegexString(when.message) }),
+    ...(when.model !== undefined && { model: parseRegexString(when.model) }),
+    ...(when.system !== undefined && { system: parseRegexString(when.system) }),
+    ...(when.format !== undefined && { format: when.format }),
+  };
+  return obj;
+}
+
+function resolveReplyRef(
+  ref: Json5ReplyRef,
+  templates: Templates,
+  filePath: string,
+): z.infer<typeof json5ReplySchema> {
+  if (typeof ref === "string" && ref.startsWith("$")) {
+    const name = ref.slice(1);
+    const resolved = templates?.[name];
+    if (!resolved) throw new Error(`Unknown template "${name}" in ${filePath}`);
+    return resolved;
+  }
+  return ref;
+}
+
+function addSequenceRule(
+  engine: RuleEngine,
+  match: Match,
+  entries: z.infer<typeof json5SequenceEntrySchema>[],
+  templates: Templates,
+  filePath: string,
+): void {
+  let index = 0;
+  const resolved = entries.map((entry) => {
+    if (typeof entry === "string" || !("reply" in entry)) {
+      return { reply: resolveReplyRef(entry, templates, filePath) };
+    }
+    return {
+      reply: resolveReplyRef(entry.reply, templates, filePath),
+      options: {
+        ...(entry.latency !== undefined && { latency: entry.latency }),
+        ...(entry.chunkSize !== undefined && { chunkSize: entry.chunkSize }),
+      },
+    };
+  });
+  const lastStep = resolved[resolved.length - 1]!;
+  const rule = engine.add(match, () => {
+    const step = resolved[index++] ?? lastStep;
+    rule.options = step.options ?? {};
+    return step.reply;
+  });
+  rule.remaining = resolved.length;
+}
+
+async function loadJson5File(filePath: string, ctx: LoadContext): Promise<void> {
+  const content = await readFile(filePath, "utf-8");
+  const parsed = json5FileSchema.parse(JSON5.parse(content));
+
+  const rules = Array.isArray(parsed) ? parsed : parsed.rules;
+  const templates = Array.isArray(parsed) ? undefined : parsed.templates;
+
+  if (!Array.isArray(parsed) && parsed.fallback !== undefined && ctx.setFallback) {
+    ctx.setFallback(parsed.fallback);
+  }
+
+  for (const r of rules) {
+    const match = compileMatch(r.when);
+    if ("replies" in r) {
+      addSequenceRule(ctx.engine, match, r.replies, templates, filePath);
+    } else {
+      const reply = resolveReplyRef(r.reply, templates, filePath);
+      const rule = ctx.engine.add(match, reply);
+      if (r.times !== undefined) {
+        rule.remaining = r.times;
+      }
+    }
+  }
+}
+
+const handlerSchema = z.custom<Handler>((val): val is Handler => {
+  if (typeof val !== "object" || val === null) return false;
+  const obj = val as Record<string, unknown>;
+  return typeof obj["match"] === "function" && typeof obj["respond"] === "function";
+});
+
+const handlerExportSchema = z.object({
+  default: z.union([handlerSchema, z.array(handlerSchema)]),
+  fallback: json5ReplySchema.optional(),
+});
+
+async function loadHandlerFile(filePath: string, ctx: LoadContext): Promise<void> {
+  const mod = await import(filePath);
+  const parsed = handlerExportSchema.safeParse(mod);
+  if (!parsed.success) {
+    throw new Error(
+      `Invalid handler file ${filePath}. Expected default export with { match: Function, respond: Function }.`,
+    );
+  }
+  const handlers = Array.isArray(parsed.data.default) ? parsed.data.default : [parsed.data.default];
+
+  if (parsed.data.fallback !== undefined && ctx.setFallback) {
+    ctx.setFallback(parsed.data.fallback);
+  }
+
+  for (const handler of handlers) {
+    ctx.engine.addHandler(handler.match, handler.respond, `(handler: ${filePath})`);
+  }
+}
+
+type FileLoader = (filePath: string, ctx: LoadContext) => Promise<void>;
+
+const loaderByExtension: ReadonlyMap<string, FileLoader> = new Map([
+  [".json5", loadJson5File],
+  [".json", loadJson5File],
+  [".ts", loadHandlerFile],
+  [".js", loadHandlerFile],
+  [".mjs", loadHandlerFile],
+]);
+
+export async function loadRulesFromPath(pathOrDir: string, ctx: LoadContext): Promise<void> {
+  const info = await stat(pathOrDir);
+
+  if (info.isFile()) {
+    const loader = loaderByExtension.get(extname(pathOrDir));
+    if (loader) await loader(pathOrDir, ctx);
+    return;
+  }
+
+  if (!info.isDirectory()) return;
+
+  const entries = (await readdir(pathOrDir)).toSorted();
+  for (const entry of entries) {
+    const fullPath = join(pathOrDir, entry);
+    const entryStat = await stat(fullPath);
+    if (entryStat.isFile()) await loadRulesFromPath(fullPath, ctx);
+  }
+}

package/src/logger.ts

@@ -0,0 +1,58 @@
+import pc from "picocolors";
+
+export const LEVEL_PRIORITY = {
+  none: 0,
+  error: 1,
+  warning: 2,
+  info: 3,
+  debug: 4,
+  all: 5,
+} as const satisfies Record<string, number>;
+
+/** Log verbosity, from `"none"` (silent) through to `"all"` (everything). */
+export type LogLevel = keyof typeof LEVEL_PRIORITY;
+
+const LEVEL_STYLE = {
+  error: { label: pc.red(pc.bold("ERROR")), symbol: pc.red("✗") },
+  warn: { label: pc.yellow(pc.bold("WARN")), symbol: pc.yellow("!") },
+  info: { label: pc.cyan("INFO"), symbol: pc.cyan("●") },
+  debug: { label: pc.dim("DEBUG"), symbol: pc.dim("·") },
+} as const;
+
+export class Logger {
+  readonly level: LogLevel;
+  private threshold: number;
+
+  constructor(level: LogLevel = "info") {
+    this.level = level;
+    this.threshold = LEVEL_PRIORITY[level];
+  }
+
+  error(msg: string, ...args: unknown[]): void {
+    if (this.threshold >= LEVEL_PRIORITY.error) {
+      const { label, symbol } = LEVEL_STYLE.error;
+      console.error(`${pc.dim(new Date().toISOString())} ${symbol} ${label} ${msg}`, ...args);
+    }
+  }
+
+  warn(msg: string, ...args: unknown[]): void {
+    if (this.threshold >= LEVEL_PRIORITY.warning) {
+      const { label, symbol } = LEVEL_STYLE.warn;
+      console.warn(`${pc.dim(new Date().toISOString())} ${symbol} ${label} ${msg}`, ...args);
+    }
+  }
+
+  info(msg: string, ...args: unknown[]): void {
+    if (this.threshold >= LEVEL_PRIORITY.info) {
+      const { label, symbol } = LEVEL_STYLE.info;
+      console.log(`${pc.dim(new Date().toISOString())} ${symbol} ${label} ${msg}`, ...args);
+    }
+  }
+
+  debug(msg: string, ...args: unknown[]): void {
+    if (this.threshold >= LEVEL_PRIORITY.debug) {
+      const { label, symbol } = LEVEL_STYLE.debug;
+      console.log(`${pc.dim(new Date().toISOString())} ${symbol} ${label} ${pc.dim(msg)}`, ...args);
+    }
+  }
+}

package/src/mock-server.ts

@@ -0,0 +1,237 @@
+import Fastify from "fastify";
+import type { FastifyInstance } from "fastify";
+import type {
+  Match, PendingRule, Reply, ReplyOptions, Resolver, Rule, RuleHandle, RuleSummary, SequenceEntry,
+} from "./types.js";
+import { RuleEngine } from "./rule-engine.js";
+import { RequestHistory } from "./history.js";
+import { openaiFormat } from "./formats/openai/index.js";
+import { anthropicFormat } from "./formats/anthropic/index.js";
+import { responsesFormat } from "./formats/responses/index.js";
+import type { Format } from "./formats/types.js";
+import { Logger } from "./logger.js";
+import type { LogLevel } from "./logger.js";
+import { createRouteHandler } from "./route-handler.js";
+
+const formats: readonly Format[] = [openaiFormat, anthropicFormat, responsesFormat];
+
+export interface MockServerOptions {
+  readonly port?: number;
+  /** Defaults to `"127.0.0.1"`. Set to `"0.0.0.0"` to listen on all interfaces. */
+  readonly host?: string;
+  /** Defaults to `"none"` (silent). */
+  readonly logLevel?: LogLevel;
+  /** Default ms delay between SSE chunks. Individual rules can override this. */
+  readonly defaultLatency?: number;
+  /** Default characters per SSE text chunk. Individual rules can override this. */
+  readonly defaultChunkSize?: number;
+}
+
+/**
+ * Mock LLM server that handles OpenAI, Anthropic, and Responses API formats.
+ * Register rules with `when()`, point your SDK at `url`, and go.
+ *
+ * Supports `await using` for automatic cleanup.
+ *
+ * @example
+ * ```ts
+ * const server = new MockServer({ logLevel: "info" });
+ * server.when("hello").reply("Hi there!");
+ * await server.start();
+ * // Point your client at server.url
+ * await server.stop();
+ * ```
+ */
+export class MockServer {
+  private readonly app: FastifyInstance;
+  private readonly engine = new RuleEngine();
+  private readonly history_ = new RequestHistory();
+  private readonly logger: Logger;
+  private readonly host: string;
+  private readonly defaultOptions: ReplyOptions;
+  private fallbackReply: Reply = "Mock server: no matching rule.";
+  private listening = false;
+
+  constructor(options: MockServerOptions = {}) {
+    this.host = options.host ?? "127.0.0.1";
+    this.logger = new Logger(options.logLevel ?? "none");
+    this.defaultOptions = {
+      ...(options.defaultLatency !== undefined && { latency: options.defaultLatency }),
+      ...(options.defaultChunkSize !== undefined && { chunkSize: options.defaultChunkSize }),
+    };
+    this.app = Fastify({ logger: false });
+
+    const deps = {
+      engine: this.engine,
+      history: this.history_,
+      logger: this.logger,
+      defaultOptions: this.defaultOptions,
+      getFallback: () => this.fallbackReply,
+    };
+
+    for (const format of formats) {
+      this.app.post(format.route, createRouteHandler(format, deps));
+    }
+  }
+
+  /**
+   * Register a matching rule. Call `.reply()` on the result to set the response.
+   *
+   * @example
+   * ```ts
+   * server.when("hello").reply("Hi!");
+   * server.when(/explain (\w+)/i).reply((req) => `Let me explain ${req.lastMessage}`);
+   * server.when({ model: /claude/ }).reply("I'm Claude.");
+   * ```
+   */
+  when(match: Match): PendingRule {
+    const engine = this.engine;
+
+    const makeHandle = (rule: Rule): RuleHandle => ({
+      times(n: number): RuleHandle {
+        rule.remaining = n;
+        return this;
+      },
+      first(): RuleHandle {
+        engine.moveToFront(rule);
+        return this;
+      },
+    });
+
+    return {
+      reply(response: Resolver, options?: ReplyOptions): RuleHandle {
+        return makeHandle(engine.add(match, response, options));
+      },
+      replySequence(entries: readonly SequenceEntry[]): RuleHandle {
+        if (entries.length === 0) throw new Error("replySequence requires at least one entry.");
+        let index = 0;
+        const last = entries[entries.length - 1]!;
+        const rule = engine.add(match, () => {
+          const entry = entries[index++] ?? last;
+          if (typeof entry === "string" || !("reply" in entry)) {
+            rule.options = {};
+            return entry;
+          }
+          rule.options = entry.options ?? {};
+          return entry.reply;
+        });
+        rule.remaining = entries.length;
+        return makeHandle(rule);
+      },
+    };
+  }
+
+  /**
+   * Register a rule that matches when the request includes a tool with this name.
+   *
+   * @example
+   * ```ts
+   * server.whenTool("get_weather").reply({
+   *   tools: [{ name: "get_weather", args: { location: "London" } }],
+   * });
+   * ```
+   */
+  whenTool(toolName: string): PendingRule {
+    return this.when({ toolName });
+  }
+
+  /**
+   * Register a rule that matches when the last message is a tool result with this call ID.
+   *
+   * @example
+   * ```ts
+   * server.whenToolResult("call_abc").reply("Got your result, cheers!");
+   * ```
+   */
+  whenToolResult(toolCallId: string): PendingRule {
+    return this.when({ toolCallId });
+  }
+
+  /**
+   * Queue a one-shot error for the very next request, regardless of content.
+   * Fires once then removes itself.
+   *
+   * @example
+   * ```ts
+   * server.nextError(429, "Rate limited");
+   * // next request gets a 429, after that normal matching resumes
+   * ```
+   */
+  nextError(status: number, message: string, type?: string): RuleHandle {
+    return this.when(() => true)
+      .reply({ error: { status, message, type } })
+      .times(1)
+      .first();
+  }
+
+  /** Set the reply used when no rule matches. Defaults to a generic message. */
+  fallback(reply: Reply): void {
+    this.fallbackReply = reply;
+  }
+
+  /** Load rules from a `.json5` file, a `.ts`/`.js` handler file, or a directory containing them. */
+  async load(pathOrDir: string): Promise<void> {
+    const before = this.engine.ruleCount;
+    const { loadRulesFromPath } = await import("./loader.js");
+    await loadRulesFromPath(pathOrDir, {
+      engine: this.engine,
+      setFallback: (reply) => { this.fallbackReply = reply; },
+    });
+    const loaded = this.engine.ruleCount - before;
+    this.logger.info(`Loaded ${loaded} rule${loaded !== 1 ? "s" : ""} from ${pathOrDir}`);
+  }
+
+  /** Every request the server has handled. */
+  get history(): RequestHistory {
+    return this.history_;
+  }
+
+  /** Returns `true` when all rules with a `.times()` limit have been consumed. */
+  isDone(): boolean {
+    return this.engine.isDone();
+  }
+
+  /** Clear all rules, request history, and reset the fallback to its default. */
+  reset(): void {
+    this.engine.clear();
+    this.history_.clear();
+    this.fallbackReply = "Mock server: no matching rule.";
+    this.logger.info("Server reset: rules and history cleared");
+  }
+
+  /** The base URL the server is listening on, e.g. `http://127.0.0.1:12345`. Throws if the server hasn't started. */
+  get url(): string {
+    if (!this.listening) throw new Error("Server is not running. Call start() first.");
+    const addr = this.app.server.address();
+    const port = addr !== null && typeof addr === "object" ? addr.port : 0;
+    return `http://${this.host}:${port}`;
+  }
+
+  get ruleCount(): number {
+    return this.engine.ruleCount;
+  }
+
+  /** A snapshot of all registered rules with their descriptions and remaining match counts. */
+  get rules(): readonly RuleSummary[] {
+    return this.engine.describe();
+  }
+
+  /** Start listening. Pass `0` (the default) for a random port. */
+  async start(port = 0): Promise<void> {
+    if (this.listening) throw new Error("Server is already running.");
+    await this.app.listen({ port, host: this.host });
+    this.listening = true;
+    this.logger.info(`Listening on ${this.url}`);
+  }
+
+  async stop(): Promise<void> {
+    if (!this.listening) return;
+    await this.app.close();
+    this.listening = false;
+    this.logger.info("Server stopped");
+  }
+
+  async [Symbol.asyncDispose](): Promise<void> {
+    await this.stop();
+  }
+}