llm-mock-server 1.0.0

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "llm-mock-server",
+  "version": "1.0.0",
+  "description": "A standalone mock LLM server for deterministic testing: OpenAI, Anthropic, and Responses API formats",
+  "type": "module",
+  "engines": {
+    "node": ">=22"
+  },
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    },
+    "./package.json": "./package.json"
+  },
+  "bin": {
+    "llm-mock-server": "./dist/cli.js"
+  },
+  "scripts": {
+    "build": "tsc",
+    "lint": "oxlint --tsconfig tsconfig.json --import-plugin --vitest-plugin src/ test/",
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "check": "tsc --noEmit && tsc --noEmit -p tsconfig.test.json && npm run lint && npm test",
+    "dev": "tsx src/cli.ts",
+    "start": "node dist/cli.js"
+  },
+  "keywords": [
+    "llm",
+    "mock",
+    "server",
+    "openai",
+    "anthropic",
+    "testing",
+    "sse"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "commander": "14.0.3",
+    "fastify": "5.8.2",
+    "json5": "2.2.3",
+    "picocolors": "1.1.1",
+    "zod": "4.3.6"
+  },
+  "devDependencies": {
+    "@types/node": "25.5.0",
+    "@vitest/coverage-v8": "4.1.0",
+    "oxlint": "1.55.0",
+    "tsx": "4.21.0",
+    "typescript": "5.9.3",
+    "vitest": "4.1.0"
+  }
+}

package/src/cli-validators.ts

@@ -0,0 +1,56 @@
+import { isIP } from "node:net";
+import { lookup } from "node:dns/promises";
+import { LEVEL_PRIORITY, type LogLevel } from "./logger.js";
+
+const VALID_LOG_LEVELS = Object.keys(LEVEL_PRIORITY) as LogLevel[];
+
+function isLogLevel(value: string): value is LogLevel {
+  return value in LEVEL_PRIORITY;
+}
+
+const MAX_PORT = 65535;
+
+export function parsePort(value: string): number {
+  const port = parseInt(value, 10);
+  if (isNaN(port) || port < 1 || port > MAX_PORT) {
+    throw new Error(`Invalid port "${value}". Must be 1-${String(MAX_PORT)}.`);
+  }
+  return port;
+}
+
+export function parseLogLevel(value: string): LogLevel {
+  if (!isLogLevel(value)) {
+    throw new Error(
+      `Invalid log level "${value}". Valid: ${VALID_LOG_LEVELS.join(", ")}`,
+    );
+  }
+  return value;
+}
+
+export async function parseHost(value: string): Promise<string> {
+  if (value === "localhost" || isIP(value) !== 0) {
+    return value;
+  }
+  try {
+    await lookup(value);
+    return value;
+  } catch {
+    throw new Error(`Invalid host "${value}". Must be a resolvable hostname or IP address.`);
+  }
+}
+
+export function parseChunkSize(value: string): number {
+  const size = parseInt(value, 10);
+  if (isNaN(size) || size < 0) {
+    throw new Error(`Invalid chunk size "${value}". Must be a non-negative integer.`);
+  }
+  return size;
+}
+
+export function parseLatency(value: string): number {
+  const ms = parseInt(value, 10);
+  if (isNaN(ms) || ms < 0) {
+    throw new Error(`Invalid latency "${value}". Must be a non-negative integer (ms).`);
+  }
+  return ms;
+}

package/src/cli.ts

@@ -0,0 +1,128 @@
+#!/usr/bin/env node
+
+import { watch } from "node:fs";
+import { Command } from "commander";
+import pc from "picocolors";
+import { MockServer } from "./mock-server.js";
+import { Logger } from "./logger.js";
+import { parsePort, parseHost, parseLogLevel, parseChunkSize, parseLatency } from "./cli-validators.js";
+
+const WATCH_DEBOUNCE_MS = 100;
+
+interface StartOptions {
+  port: string;
+  host: string;
+  rules?: string;
+  handler?: string;
+  latency: string;
+  chunkSize: string;
+  fallback?: string;
+  logLevel: string;
+  watch?: boolean;
+}
+
+async function start(options: StartOptions): Promise<void> {
+  const logLevel = parseLogLevel(options.logLevel);
+  const logger = new Logger(logLevel);
+  const port = parsePort(options.port);
+  const host = await parseHost(options.host);
+  const latency = parseLatency(options.latency);
+  const chunkSize = parseChunkSize(options.chunkSize);
+
+  const server = new MockServer({
+    port,
+    host,
+    logLevel,
+    ...(latency > 0 && { defaultLatency: latency }),
+    ...(chunkSize > 0 && { defaultChunkSize: chunkSize }),
+  });
+
+  if (options.fallback) {
+    server.fallback(options.fallback);
+  }
+
+  if (options.rules) {
+    await server.load(options.rules);
+  }
+  if (options.handler) {
+    await server.load(options.handler);
+  }
+
+  const quiet = logLevel === "none";
+
+  await server.start(port);
+
+  if (!quiet) {
+    console.log();
+    console.log(`  ${pc.bold(pc.cyan("llm-mock-server"))} ${pc.dim("v1.0.0")}`);
+    console.log();
+    console.log(`  ${pc.dim("Port")}       ${pc.bold(String(port))}`);
+    console.log(`  ${pc.dim("Rules")}      ${pc.bold(String(server.ruleCount))} loaded`);
+    if (latency > 0) {
+      console.log(`  ${pc.dim("Latency")}    ${pc.bold(`${String(latency)}ms`)} per chunk`);
+    }
+    console.log(
+      `  ${pc.dim("Endpoints")}  ${pc.green("/v1/chat/completions")}, ${pc.green("/v1/messages")}, ${pc.green("/v1/responses")}`,
+    );
+    console.log();
+  }
+
+  if (options.watch && options.rules) {
+    const rulesPath = options.rules;
+    let reloading = false;
+    watch(rulesPath, { recursive: true }, () => {
+      if (reloading) return;
+      reloading = true;
+      setTimeout(async () => {
+        try {
+          server.reset();
+          await server.load(rulesPath);
+          if (options.fallback) server.fallback(options.fallback);
+          logger.info(`Reloaded rules from ${rulesPath}`);
+        } catch (err) {
+          logger.error("Failed to reload rules", err);
+        }
+        reloading = false;
+      }, WATCH_DEBOUNCE_MS);
+    });
+    logger.info(`Watching ${rulesPath} for changes`);
+  }
+
+  let shuttingDown = false;
+  const shutdown = async (signal: string) => {
+    if (shuttingDown) return;
+    shuttingDown = true;
+
+    logger.info(`Got ${signal}, shutting down...`);
+    await server.stop();
+    logger.info("Clean shutdown complete");
+    process.exit(0);
+  };
+
+  process.on("SIGINT", () => { shutdown("SIGINT").catch(() => process.exit(1)); });
+  process.on("SIGTERM", () => { shutdown("SIGTERM").catch(() => process.exit(1)); });
+}
+
+const program = new Command()
+  .name("llm-mock-server")
+  .description("Mock LLM server for deterministic testing")
+  .version("1.0.0");
+
+program
+  .command("start", { isDefault: true })
+  .description("Start the mock server")
+  .option("-p, --port <number>", "port to listen on", "5555")
+  .option("-H, --host <address>", "host to bind to", "127.0.0.1")
+  .option("-r, --rules <path>", "path to .json5 rules file or directory")
+  .option("--handler <path>", "path to .ts handler file")
+  .option("-l, --latency <ms>", "latency between SSE chunks (ms)", "0")
+  .option("-c, --chunk-size <chars>", "characters per SSE chunk", "0")
+  .option("-f, --fallback <text>", "fallback reply text")
+  .option("-w, --watch", "watch rules path and reload on changes")
+  .option("--log-level <level>", "log verbosity", "info")
+  .action((options: StartOptions) => start(options));
+
+program.parseAsync().catch((err: unknown) => {
+  console.error("Fatal error:", err);
+  process.exit(1);
+});

package/src/formats/anthropic/index.ts

@@ -0,0 +1,14 @@
+import type { Format } from "../types.js";
+import { isStreaming } from "../parse-helpers.js";
+import { parseRequest } from "./parse.js";
+import { serialize, serializeComplete, serializeError } from "./serialize.js";
+
+export const anthropicFormat: Format = {
+  name: "anthropic",
+  route: "/v1/messages",
+  parseRequest,
+  isStreaming,
+  serialize,
+  serializeComplete,
+  serializeError,
+};

package/src/formats/anthropic/parse.ts

@@ -0,0 +1,48 @@
+import type { MockRequest, Message, ToolDef } from "../../types.js";
+import { buildMockRequest, type RequestMeta } from "../parse-helpers.js";
+import { AnthropicRequestSchema, type AnthropicRequest } from "./schema.js";
+
+function extractSystem(system: AnthropicRequest["system"]): Message[] {
+  if (system == null) return [];
+  if (typeof system === "string") return system ? [{ role: "system", content: system }] : [];
+  const text = system.map((b) => b.text).join("\n");
+  return text ? [{ role: "system", content: text }] : [];
+}
+
+function extractContent(content: AnthropicRequest["messages"][number]["content"]): { content: string; toolCallId?: string | undefined } {
+  if (typeof content === "string") return { content };
+  const text = content
+    .filter((b) => b.type === "text")
+    .map((b) => b.text)
+    .join("\n");
+  const toolResult = content.find((b) => b.type === "tool_result");
+  const toolCallId = toolResult?.type === "tool_result" ? toolResult.tool_use_id : undefined;
+  return { content: text, toolCallId };
+}
+
+function parseMessages(req: AnthropicRequest): readonly Message[] {
+  const system = extractSystem(req.system);
+  const conversation = req.messages.map((m) => {
+    const extracted = extractContent(m.content);
+    return {
+      role: m.role,
+      content: extracted.content,
+      ...(extracted.toolCallId !== undefined && { toolCallId: extracted.toolCallId }),
+    };
+  });
+  return [...system, ...conversation];
+}
+
+function parseTools(req: AnthropicRequest): readonly ToolDef[] | undefined {
+  if (!req.tools) return undefined;
+  return req.tools.map((t) => ({
+    name: t.name,
+    description: t.description,
+    parameters: t.input_schema,
+  }));
+}
+
+export function parseRequest(body: unknown, meta?: RequestMeta): MockRequest {
+  const req = AnthropicRequestSchema.parse(body);
+  return buildMockRequest("anthropic", req, parseMessages(req), parseTools(req), "claude-sonnet-4-6", body, meta);
+}

package/src/formats/anthropic/schema.ts

@@ -0,0 +1,133 @@
+import { z } from "zod";
+
+const TextBlockSchema = z.object({ type: z.literal("text"), text: z.string() });
+
+const ToolUseBlockSchema = z.object({
+  type: z.literal("tool_use"),
+  id: z.string(),
+  name: z.string(),
+  input: z.record(z.string(), z.unknown()),
+});
+
+const ToolResultBlockSchema = z.object({
+  type: z.literal("tool_result"),
+  tool_use_id: z.string(),
+  content: z.union([z.string(), z.array(TextBlockSchema)]).optional(),
+});
+
+const KnownContentBlockSchema = z.discriminatedUnion("type", [
+  TextBlockSchema,
+  ToolUseBlockSchema,
+  ToolResultBlockSchema,
+]);
+
+const LooseContentBlockSchema = z.union([
+  KnownContentBlockSchema,
+  z.looseObject({ type: z.string() }),
+]);
+
+const KNOWN_BLOCK_TYPES = new Set(["text", "tool_use", "tool_result"]);
+
+type KnownBlock = z.infer<typeof KnownContentBlockSchema>;
+
+const MessageSchema = z.object({
+  role: z.enum(["user", "assistant"]),
+  content: z.union([
+    z.string(),
+    z.array(LooseContentBlockSchema).transform((blocks) =>
+      blocks.filter((b): b is KnownBlock => KNOWN_BLOCK_TYPES.has(b.type)),
+    ),
+  ]),
+});
+
+const ToolDefinitionSchema = z.object({
+  name: z.string(),
+  description: z.string().optional(),
+  input_schema: z.record(z.string(), z.unknown()),
+});
+
+export const AnthropicRequestSchema = z.looseObject({
+  model: z.string().min(1),
+  max_tokens: z.number().int().positive(),
+  system: z.union([z.string(), z.array(TextBlockSchema)]).optional(),
+  messages: z.array(MessageSchema).min(1),
+  tools: z.array(ToolDefinitionSchema).optional(),
+  stream: z.boolean().optional(),
+  temperature: z.number().optional(),
+  top_p: z.number().optional(),
+  top_k: z.number().optional(),
+  stop_sequences: z.array(z.string()).optional(),
+  metadata: z.record(z.string(), z.unknown()).optional(),
+  cache_control: z.unknown().optional(),
+  container: z.string().optional(),
+  inference_geo: z.string().optional(),
+  output_config: z.unknown().optional(),
+  service_tier: z.string().optional(),
+  thinking: z.unknown().optional(),
+  tool_choice: z.unknown().optional(),
+});
+
+export type AnthropicRequest = z.infer<typeof AnthropicRequestSchema>;
+
+const ResponseContentBlockSchema = z.object({
+  type: z.string(),
+  text: z.string().optional(),
+  thinking: z.string().optional(),
+  id: z.string().optional(),
+  name: z.string().optional(),
+  input: z.unknown().optional(),
+});
+
+export const AnthropicMessageStartSchema = z.object({
+  message: z.object({
+    id: z.string(),
+    type: z.literal("message"),
+    role: z.literal("assistant"),
+    content: z.array(z.unknown()),
+    model: z.string(),
+    stop_reason: z.string().nullable(),
+    usage: z.object({ input_tokens: z.number(), output_tokens: z.number() }),
+  }),
+});
+
+export type AnthropicMessageStart = z.infer<typeof AnthropicMessageStartSchema>;
+
+export const AnthropicBlockEventSchema = z.object({
+  index: z.number(),
+  content_block: ResponseContentBlockSchema.optional(),
+  delta: z.object({
+    type: z.string(),
+    text: z.string().optional(),
+    thinking: z.string().optional(),
+    partial_json: z.string().optional(),
+  }).optional(),
+});
+
+export type AnthropicBlockEvent = z.infer<typeof AnthropicBlockEventSchema>;
+
+export const AnthropicDeltaSchema = z.object({
+  delta: z.object({ stop_reason: z.string(), stop_sequence: z.string().nullable() }),
+  usage: z.object({ output_tokens: z.number() }),
+});
+
+export type AnthropicDelta = z.infer<typeof AnthropicDeltaSchema>;
+
+export const AnthropicCompleteSchema = z.object({
+  id: z.string(),
+  type: z.literal("message"),
+  role: z.literal("assistant"),
+  model: z.string(),
+  content: z.array(ResponseContentBlockSchema),
+  stop_reason: z.string(),
+  stop_sequence: z.string().nullable(),
+  usage: z.object({ input_tokens: z.number(), output_tokens: z.number() }),
+});
+
+export type AnthropicComplete = z.infer<typeof AnthropicCompleteSchema>;
+
+export const AnthropicErrorSchema = z.object({
+  type: z.literal("error"),
+  error: z.object({ type: z.string(), message: z.string() }),
+});
+
+export type AnthropicError = z.infer<typeof AnthropicErrorSchema>;

package/src/formats/anthropic/serialize.ts

@@ -0,0 +1,91 @@
+import type { ReplyObject, ReplyOptions } from "../../types.js";
+import type { SSEChunk } from "../types.js";
+import { splitText, genId, toolId, shouldEmitText, finishReason, DEFAULT_USAGE } from "../parse-helpers.js";
+
+function contentBlock(index: number, startBlock: unknown, deltas: SSEChunk[]): SSEChunk[] {
+  return [
+    { event: "content_block_start", data: JSON.stringify({ type: "content_block_start", index, content_block: startBlock }) },
+    ...deltas,
+    { event: "content_block_stop", data: JSON.stringify({ type: "content_block_stop", index }) },
+  ];
+}
+
+function delta(index: number, payload: Record<string, unknown>): SSEChunk {
+  return { event: "content_block_delta", data: JSON.stringify({ type: "content_block_delta", index, delta: payload }) };
+}
+
+function reasoningBlock(i: number, reasoning: string): SSEChunk[] {
+  return contentBlock(i, { type: "thinking", thinking: "" }, [
+    delta(i, { type: "thinking_delta", thinking: reasoning }),
+  ]);
+}
+
+function textBlock(i: number, text: string, chunkSize: number): SSEChunk[] {
+  return contentBlock(
+    i,
+    { type: "text", text: "" },
+    splitText(text, chunkSize).map((piece) => delta(i, { type: "text_delta", text: piece })),
+  );
+}
+
+function toolBlocks(startIndex: number, tools: ReplyObject["tools"]): SSEChunk[] {
+  return (tools ?? []).flatMap((tool, i) => {
+    const idx = startIndex + i;
+    const id = toolId(tool, "toolu", idx);
+    return contentBlock(
+      idx,
+      { type: "tool_use", id, name: tool.name, input: {} },
+      [delta(idx, { type: "input_json_delta", partial_json: JSON.stringify(tool.args) })],
+    );
+  });
+}
+
+export function serialize(reply: ReplyObject, model: string, options: ReplyOptions = {}): readonly SSEChunk[] {
+  const id = genId("msg");
+  const usage = reply.usage ?? DEFAULT_USAGE;
+  let idx = 0;
+
+  const reasoningChunks = reply.reasoning ? reasoningBlock(idx++, reply.reasoning) : [];
+  const textChunks = shouldEmitText(reply) ? textBlock(idx++, reply.text ?? "", options.chunkSize ?? 0) : [];
+  const toolChunks = toolBlocks(idx, reply.tools);
+
+  return [
+    { event: "message_start", data: JSON.stringify({
+      type: "message_start",
+      message: { id, type: "message", role: "assistant", model, content: [], stop_reason: null, usage: { input_tokens: usage.input, output_tokens: 0 } },
+    })},
+    ...reasoningChunks,
+    ...textChunks,
+    ...toolChunks,
+    { event: "message_delta", data: JSON.stringify({
+      type: "message_delta",
+      delta: { stop_reason: finishReason(reply, "tool_use", "end_turn"), stop_sequence: null },
+      usage: { output_tokens: usage.output },
+    })},
+    { event: "message_stop", data: JSON.stringify({ type: "message_stop" }) },
+  ];
+}
+
+export function serializeComplete(reply: ReplyObject, model: string): unknown {
+  const id = genId("msg");
+  const usage = reply.usage ?? DEFAULT_USAGE;
+
+  const content: unknown[] = [
+    ...(reply.reasoning ? [{ type: "thinking", thinking: reply.reasoning }] : []),
+    ...(shouldEmitText(reply) ? [{ type: "text", text: reply.text ?? "" }] : []),
+    ...(reply.tools ?? []).map((tool) => ({
+      type: "tool_use", id: toolId(tool, "toolu", 0), name: tool.name, input: tool.args,
+    })),
+  ];
+
+  return {
+    id, type: "message", role: "assistant", model, content,
+    stop_reason: finishReason(reply, "tool_use", "end_turn"),
+    stop_sequence: null,
+    usage: { input_tokens: usage.input, output_tokens: usage.output },
+  };
+}
+
+export function serializeError(error: { status: number; message: string; type?: string }): unknown {
+  return { type: "error", error: { type: error.type ?? "api_error", message: error.message } };
+}

package/src/formats/openai/index.ts

@@ -0,0 +1,14 @@
+import type { Format } from "../types.js";
+import { isStreaming } from "../parse-helpers.js";
+import { parseRequest } from "./parse.js";
+import { serialize, serializeComplete, serializeError } from "./serialize.js";
+
+export const openaiFormat: Format = {
+  name: "openai",
+  route: "/v1/chat/completions",
+  parseRequest,
+  isStreaming,
+  serialize,
+  serializeComplete,
+  serializeError,
+};

package/src/formats/openai/parse.ts

@@ -0,0 +1,34 @@
+import type { MockRequest, Message, ToolDef } from "../../types.js";
+import { buildMockRequest, type RequestMeta } from "../parse-helpers.js";
+import { OpenAIRequestSchema, type OpenAIRequest } from "./schema.js";
+
+function extractContent(content: OpenAIRequest["messages"][number]["content"]): string {
+  if (content == null) return "";
+  if (typeof content === "string") return content;
+  return content
+    .filter((p) => p.type === "text" && p.text !== undefined)
+    .map((p) => p.text!)
+    .join("\n");
+}
+
+function parseMessages(req: OpenAIRequest): readonly Message[] {
+  return req.messages.map((m) => ({
+    role: m.role === "developer" ? "system" : (m.role ?? "user"),
+    content: extractContent(m.content),
+    ...(m.tool_call_id !== undefined && { toolCallId: m.tool_call_id }),
+  }));
+}
+
+function parseTools(req: OpenAIRequest): readonly ToolDef[] | undefined {
+  if (!req.tools) return undefined;
+  return req.tools.map((t) => ({
+    name: t.function.name,
+    description: t.function.description,
+    parameters: t.function.parameters,
+  }));
+}
+
+export function parseRequest(body: unknown, meta?: RequestMeta): MockRequest {
+  const req = OpenAIRequestSchema.parse(body);
+  return buildMockRequest("openai", req, parseMessages(req), parseTools(req), "gpt-5.4", body, meta);
+}