mcp-eval-runner 1.0.0

package/dist/fixture-library.js

@@ -0,0 +1,85 @@
+/**
+ * Shared fixture discovery and publishing for mcp-eval-runner.
+ * Supports discovering fixtures across multiple directories and publishing
+ * (copying) fixtures to a destination directory.
+ */
+import fs from "fs";
+import path from "path";
+import { loadFixturesFromDir } from "./fixture.js";
+/**
+ * Discover all fixture files across the given directories.
+ * Returns a deduplicated list (by name, first occurrence wins).
+ *
+ * @param dirs - Array of directory paths to scan for fixtures
+ */
+export function discoverFixtures(dirs) {
+    const seen = new Set();
+    const entries = [];
+    for (const dir of dirs) {
+        if (!fs.existsSync(dir)) {
+            continue;
+        }
+        const fixtures = loadFixturesFromDir(dir);
+        for (const fixture of fixtures) {
+            if (seen.has(fixture.name)) {
+                continue;
+            }
+            seen.add(fixture.name);
+            // Find the actual file path for this fixture
+            const candidates = [
+                path.join(dir, `${fixture.name}.yaml`),
+                path.join(dir, `${fixture.name}.yml`),
+                path.join(dir, `${fixture.name}.json`),
+                // Also try the sanitized name
+                path.join(dir, `${fixture.name.replace(/[^a-z0-9_-]/gi, "_")}.yaml`),
+                path.join(dir, `${fixture.name.replace(/[^a-z0-9_-]/gi, "_")}.yml`),
+                path.join(dir, `${fixture.name.replace(/[^a-z0-9_-]/gi, "_")}.json`),
+            ];
+            let filePath = dir;
+            for (const candidate of candidates) {
+                if (fs.existsSync(candidate)) {
+                    filePath = candidate;
+                    break;
+                }
+            }
+            entries.push({
+                name: fixture.name,
+                path: filePath,
+                suite_count: 1, // Each fixture file is one suite
+                case_count: fixture.steps.length,
+            });
+        }
+    }
+    return entries;
+}
+/**
+ * Publish (copy) a fixture YAML/JSON file to the destination directory.
+ * Creates the destination directory if it does not exist.
+ *
+ * @param fixture - The fixture object or a path to a fixture file
+ * @param dest    - Destination directory path
+ */
+export function publishFixture(fixture, dest) {
+    fs.mkdirSync(dest, { recursive: true });
+    // If fixture is a string, treat it as a file path to copy
+    if (typeof fixture === "string") {
+        const srcPath = fixture;
+        if (!fs.existsSync(srcPath)) {
+            throw new Error(`Fixture file not found: ${srcPath}`);
+        }
+        const fileName = path.basename(srcPath);
+        const destPath = path.join(dest, fileName);
+        fs.copyFileSync(srcPath, destPath);
+        return;
+    }
+    // If fixture is an object, serialize it as YAML-like JSON and write
+    if (typeof fixture === "object" && fixture !== null) {
+        const obj = fixture;
+        const name = obj.name ?? "fixture";
+        const safeName = String(name).replace(/[^a-z0-9_-]/gi, "_");
+        const destPath = path.join(dest, `${safeName}.json`);
+        fs.writeFileSync(destPath, JSON.stringify(fixture, null, 2), "utf-8");
+        return;
+    }
+    throw new Error(`Invalid fixture: expected a file path string or fixture object`);
+}

package/dist/fixture.d.ts

@@ -0,0 +1,87 @@
+/**
+ * YAML fixture loading and validation for MCP Eval Runner.
+ *
+ * Fixture JSON Schema:
+ * {
+ *   "type": "object",
+ *   "required": ["name", "steps"],
+ *   "properties": {
+ *     "name": { "type": "string" },
+ *     "description": { "type": "string" },
+ *     "server": {
+ *       "type": "object",
+ *       "description": "Optional server config for live execution mode",
+ *       "properties": {
+ *         "command": { "type": "string" },
+ *         "args": { "type": "array", "items": { "type": "string" } },
+ *         "env": { "type": "object" }
+ *       },
+ *       "required": ["command"]
+ *     },
+ *     "steps": {
+ *       "type": "array",
+ *       "items": {
+ *         "type": "object",
+ *         "required": ["id", "tool"],
+ *         "properties": {
+ *           "id": { "type": "string" },
+ *           "description": { "type": "string" },
+ *           "tool": { "type": "string" },
+ *           "input": { "type": "object" },
+ *           "expected_output": { "type": "string" },
+ *           "expect": {
+ *             "type": "object",
+ *             "properties": {
+ *               "output_contains": { "type": "string" },
+ *               "output_not_contains": { "type": "string" },
+ *               "output_equals": { "type": "string" },
+ *               "output_matches": { "type": "string" },
+ *               "tool_called": { "type": "string" },
+ *               "latency_under": { "type": "number" }
+ *             }
+ *           }
+ *         }
+ *       }
+ *     }
+ *   }
+ * }
+ */
+import type { Assertion } from "./assertions.js";
+export interface ServerConfig {
+    command: string;
+    args?: string[];
+    env?: Record<string, string>;
+}
+export interface FixtureStep {
+    id: string;
+    description?: string;
+    tool: string;
+    input: Record<string, unknown>;
+    expected_output?: string;
+    expect?: Assertion;
+}
+export interface Fixture {
+    name: string;
+    description?: string;
+    server?: ServerConfig;
+    steps: FixtureStep[];
+}
+export declare class FixtureValidationError extends Error {
+    constructor(message: string);
+}
+/**
+ * Load and validate a fixture from a YAML or JSON file.
+ */
+export declare function loadFixture(filePath: string): Fixture;
+/**
+ * Validate a raw parsed object against the fixture schema.
+ */
+export declare function validateFixture(raw: unknown, source?: string): Fixture;
+/**
+ * Load all fixtures from a directory.
+ */
+export declare function loadFixturesFromDir(fixturesDir: string): Fixture[];
+/**
+ * Write a fixture to a YAML file in the fixtures directory.
+ */
+export declare function writeFixture(fixturesDir: string, fixture: Fixture): string;

package/dist/fixture.js

@@ -0,0 +1,170 @@
+/**
+ * YAML fixture loading and validation for MCP Eval Runner.
+ *
+ * Fixture JSON Schema:
+ * {
+ *   "type": "object",
+ *   "required": ["name", "steps"],
+ *   "properties": {
+ *     "name": { "type": "string" },
+ *     "description": { "type": "string" },
+ *     "server": {
+ *       "type": "object",
+ *       "description": "Optional server config for live execution mode",
+ *       "properties": {
+ *         "command": { "type": "string" },
+ *         "args": { "type": "array", "items": { "type": "string" } },
+ *         "env": { "type": "object" }
+ *       },
+ *       "required": ["command"]
+ *     },
+ *     "steps": {
+ *       "type": "array",
+ *       "items": {
+ *         "type": "object",
+ *         "required": ["id", "tool"],
+ *         "properties": {
+ *           "id": { "type": "string" },
+ *           "description": { "type": "string" },
+ *           "tool": { "type": "string" },
+ *           "input": { "type": "object" },
+ *           "expected_output": { "type": "string" },
+ *           "expect": {
+ *             "type": "object",
+ *             "properties": {
+ *               "output_contains": { "type": "string" },
+ *               "output_not_contains": { "type": "string" },
+ *               "output_equals": { "type": "string" },
+ *               "output_matches": { "type": "string" },
+ *               "tool_called": { "type": "string" },
+ *               "latency_under": { "type": "number" }
+ *             }
+ *           }
+ *         }
+ *       }
+ *     }
+ *   }
+ * }
+ */
+import fs from "fs";
+import path from "path";
+import yaml from "js-yaml";
+export class FixtureValidationError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "FixtureValidationError";
+    }
+}
+/**
+ * Parse optional server config block from a fixture.
+ */
+function parseServerConfig(raw) {
+    if (typeof raw !== "object" || raw === null)
+        return undefined;
+    const obj = raw;
+    if (typeof obj["command"] !== "string" || !obj["command"].trim())
+        return undefined;
+    return {
+        command: obj["command"],
+        args: Array.isArray(obj["args"]) ? obj["args"] : [],
+        env: typeof obj["env"] === "object" && obj["env"] !== null
+            ? obj["env"]
+            : undefined,
+    };
+}
+/**
+ * Load and validate a fixture from a YAML or JSON file.
+ */
+export function loadFixture(filePath) {
+    const content = fs.readFileSync(filePath, "utf-8");
+    let raw;
+    if (filePath.endsWith(".yaml") || filePath.endsWith(".yml")) {
+        raw = yaml.load(content);
+    }
+    else if (filePath.endsWith(".json")) {
+        raw = JSON.parse(content);
+    }
+    else {
+        throw new FixtureValidationError(`Unsupported fixture format: ${filePath}. Use .yaml, .yml, or .json`);
+    }
+    return validateFixture(raw, filePath);
+}
+/**
+ * Validate a raw parsed object against the fixture schema.
+ */
+export function validateFixture(raw, source) {
+    if (typeof raw !== "object" || raw === null) {
+        throw new FixtureValidationError(`${source ?? "Fixture"}: must be an object`);
+    }
+    const obj = raw;
+    if (typeof obj["name"] !== "string" || obj["name"].trim() === "") {
+        throw new FixtureValidationError(`${source ?? "Fixture"}: "name" must be a non-empty string`);
+    }
+    if (!Array.isArray(obj["steps"])) {
+        throw new FixtureValidationError(`${source ?? "Fixture"}: "steps" must be an array`);
+    }
+    const steps = obj["steps"].map((step, idx) => {
+        if (typeof step !== "object" || step === null) {
+            throw new FixtureValidationError(`${source ?? "Fixture"}: step[${idx}] must be an object`);
+        }
+        const s = step;
+        if (typeof s["id"] !== "string" || s["id"].trim() === "") {
+            throw new FixtureValidationError(`${source ?? "Fixture"}: step[${idx}].id must be a non-empty string`);
+        }
+        if (typeof s["tool"] !== "string" || s["tool"].trim() === "") {
+            throw new FixtureValidationError(`${source ?? "Fixture"}: step[${idx}].tool must be a non-empty string`);
+        }
+        return {
+            id: s["id"],
+            description: typeof s["description"] === "string" ? s["description"] : undefined,
+            tool: s["tool"],
+            input: typeof s["input"] === "object" && s["input"] !== null
+                ? s["input"]
+                : {},
+            expected_output: typeof s["expected_output"] === "string" ? s["expected_output"] : undefined,
+            expect: typeof s["expect"] === "object" && s["expect"] !== null
+                ? s["expect"]
+                : undefined,
+        };
+    });
+    return {
+        name: obj["name"],
+        description: typeof obj["description"] === "string" ? obj["description"] : undefined,
+        server: parseServerConfig(obj["server"]),
+        steps,
+    };
+}
+/**
+ * Load all fixtures from a directory.
+ */
+export function loadFixturesFromDir(fixturesDir) {
+    if (!fs.existsSync(fixturesDir)) {
+        return [];
+    }
+    const files = fs
+        .readdirSync(fixturesDir)
+        .filter((f) => f.endsWith(".yaml") || f.endsWith(".yml") || f.endsWith(".json"))
+        .sort();
+    const fixtures = [];
+    for (const file of files) {
+        try {
+            fixtures.push(loadFixture(path.join(fixturesDir, file)));
+        }
+        catch (err) {
+            // Log but continue loading other fixtures
+            console.error(`Warning: failed to load fixture ${file}: ${err}`);
+        }
+    }
+    return fixtures;
+}
+/**
+ * Write a fixture to a YAML file in the fixtures directory.
+ */
+export function writeFixture(fixturesDir, fixture) {
+    fs.mkdirSync(fixturesDir, { recursive: true });
+    const fileName = `${fixture.name.replace(/[^a-z0-9_-]/gi, "_")}.yaml`;
+    const filePath = path.join(fixturesDir, fileName);
+    const content = yaml.dump(fixture, { lineWidth: 100 });
+    fs.writeFileSync(filePath, content, "utf-8");
+    return filePath;
+}

package/dist/http-server.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Streamable HTTP transport server for mcp-eval-runner.
+ * Starts an Express HTTP server that handles MCP requests via
+ * the StreamableHTTPServerTransport in stateless mode.
+ */
+import type { ServerOptions } from "./server.js";
+export declare function startHttpServer(port: number, opts: ServerOptions): Promise<void>;

package/dist/http-server.js

@@ -0,0 +1,34 @@
+/**
+ * Streamable HTTP transport server for mcp-eval-runner.
+ * Starts an Express HTTP server that handles MCP requests via
+ * the StreamableHTTPServerTransport in stateless mode.
+ */
+import express from "express";
+import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
+import { createServer } from "./server.js";
+import { createAuthMiddleware } from "./auth.js";
+import { createRateLimiter } from "./rate-limiter.js";
+export async function startHttpServer(port, opts) {
+    const app = express();
+    app.use(express.json());
+    const server = await createServer(opts);
+    // Apply auth and rate limiting before /mcp route
+    app.use("/mcp", createAuthMiddleware());
+    app.use("/mcp", createRateLimiter(60, 60000));
+    app.post("/mcp", async (req, res) => {
+        const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });
+        await server.connect(transport);
+        await transport.handleRequest(req, res, req.body);
+    });
+    app.get("/mcp", async (req, res) => {
+        const transport = new StreamableHTTPServerTransport({ sessionIdGenerator: undefined });
+        await server.connect(transport);
+        await transport.handleRequest(req, res);
+    });
+    app.delete("/mcp", (_req, res) => {
+        res.status(405).json({ error: "Method not allowed" });
+    });
+    app.listen(port, () => {
+        console.error(`MCP Eval Runner HTTP server listening on port ${port}`);
+    });
+}

package/dist/index.d.ts

@@ -0,0 +1,15 @@
+#!/usr/bin/env node
+/**
+ * CLI entry point for mcp-eval-runner.
+ *
+ * Usage:
+ *   mcp-eval-runner [options]
+ *
+ * Options:
+ *   --fixtures, --fixtures-dir  Path to fixtures directory (default: ./evals)
+ *   --db, --db-path             Path to SQLite database (default: ~/.mcp/evals.db)
+ *   --timeout                   Timeout per step in ms (default: 30000)
+ *   --format                    Output format: console|json|html (default: console)
+ *   --watch                     Watch fixtures directory for changes and re-run affected fixtures
+ */
+export {};

package/dist/index.js

@@ -0,0 +1,158 @@
+#!/usr/bin/env node
+/**
+ * CLI entry point for mcp-eval-runner.
+ *
+ * Usage:
+ *   mcp-eval-runner [options]
+ *
+ * Options:
+ *   --fixtures, --fixtures-dir  Path to fixtures directory (default: ./evals)
+ *   --db, --db-path             Path to SQLite database (default: ~/.mcp/evals.db)
+ *   --timeout                   Timeout per step in ms (default: 30000)
+ *   --format                    Output format: console|json|html (default: console)
+ *   --watch                     Watch fixtures directory for changes and re-run affected fixtures
+ */
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+import path from "path";
+import os from "os";
+import { startServer } from "./server.js";
+import { startHttpServer } from "./http-server.js";
+import { EvalDb } from "./db.js";
+import { loadFixture } from "./fixture.js";
+import { runCase } from "./runner.js";
+import { formatResult } from "./reporter.js";
+const argv = yargs(hideBin(process.argv))
+    .option("fixtures", {
+    alias: "fixtures-dir",
+    type: "string",
+    description: "Path to fixtures directory",
+    default: "./evals",
+})
+    .option("db", {
+    alias: "db-path",
+    type: "string",
+    description: "Path to SQLite database",
+    default: "~/.mcp/evals.db",
+})
+    .option("timeout", {
+    type: "number",
+    description: "Timeout per step in milliseconds",
+    default: 30000,
+})
+    .option("format", {
+    type: "string",
+    choices: ["console", "json", "html"],
+    description: "Output format",
+    default: "console",
+})
+    .option("watch", {
+    type: "boolean",
+    description: "Watch fixtures directory for changes and re-run affected fixtures",
+    default: false,
+})
+    .option("concurrency", {
+    type: "number",
+    description: "Number of test cases to run in parallel (default: 1)",
+    default: 1,
+})
+    .option("http-port", {
+    type: "number",
+    description: "Start an HTTP server on this port instead of stdio transport",
+})
+    .help()
+    .parseSync();
+// Resolve fixtures dir relative to cwd
+const fixturesDir = path.resolve(process.cwd(), argv.fixtures);
+// Expand ~ in db path
+const rawDb = argv.db;
+const dbPath = rawDb.startsWith("~/") ? path.join(os.homedir(), rawDb.slice(2)) : rawDb;
+const format = argv.format;
+// Enhanced watch mode: re-run the affected fixture when a file changes
+if (argv.watch) {
+    import("chokidar").then(({ default: chokidar }) => {
+        const watcher = chokidar.watch(fixturesDir, {
+            ignoreInitial: true,
+            persistent: true,
+        });
+        const db = new EvalDb(dbPath);
+        const runnerOpts = {
+            fixturesDir,
+            dbPath,
+            timeoutMs: argv.timeout,
+            format,
+        };
+        watcher.on("all", async (event, filePath) => {
+            process.stderr.write(`[mcp-eval-runner] watch: ${event} ${filePath}\n`);
+            // Only re-run on add / change of fixture files
+            const isFixture = filePath.endsWith(".yaml") || filePath.endsWith(".yml") || filePath.endsWith(".json");
+            if (!isFixture || (event !== "add" && event !== "change")) {
+                return;
+            }
+            try {
+                process.stderr.write(`[mcp-eval-runner] Re-running fixture: ${filePath}\n`);
+                const fixture = loadFixture(filePath);
+                const caseResult = await runCase(fixture, { timeoutMs: argv.timeout });
+                const suiteResult = {
+                    run_id: crypto.randomUUID(),
+                    suite_name: fixture.name,
+                    started_at: Date.now() - caseResult.duration_ms,
+                    ended_at: Date.now(),
+                    total_cases: 1,
+                    passed: caseResult.status === "pass" ? 1 : 0,
+                    failed: caseResult.status !== "pass" ? 1 : 0,
+                    cases: [caseResult],
+                };
+                // Persist result
+                db.insertRun({
+                    id: suiteResult.run_id,
+                    suite_name: suiteResult.suite_name,
+                    started_at: suiteResult.started_at,
+                    ended_at: suiteResult.ended_at,
+                    total_cases: 1,
+                    passed: suiteResult.passed,
+                    failed: suiteResult.failed,
+                    format: runnerOpts.format,
+                });
+                db.insertCaseResult({
+                    id: crypto.randomUUID(),
+                    run_id: suiteResult.run_id,
+                    case_name: caseResult.case_name,
+                    status: caseResult.status,
+                    duration_ms: caseResult.duration_ms,
+                    error_message: caseResult.error ?? null,
+                    assertions_json: JSON.stringify(caseResult.steps.map((s) => s.assertions)),
+                    created_at: Date.now(),
+                });
+                const formatted = formatResult(suiteResult, format);
+                // Write results to stderr so they don't interfere with MCP stdio transport
+                process.stderr.write(formatted + "\n");
+            }
+            catch (err) {
+                process.stderr.write(`[mcp-eval-runner] Error re-running fixture ${filePath}: ${err}\n`);
+            }
+        });
+        process.stderr.write(`[mcp-eval-runner] Watching ${fixturesDir} for changes...\n`);
+    });
+}
+const serverOpts = {
+    fixturesDir,
+    dbPath,
+    timeoutMs: argv.timeout,
+    format,
+    watch: argv.watch,
+    concurrency: argv.concurrency,
+};
+const httpPort = argv["http-port"];
+if (httpPort !== undefined) {
+    startHttpServer(httpPort, serverOpts).catch((err) => {
+        process.stderr.write(`Fatal error: ${err}\n`);
+        process.exit(1);
+    });
+}
+else {
+    startServer(serverOpts).catch((err) => {
+        process.stderr.write(`Fatal error: ${err}\n`);
+        process.exit(1);
+    });
+}

package/dist/llm-judge.d.ts

@@ -0,0 +1,24 @@
+/**
+ * LLM-as-judge assertion for mcp-eval-runner.
+ * Calls an external LLM API via HTTP POST to score semantic similarity
+ * between actual and expected outputs.
+ *
+ * Credentials: LLM_JUDGE_API_KEY + LLM_JUDGE_BASE_URL env vars.
+ * Assertion type: llm_judge with prompt_template, min_score, model fields.
+ */
+import type { AssertionResult } from "./assertions.js";
+export interface LlmJudgeAssertion {
+    prompt_template: string;
+    min_score: number;
+    model: string;
+    expected?: string;
+}
+/**
+ * Run an LLM-as-judge assertion.
+ *
+ * Renders the prompt template with {actual} and {expected} placeholders,
+ * calls the LLM API, and extracts a score from the response (0.0–1.0).
+ * Expects the LLM to respond with a JSON object containing a "score" field,
+ * or a plain number.
+ */
+export declare function runLlmJudge(assertion: LlmJudgeAssertion, actual: string, expected: string): Promise<AssertionResult>;