mcp-eval-runner 1.0.0

package/dist/llm-judge.js

@@ -0,0 +1,139 @@
+/**
+ * LLM-as-judge assertion for mcp-eval-runner.
+ * Calls an external LLM API via HTTP POST to score semantic similarity
+ * between actual and expected outputs.
+ *
+ * Credentials: LLM_JUDGE_API_KEY + LLM_JUDGE_BASE_URL env vars.
+ * Assertion type: llm_judge with prompt_template, min_score, model fields.
+ */
+import https from "https";
+import http from "http";
+/**
+ * Make an HTTP/HTTPS POST request and return the parsed JSON response.
+ */
+function httpPost(url, body, headers) {
+    return new Promise((resolve, reject) => {
+        const parsed = new URL(url);
+        const isHttps = parsed.protocol === "https:";
+        const transport = isHttps ? https : http;
+        const bodyStr = JSON.stringify(body);
+        const reqHeaders = {
+            "Content-Type": "application/json",
+            "Content-Length": Buffer.byteLength(bodyStr).toString(),
+            ...headers,
+        };
+        const options = {
+            hostname: parsed.hostname,
+            port: parsed.port || (isHttps ? 443 : 80),
+            path: parsed.pathname + (parsed.search || ""),
+            method: "POST",
+            headers: reqHeaders,
+        };
+        const req = transport.request(options, (res) => {
+            const chunks = [];
+            res.on("data", (chunk) => chunks.push(chunk));
+            res.on("end", () => {
+                try {
+                    const text = Buffer.concat(chunks).toString("utf-8");
+                    resolve(JSON.parse(text));
+                }
+                catch (err) {
+                    reject(new Error(`Failed to parse LLM API response: ${err}`));
+                }
+            });
+        });
+        req.on("error", reject);
+        req.write(bodyStr);
+        req.end();
+    });
+}
+/**
+ * Run an LLM-as-judge assertion.
+ *
+ * Renders the prompt template with {actual} and {expected} placeholders,
+ * calls the LLM API, and extracts a score from the response (0.0–1.0).
+ * Expects the LLM to respond with a JSON object containing a "score" field,
+ * or a plain number.
+ */
+export async function runLlmJudge(assertion, actual, expected) {
+    const apiKey = process.env.LLM_JUDGE_API_KEY;
+    const baseUrl = process.env.LLM_JUDGE_BASE_URL;
+    if (!apiKey || !baseUrl) {
+        return {
+            type: "llm_judge",
+            passed: false,
+            message: "LLM judge not configured: missing LLM_JUDGE_API_KEY or LLM_JUDGE_BASE_URL env vars",
+        };
+    }
+    // Render the prompt template
+    const prompt = assertion.prompt_template
+        .replace(/\{actual\}/g, actual)
+        .replace(/\{expected\}/g, expected);
+    const requestBody = {
+        model: assertion.model,
+        messages: [{ role: "user", content: prompt }],
+        max_tokens: 256,
+    };
+    const endpoint = baseUrl.replace(/\/$/, "") + "/chat/completions";
+    let response;
+    try {
+        response = await httpPost(endpoint, requestBody, {
+            Authorization: `Bearer ${apiKey}`,
+        });
+    }
+    catch (err) {
+        return {
+            type: "llm_judge",
+            passed: false,
+            message: `LLM judge HTTP error: ${err instanceof Error ? err.message : String(err)}`,
+        };
+    }
+    if (response.error) {
+        return {
+            type: "llm_judge",
+            passed: false,
+            message: `LLM judge API error: ${response.error.message}`,
+        };
+    }
+    const content = response.choices?.[0]?.message?.content ?? "";
+    let score;
+    // Try parsing as JSON with a "score" field first
+    try {
+        // Extract JSON from the response if it's embedded in text
+        const jsonMatch = content.match(/\{[^}]*"score"\s*:\s*([0-9.]+)[^}]*\}/);
+        if (jsonMatch) {
+            const parsed = JSON.parse(jsonMatch[0]);
+            if (typeof parsed.score === "number") {
+                score = parsed.score;
+            }
+        }
+    }
+    catch {
+        // fall through to numeric parsing
+    }
+    // Try parsing the entire content as a plain number
+    if (score === undefined) {
+        const trimmed = content.trim();
+        const num = parseFloat(trimmed);
+        if (!isNaN(num)) {
+            score = num;
+        }
+    }
+    if (score === undefined) {
+        return {
+            type: "llm_judge",
+            passed: false,
+            message: `LLM judge could not extract a score from response: "${content}"`,
+        };
+    }
+    // Clamp score to [0, 1]
+    const clampedScore = Math.max(0, Math.min(1, score));
+    const passed = clampedScore >= assertion.min_score;
+    return {
+        type: "llm_judge",
+        passed,
+        message: passed
+            ? `LLM judge score ${clampedScore.toFixed(3)} >= threshold ${assertion.min_score}`
+            : `LLM judge score ${clampedScore.toFixed(3)} < threshold ${assertion.min_score}`,
+    };
+}

package/dist/rate-limiter.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Sliding window in-memory rate limiter middleware for mcp-eval-runner.
+ * Tracks requests per IP address or API key (X-API-Key header).
+ * Returns 429 Too Many Requests when the limit is exceeded.
+ */
+import type { RequestHandler } from "express";
+/**
+ * Create a sliding window rate limiter Express middleware.
+ *
+ * @param maxRequests - Maximum number of requests allowed per window (default: 60)
+ * @param windowMs - Window duration in milliseconds (default: 60000 = 60s)
+ */
+export declare function createRateLimiter(maxRequests?: number, windowMs?: number): RequestHandler;

package/dist/rate-limiter.js

@@ -0,0 +1,36 @@
+/**
+ * Sliding window in-memory rate limiter middleware for mcp-eval-runner.
+ * Tracks requests per IP address or API key (X-API-Key header).
+ * Returns 429 Too Many Requests when the limit is exceeded.
+ */
+/**
+ * Create a sliding window rate limiter Express middleware.
+ *
+ * @param maxRequests - Maximum number of requests allowed per window (default: 60)
+ * @param windowMs - Window duration in milliseconds (default: 60000 = 60s)
+ */
+export function createRateLimiter(maxRequests = 60, windowMs = 60000) {
+    const store = new Map();
+    return (req, res, next) => {
+        const now = Date.now();
+        const windowStart = now - windowMs;
+        // Identify the client by API key or IP address
+        const apiKey = req.headers["x-api-key"];
+        const clientKey = (Array.isArray(apiKey) ? apiKey[0] : apiKey) ?? req.ip ?? "unknown";
+        // Get or create the window entry for this client
+        let entry = store.get(clientKey);
+        if (!entry) {
+            entry = { timestamps: [] };
+            store.set(clientKey, entry);
+        }
+        // Prune timestamps outside the current window
+        entry.timestamps = entry.timestamps.filter((ts) => ts > windowStart);
+        if (entry.timestamps.length >= maxRequests) {
+            res.status(429).json({ error: "Too Many Requests: rate limit exceeded" });
+            return;
+        }
+        // Record this request
+        entry.timestamps.push(now);
+        next();
+    };
+}

package/dist/reporter.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Console/JSON/HTML output formatting for MCP Eval Runner.
+ */
+import type { SuiteRunResult } from "./runner.js";
+export declare function formatConsole(result: SuiteRunResult): string;
+export declare function formatJson(result: SuiteRunResult): string;
+export declare function formatHtml(result: SuiteRunResult): string;
+export declare function formatResult(result: SuiteRunResult, format: "console" | "json" | "html"): string;

package/dist/reporter.js

@@ -0,0 +1,163 @@
+/**
+ * Console/JSON/HTML output formatting for MCP Eval Runner.
+ */
+// ANSI color codes
+const RESET = "\x1b[0m";
+const GREEN = "\x1b[32m";
+const RED = "\x1b[31m";
+const YELLOW = "\x1b[33m";
+const BOLD = "\x1b[1m";
+const DIM = "\x1b[2m";
+function pass(s) {
+    return `${GREEN}${s}${RESET}`;
+}
+function fail(s) {
+    return `${RED}${s}${RESET}`;
+}
+function warn(s) {
+    return `${YELLOW}${s}${RESET}`;
+}
+function bold(s) {
+    return `${BOLD}${s}${RESET}`;
+}
+function dim(s) {
+    return `${DIM}${s}${RESET}`;
+}
+function statusIcon(status) {
+    switch (status) {
+        case "pass":
+            return pass("✓");
+        case "fail":
+            return fail("✗");
+        case "error":
+            return warn("!");
+    }
+}
+function formatAssertions(assertions, indent) {
+    return assertions
+        .map((a) => {
+        const icon = a.passed ? pass("  ✓") : fail("  ✗");
+        return `${indent}${icon} [${a.type}] ${a.passed ? dim(a.message) : fail(a.message)}`;
+    })
+        .join("\n");
+}
+function formatStep(step, indent = "    ") {
+    const icon = statusIcon(step.status);
+    const lines = [
+        `${indent}${icon} step:${step.step_id} (tool: ${step.tool}) ${dim(`${step.duration_ms}ms`)}`,
+    ];
+    if (step.assertions.length > 0) {
+        lines.push(formatAssertions(step.assertions, indent + "  "));
+    }
+    if (step.error) {
+        lines.push(`${indent}  ${fail("Error: " + step.error)}`);
+    }
+    return lines.join("\n");
+}
+function formatCase(c) {
+    const icon = statusIcon(c.status);
+    const lines = [`  ${icon} ${bold(c.case_name)} ${dim(`(${c.duration_ms}ms)`)}`];
+    for (const step of c.steps) {
+        lines.push(formatStep(step));
+    }
+    if (c.error) {
+        lines.push(`    ${fail("Error: " + c.error)}`);
+    }
+    return lines.join("\n");
+}
+export function formatConsole(result) {
+    const lines = [];
+    lines.push("");
+    lines.push(bold(`MCP Eval Runner — Suite: ${result.suite_name}`));
+    lines.push(dim(`Run ID: ${result.run_id}`));
+    lines.push(dim(`Started: ${new Date(result.started_at).toISOString()}  Duration: ${result.ended_at - result.started_at}ms`));
+    lines.push("");
+    for (const c of result.cases) {
+        lines.push(formatCase(c));
+    }
+    lines.push("");
+    const summary = result.failed === 0
+        ? pass(`✓ ${result.passed}/${result.total_cases} passed`)
+        : fail(`✗ ${result.failed}/${result.total_cases} failed`) +
+            (result.passed > 0 ? `, ${pass(String(result.passed))} passed` : "");
+    lines.push(bold("Summary: ") + summary);
+    lines.push("");
+    return lines.join("\n");
+}
+export function formatJson(result) {
+    return JSON.stringify(result, null, 2);
+}
+export function formatHtml(result) {
+    const rows = result.cases
+        .map((c) => {
+        const color = c.status === "pass" ? "#22c55e" : c.status === "fail" ? "#ef4444" : "#f59e0b";
+        const stepsHtml = c.steps
+            .map((s) => {
+            const sc = s.status === "pass" ? "#22c55e" : "#ef4444";
+            const assertionsHtml = s.assertions
+                .map((a) => `<li style="color:${a.passed ? "#22c55e" : "#ef4444"}">${a.type}: ${escapeHtml(a.message)}</li>`)
+                .join("");
+            return `
+          <tr>
+            <td style="padding-left:2em;color:${sc}">${escapeHtml(s.step_id)}</td>
+            <td>${escapeHtml(s.tool)}</td>
+            <td style="color:${sc}">${s.status}</td>
+            <td>${s.duration_ms}ms</td>
+            <td><ul>${assertionsHtml}</ul></td>
+          </tr>`;
+        })
+            .join("");
+        return `
+        <tr>
+          <td colspan="5" style="font-weight:bold;color:${color}">${escapeHtml(c.case_name)} — ${c.status} (${c.duration_ms}ms)</td>
+        </tr>
+        ${stepsHtml}`;
+    })
+        .join("");
+    return `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <title>MCP Eval Runner Report</title>
+  <style>
+    body { font-family: monospace; background: #111; color: #eee; padding: 2em; }
+    table { border-collapse: collapse; width: 100%; }
+    th, td { text-align: left; padding: 4px 8px; border-bottom: 1px solid #333; }
+    th { background: #222; }
+    .pass { color: #22c55e; }
+    .fail { color: #ef4444; }
+  </style>
+</head>
+<body>
+  <h1>MCP Eval Runner</h1>
+  <p>Suite: <strong>${escapeHtml(result.suite_name)}</strong> | Run ID: ${escapeHtml(result.run_id)}</p>
+  <p>Started: ${new Date(result.started_at).toISOString()} | Duration: ${result.ended_at - result.started_at}ms</p>
+  <p class="${result.failed === 0 ? "pass" : "fail"}">
+    ${result.passed}/${result.total_cases} passed, ${result.failed} failed
+  </p>
+  <table>
+    <thead>
+      <tr><th>Case / Step</th><th>Tool</th><th>Status</th><th>Duration</th><th>Assertions</th></tr>
+    </thead>
+    <tbody>${rows}</tbody>
+  </table>
+</body>
+</html>`;
+}
+function escapeHtml(s) {
+    return s
+        .replace(/&/g, "&amp;")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;")
+        .replace(/"/g, "&quot;");
+}
+export function formatResult(result, format) {
+    switch (format) {
+        case "json":
+            return formatJson(result);
+        case "html":
+            return formatHtml(result);
+        default:
+            return formatConsole(result);
+    }
+}

package/dist/runner.d.ts

@@ -0,0 +1,57 @@
+/**
+ * Test execution engine for MCP Eval Runner.
+ *
+ * Supports two execution modes:
+ *
+ * 1. Live mode (default when server config is present in fixture):
+ *    Spawns the target MCP server as a child process (stdio transport) or
+ *    connects via HTTP, calls each step's tool with the provided input, and
+ *    evaluates assertions against the real response.
+ *
+ * 2. Simulation mode (fallback when no server config is present):
+ *    Evaluates assertions against `expected_output` from the fixture.
+ *    Useful for authoring and CI dry-runs without a running server.
+ *
+ * Step output piping:
+ *    Steps can reference the output of a previous step using the
+ *    `{{steps.<step_id>.output}}` placeholder in their `input` values.
+ */
+import { type AssertionResult } from "./assertions.js";
+import type { Fixture } from "./fixture.js";
+import type { EvalDb } from "./db.js";
+export interface StepRunResult {
+    step_id: string;
+    tool: string;
+    status: "pass" | "fail" | "error";
+    duration_ms: number;
+    output: string;
+    assertions: AssertionResult[];
+    error?: string;
+    mode: "live" | "simulation";
+}
+export interface CaseRunResult {
+    case_name: string;
+    status: "pass" | "fail" | "error";
+    duration_ms: number;
+    steps: StepRunResult[];
+    error?: string;
+}
+export interface SuiteRunResult {
+    run_id: string;
+    suite_name: string;
+    started_at: number;
+    ended_at: number;
+    total_cases: number;
+    passed: number;
+    failed: number;
+    cases: CaseRunResult[];
+}
+export interface RunnerOptions {
+    fixturesDir: string;
+    dbPath: string;
+    timeoutMs: number;
+    format: "console" | "json" | "html";
+    concurrency?: number;
+}
+export declare function runCase(fixture: Fixture, options: Pick<RunnerOptions, "timeoutMs">): Promise<CaseRunResult>;
+export declare function runSuite(fixtures: Fixture[], suiteName: string, options: RunnerOptions, db: EvalDb): Promise<SuiteRunResult>;