mcp-eval-runner 1.0.0

package/dist/assertions.js

@@ -0,0 +1,187 @@
+/**
+ * Assertion evaluators for MCP Eval Runner.
+ * Each assertion checks a specific property of a step result.
+ *
+ * Supported assertion types:
+ *   output_contains: "substring"       — output includes substring
+ *   output_not_contains: "substring"   — output must NOT include substring
+ *   output_equals: "exact string"      — output exactly matches
+ *   output_matches: "regex"            — output matches a regular expression
+ *   tool_called: "tool_name"           — step used the named tool
+ *   latency_under: 500                 — latency in ms must be below threshold
+ *   schema_match: { type: "object", properties: {...}, required: [...] }
+ *                                      — output (parsed as JSON) matches JSON Schema
+ *   llm_judge: { prompt_template, min_score, model, expected }
+ *                                      — semantic similarity via LLM judge
+ */
+import { runLlmJudge } from "./llm-judge.js";
+// ── Minimal inline JSON Schema validator ─────────────────────────────────────
+function validateJsonSchema(schema, value, path = "") {
+    // type check
+    if (schema.type !== undefined) {
+        const actualType = Array.isArray(value) ? "array" : typeof value;
+        const nullType = value === null ? "null" : null;
+        const effectiveType = nullType ?? actualType;
+        if (effectiveType !== schema.type) {
+            return `${path || "value"}: expected type "${schema.type}", got "${effectiveType}"`;
+        }
+    }
+    if (schema.type === "object" ||
+        (schema.properties && value !== null && typeof value === "object" && !Array.isArray(value))) {
+        const obj = value;
+        // required fields
+        if (schema.required) {
+            for (const key of schema.required) {
+                if (!(key in obj)) {
+                    return `${path || "value"}: missing required property "${key}"`;
+                }
+            }
+        }
+        // properties
+        if (schema.properties) {
+            for (const [key, subSchema] of Object.entries(schema.properties)) {
+                if (key in obj) {
+                    const err = validateJsonSchema(subSchema, obj[key], path ? `${path}.${key}` : key);
+                    if (err)
+                        return err;
+                }
+            }
+        }
+        // additionalProperties
+        if (schema.additionalProperties === false && schema.properties) {
+            for (const key of Object.keys(obj)) {
+                if (!(key in schema.properties)) {
+                    return `${path || "value"}: unexpected additional property "${key}"`;
+                }
+            }
+        }
+    }
+    if (schema.type === "array" && Array.isArray(value) && schema.items) {
+        for (let i = 0; i < value.length; i++) {
+            const err = validateJsonSchema(schema.items, value[i], `${path || "value"}[${i}]`);
+            if (err)
+                return err;
+        }
+    }
+    return null;
+}
+/**
+ * Evaluate a single assertion against a step result.
+ */
+export function evaluateAssertion(assertion, result) {
+    const results = [];
+    if (assertion.output_contains !== undefined) {
+        const passed = result.output.includes(assertion.output_contains);
+        results.push({
+            type: "output_contains",
+            passed,
+            message: passed
+                ? `Output contains "${assertion.output_contains}"`
+                : `Expected output to contain "${assertion.output_contains}", but got: "${result.output}"`,
+        });
+    }
+    if (assertion.output_not_contains !== undefined) {
+        const passed = !result.output.includes(assertion.output_not_contains);
+        results.push({
+            type: "output_not_contains",
+            passed,
+            message: passed
+                ? `Output does not contain "${assertion.output_not_contains}"`
+                : `Expected output NOT to contain "${assertion.output_not_contains}"`,
+        });
+    }
+    if (assertion.output_matches !== undefined) {
+        let passed = false;
+        let message;
+        try {
+            const regex = new RegExp(assertion.output_matches);
+            passed = regex.test(result.output);
+            message = passed
+                ? `Output matches pattern /${assertion.output_matches}/`
+                : `Output does not match pattern /${assertion.output_matches}/`;
+        }
+        catch {
+            message = `Invalid regex pattern: "${assertion.output_matches}"`;
+        }
+        results.push({ type: "output_matches", passed, message });
+    }
+    if (assertion.output_equals !== undefined) {
+        const passed = result.output === assertion.output_equals;
+        results.push({
+            type: "output_equals",
+            passed,
+            message: passed
+                ? `Output equals "${assertion.output_equals}"`
+                : `Expected output to equal "${assertion.output_equals}", but got: "${result.output}"`,
+        });
+    }
+    if (assertion.tool_called !== undefined) {
+        const passed = result.tool === assertion.tool_called;
+        results.push({
+            type: "tool_called",
+            passed,
+            message: passed
+                ? `Tool "${assertion.tool_called}" was called`
+                : `Expected tool "${assertion.tool_called}" to be called, but got "${result.tool}"`,
+        });
+    }
+    if (assertion.latency_under !== undefined) {
+        const passed = result.latency_ms < assertion.latency_under;
+        results.push({
+            type: "latency_under",
+            passed,
+            message: passed
+                ? `Latency ${result.latency_ms}ms < ${assertion.latency_under}ms`
+                : `Expected latency under ${assertion.latency_under}ms, but got ${result.latency_ms}ms`,
+        });
+    }
+    if (assertion.schema_match !== undefined) {
+        let parsed;
+        let parseError;
+        try {
+            parsed = JSON.parse(result.output);
+        }
+        catch {
+            parseError = `Output is not valid JSON: "${result.output}"`;
+        }
+        if (parseError) {
+            results.push({
+                type: "schema_match",
+                passed: false,
+                message: parseError,
+            });
+        }
+        else {
+            const err = validateJsonSchema(assertion.schema_match, parsed);
+            const passed = err === null;
+            results.push({
+                type: "schema_match",
+                passed,
+                message: passed ? `Output matches JSON schema` : `Schema validation failed: ${err}`,
+            });
+        }
+    }
+    return results;
+}
+/**
+ * Evaluate all assertions for a step, returning aggregate pass/fail.
+ */
+export function evaluateAllAssertions(assertions, result) {
+    const results = evaluateAssertion(assertions, result);
+    const passed = results.every((r) => r.passed);
+    return { passed, results };
+}
+/**
+ * Evaluate all assertions for a step including async assertion types (e.g. llm_judge).
+ */
+export async function evaluateAllAssertionsAsync(assertions, result) {
+    const results = evaluateAssertion(assertions, result);
+    // Handle async llm_judge assertion
+    if (assertions.llm_judge !== undefined) {
+        const expected = assertions.llm_judge.expected ?? result.output;
+        const judgeResult = await runLlmJudge(assertions.llm_judge, result.output, expected);
+        results.push(judgeResult);
+    }
+    const passed = results.every((r) => r.passed);
+    return { passed, results };
+}

package/dist/audit-log.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Append-only audit log for mcp-eval-runner eval runs.
+ * Logs are written to ~/.mcp/eval-runner-audit.jsonl (one JSON object per line).
+ * Supports recording individual entries and exporting filtered ranges.
+ */
+export interface AuditEntry {
+    timestamp: string;
+    run_id: string;
+    fixture_name: string;
+    passed: boolean;
+    duration_ms: number;
+    user_id?: string;
+}
+export declare class AuditLog {
+    private filePath;
+    constructor(filePath?: string);
+    /**
+     * Append a single audit entry to the log file.
+     */
+    record(entry: AuditEntry): void;
+    /**
+     * Export audit entries, optionally filtered by ISO timestamp range [from, to].
+     * Both `from` and `to` are inclusive ISO date strings.
+     */
+    export(from?: string, to?: string): AuditEntry[];
+}

package/dist/audit-log.js

@@ -0,0 +1,57 @@
+/**
+ * Append-only audit log for mcp-eval-runner eval runs.
+ * Logs are written to ~/.mcp/eval-runner-audit.jsonl (one JSON object per line).
+ * Supports recording individual entries and exporting filtered ranges.
+ */
+import fs from "fs";
+import path from "path";
+import os from "os";
+const DEFAULT_AUDIT_PATH = path.join(os.homedir(), ".mcp", "eval-runner-audit.jsonl");
+export class AuditLog {
+    filePath;
+    constructor(filePath) {
+        this.filePath = filePath ?? DEFAULT_AUDIT_PATH;
+        // Ensure parent directory exists
+        fs.mkdirSync(path.dirname(this.filePath), { recursive: true });
+    }
+    /**
+     * Append a single audit entry to the log file.
+     */
+    record(entry) {
+        const line = JSON.stringify(entry) + "\n";
+        fs.appendFileSync(this.filePath, line, "utf-8");
+    }
+    /**
+     * Export audit entries, optionally filtered by ISO timestamp range [from, to].
+     * Both `from` and `to` are inclusive ISO date strings.
+     */
+    export(from, to) {
+        if (!fs.existsSync(this.filePath)) {
+            return [];
+        }
+        const raw = fs.readFileSync(this.filePath, "utf-8");
+        const lines = raw
+            .split("\n")
+            .map((l) => l.trim())
+            .filter(Boolean);
+        const entries = [];
+        for (const line of lines) {
+            try {
+                entries.push(JSON.parse(line));
+            }
+            catch {
+                // Skip malformed lines
+            }
+        }
+        let filtered = entries;
+        if (from !== undefined) {
+            const fromTs = new Date(from).getTime();
+            filtered = filtered.filter((e) => new Date(e.timestamp).getTime() >= fromTs);
+        }
+        if (to !== undefined) {
+            const toTs = new Date(to).getTime();
+            filtered = filtered.filter((e) => new Date(e.timestamp).getTime() <= toTs);
+        }
+        return filtered;
+    }
+}

package/dist/auth.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Authentication middleware for mcp-eval-runner HTTP server.
+ * Supports X-API-Key header validation and HMAC-SHA256 JWT Bearer token validation.
+ * Pass-through when neither MCP_API_KEY nor MCP_JWT_SECRET env vars are set.
+ */
+import type { RequestHandler } from "express";
+/**
+ * Create Express authentication middleware.
+ *
+ * Behavior:
+ * - If MCP_API_KEY is set, validates X-API-Key header; returns 401 if missing or mismatched.
+ * - If MCP_JWT_SECRET is set, validates Authorization: Bearer <token> as HMAC-SHA256 JWT.
+ * - If neither env var is set, passes through all requests.
+ */
+export declare function createAuthMiddleware(): RequestHandler;

package/dist/auth.js

@@ -0,0 +1,83 @@
+/**
+ * Authentication middleware for mcp-eval-runner HTTP server.
+ * Supports X-API-Key header validation and HMAC-SHA256 JWT Bearer token validation.
+ * Pass-through when neither MCP_API_KEY nor MCP_JWT_SECRET env vars are set.
+ */
+import crypto from "crypto";
+/**
+ * Decode a base64url-encoded string to a UTF-8 string.
+ */
+function _base64urlDecode(input) {
+    // Convert base64url to base64
+    const base64 = input.replace(/-/g, "+").replace(/_/g, "/");
+    const padded = base64.padEnd(base64.length + ((4 - (base64.length % 4)) % 4), "=");
+    return Buffer.from(padded, "base64").toString("utf-8");
+}
+/**
+ * Verify a JWT token using HMAC-SHA256 with the given secret.
+ * JWT format: base64url(header).base64url(payload).base64url(signature)
+ * Signature = HMAC-SHA256(secret, "header.payload")
+ */
+function verifyJwt(token, secret) {
+    const parts = token.split(".");
+    if (parts.length !== 3) {
+        return false;
+    }
+    const [headerB64, payloadB64, signatureB64] = parts;
+    const signingInput = `${headerB64}.${payloadB64}`;
+    // Compute expected signature
+    const expectedSig = crypto.createHmac("sha256", secret).update(signingInput).digest("base64url");
+    // Constant-time comparison to prevent timing attacks
+    try {
+        const expectedBuf = Buffer.from(expectedSig, "base64url");
+        const actualBuf = Buffer.from(signatureB64, "base64url");
+        if (expectedBuf.length !== actualBuf.length) {
+            return false;
+        }
+        return crypto.timingSafeEqual(expectedBuf, actualBuf);
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Create Express authentication middleware.
+ *
+ * Behavior:
+ * - If MCP_API_KEY is set, validates X-API-Key header; returns 401 if missing or mismatched.
+ * - If MCP_JWT_SECRET is set, validates Authorization: Bearer <token> as HMAC-SHA256 JWT.
+ * - If neither env var is set, passes through all requests.
+ */
+export function createAuthMiddleware() {
+    return (req, res, next) => {
+        const apiKey = process.env.MCP_API_KEY;
+        const jwtSecret = process.env.MCP_JWT_SECRET;
+        // If neither auth mechanism is configured, pass through
+        if (!apiKey && !jwtSecret) {
+            next();
+            return;
+        }
+        // Validate X-API-Key if MCP_API_KEY is set
+        if (apiKey) {
+            const providedKey = req.headers["x-api-key"];
+            if (!providedKey || providedKey !== apiKey) {
+                res.status(401).json({ error: "Unauthorized: invalid or missing API key" });
+                return;
+            }
+        }
+        // Validate JWT Bearer token if MCP_JWT_SECRET is set
+        if (jwtSecret) {
+            const authHeader = req.headers["authorization"];
+            if (!authHeader || !authHeader.startsWith("Bearer ")) {
+                res.status(401).json({ error: "Unauthorized: missing Bearer token" });
+                return;
+            }
+            const token = authHeader.slice("Bearer ".length).trim();
+            if (!verifyJwt(token, jwtSecret)) {
+                res.status(401).json({ error: "Unauthorized: invalid JWT signature" });
+                return;
+            }
+        }
+        next();
+    };
+}

package/dist/db.d.ts

@@ -0,0 +1,40 @@
+/**
+ * SQLite schema and queries for MCP Eval Runner run history.
+ * Uses the built-in node:sqlite module (Node.js >= 22.5).
+ */
+export interface RunRecord {
+    id: string;
+    suite_name: string;
+    started_at: number;
+    ended_at: number | null;
+    total_cases: number;
+    passed: number;
+    failed: number;
+    format: string;
+}
+export interface CaseResultRecord {
+    id: string;
+    run_id: string;
+    case_name: string;
+    status: "pass" | "fail" | "error";
+    duration_ms: number | null;
+    error_message: string | null;
+    assertions_json: string | null;
+    created_at: number;
+}
+/**
+ * Expand ~ to the user's home directory.
+ */
+export declare function expandHome(p: string): string;
+export declare class EvalDb {
+    private db;
+    constructor(dbPath: string);
+    insertRun(run: RunRecord): void;
+    updateRun(id: string, updates: Partial<Pick<RunRecord, "ended_at" | "total_cases" | "passed" | "failed">>): void;
+    insertCaseResult(result: CaseResultRecord): void;
+    getLastRun(suiteName?: string): RunRecord | undefined;
+    getRunById(id: string): RunRecord | undefined;
+    getCaseResultsForRun(runId: string): CaseResultRecord[];
+    getAllRuns(limit?: number): RunRecord[];
+    close(): void;
+}

package/dist/db.js

@@ -0,0 +1,94 @@
+/**
+ * SQLite schema and queries for MCP Eval Runner run history.
+ * Uses the built-in node:sqlite module (Node.js >= 22.5).
+ */
+// node:sqlite is a stable built-in since Node 22.5
+import { DatabaseSync } from "node:sqlite";
+import path from "path";
+import os from "os";
+import fs from "fs";
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS runs (
+  id TEXT PRIMARY KEY,
+  suite_name TEXT NOT NULL,
+  started_at INTEGER NOT NULL,
+  ended_at INTEGER,
+  total_cases INTEGER NOT NULL DEFAULT 0,
+  passed INTEGER NOT NULL DEFAULT 0,
+  failed INTEGER NOT NULL DEFAULT 0,
+  format TEXT NOT NULL DEFAULT 'console'
+);
+
+CREATE TABLE IF NOT EXISTS case_results (
+  id TEXT PRIMARY KEY,
+  run_id TEXT NOT NULL,
+  case_name TEXT NOT NULL,
+  status TEXT NOT NULL,
+  duration_ms INTEGER,
+  error_message TEXT,
+  assertions_json TEXT,
+  created_at INTEGER NOT NULL,
+  FOREIGN KEY (run_id) REFERENCES runs(id)
+);
+`;
+/**
+ * Expand ~ to the user's home directory.
+ */
+export function expandHome(p) {
+    if (p.startsWith("~/") || p === "~") {
+        return path.join(os.homedir(), p.slice(1));
+    }
+    return p;
+}
+export class EvalDb {
+    db;
+    constructor(dbPath) {
+        const resolved = expandHome(dbPath);
+        const dir = path.dirname(resolved);
+        fs.mkdirSync(dir, { recursive: true });
+        this.db = new DatabaseSync(resolved);
+        this.db.exec(SCHEMA);
+    }
+    insertRun(run) {
+        const stmt = this.db.prepare(`INSERT INTO runs (id, suite_name, started_at, ended_at, total_cases, passed, failed, format)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
+        stmt.run(run.id, run.suite_name, run.started_at, run.ended_at, run.total_cases, run.passed, run.failed, run.format);
+    }
+    updateRun(id, updates) {
+        const entries = Object.entries(updates);
+        if (entries.length === 0)
+            return;
+        const fields = entries.map(([k]) => `${k} = ?`).join(", ");
+        const values = entries.map(([, v]) => v);
+        this.db.prepare(`UPDATE runs SET ${fields} WHERE id = ?`).run(...values, id);
+    }
+    insertCaseResult(result) {
+        const stmt = this.db.prepare(`INSERT INTO case_results (id, run_id, case_name, status, duration_ms, error_message, assertions_json, created_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?)`);
+        stmt.run(result.id, result.run_id, result.case_name, result.status, result.duration_ms, result.error_message, result.assertions_json, result.created_at);
+    }
+    getLastRun(suiteName) {
+        if (suiteName) {
+            return this.db
+                .prepare(`SELECT * FROM runs WHERE suite_name = ? ORDER BY started_at DESC LIMIT 1`)
+                .get(suiteName);
+        }
+        return this.db.prepare(`SELECT * FROM runs ORDER BY started_at DESC LIMIT 1`).get();
+    }
+    getRunById(id) {
+        return this.db.prepare(`SELECT * FROM runs WHERE id = ?`).get(id);
+    }
+    getCaseResultsForRun(runId) {
+        return this.db
+            .prepare(`SELECT * FROM case_results WHERE run_id = ? ORDER BY created_at ASC`)
+            .all(runId);
+    }
+    getAllRuns(limit = 50) {
+        return this.db
+            .prepare(`SELECT * FROM runs ORDER BY started_at DESC LIMIT ?`)
+            .all(limit);
+    }
+    close() {
+        this.db.close();
+    }
+}

package/dist/deployment-gate.d.ts

@@ -0,0 +1,27 @@
+/**
+ * CI deployment gate for mcp-eval-runner.
+ * Evaluates whether a recent set of runs meets a minimum pass rate threshold.
+ * Intended for use in CI pipelines to block deploys on regressions.
+ */
+import type { EvalDb } from "./db.js";
+export interface GateConfig {
+    /** Optional: filter runs by workflow/suite name */
+    workflow_name?: string;
+    /** Minimum acceptable pass rate (0.0 – 1.0) */
+    min_pass_rate: number;
+    /** Number of most-recent runs to consider (default: 10) */
+    lookback_runs?: number;
+}
+export interface GateResult {
+    passed: boolean;
+    current_rate: number;
+    threshold: number;
+    run_count: number;
+}
+/**
+ * Evaluate the deployment gate against recent run history.
+ *
+ * Queries the last `lookback_runs` runs (optionally filtered by suite name),
+ * computes the aggregate pass rate, and compares it to `min_pass_rate`.
+ */
+export declare function evaluateGate(db: EvalDb, config: GateConfig): GateResult;

package/dist/deployment-gate.js

@@ -0,0 +1,43 @@
+/**
+ * CI deployment gate for mcp-eval-runner.
+ * Evaluates whether a recent set of runs meets a minimum pass rate threshold.
+ * Intended for use in CI pipelines to block deploys on regressions.
+ */
+/**
+ * Evaluate the deployment gate against recent run history.
+ *
+ * Queries the last `lookback_runs` runs (optionally filtered by suite name),
+ * computes the aggregate pass rate, and compares it to `min_pass_rate`.
+ */
+export function evaluateGate(db, config) {
+    const lookback = config.lookback_runs ?? 10;
+    const threshold = config.min_pass_rate;
+    // Fetch recent runs from the database
+    const allRuns = db.getAllRuns(lookback * 10); // over-fetch then filter
+    const relevantRuns = config.workflow_name
+        ? allRuns.filter((r) => r.suite_name === config.workflow_name).slice(0, lookback)
+        : allRuns.slice(0, lookback);
+    const runCount = relevantRuns.length;
+    if (runCount === 0) {
+        return {
+            passed: false,
+            current_rate: 0,
+            threshold,
+            run_count: 0,
+        };
+    }
+    // Aggregate total_cases and passed across all relevant runs
+    let totalCases = 0;
+    let totalPassed = 0;
+    for (const run of relevantRuns) {
+        totalCases += run.total_cases;
+        totalPassed += run.passed;
+    }
+    const currentRate = totalCases > 0 ? totalPassed / totalCases : 0;
+    return {
+        passed: currentRate >= threshold,
+        current_rate: currentRate,
+        threshold,
+        run_count: runCount,
+    };
+}

package/dist/fixture-library.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Shared fixture discovery and publishing for mcp-eval-runner.
+ * Supports discovering fixtures across multiple directories and publishing
+ * (copying) fixtures to a destination directory.
+ */
+export interface FixtureEntry {
+    name: string;
+    path: string;
+    suite_count: number;
+    case_count: number;
+}
+/**
+ * Discover all fixture files across the given directories.
+ * Returns a deduplicated list (by name, first occurrence wins).
+ *
+ * @param dirs - Array of directory paths to scan for fixtures
+ */
+export declare function discoverFixtures(dirs: string[]): FixtureEntry[];
+/**
+ * Publish (copy) a fixture YAML/JSON file to the destination directory.
+ * Creates the destination directory if it does not exist.
+ *
+ * @param fixture - The fixture object or a path to a fixture file
+ * @param dest    - Destination directory path
+ */
+export declare function publishFixture(fixture: unknown, dest: string): void;