npm - mcp-eval-runner - Versions diffs - 1.0.0 - Mend

mcp-eval-runner 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (48) hide show

package/.env.example +39 -0
package/CHANGELOG.md +67 -0
package/LICENSE +21 -0
package/README.md +328 -0
package/dist/assertions.d.ts +63 -0
package/dist/assertions.js +187 -0
package/dist/audit-log.d.ts +26 -0
package/dist/audit-log.js +57 -0
package/dist/auth.d.ts +15 -0
package/dist/auth.js +83 -0
package/dist/db.d.ts +40 -0
package/dist/db.js +94 -0
package/dist/deployment-gate.d.ts +27 -0
package/dist/deployment-gate.js +43 -0
package/dist/fixture-library.d.ts +26 -0
package/dist/fixture-library.js +85 -0
package/dist/fixture.d.ts +87 -0
package/dist/fixture.js +170 -0
package/dist/http-server.d.ts +7 -0
package/dist/http-server.js +34 -0
package/dist/index.d.ts +15 -0
package/dist/index.js +158 -0
package/dist/llm-judge.d.ts +24 -0
package/dist/llm-judge.js +139 -0
package/dist/rate-limiter.d.ts +13 -0
package/dist/rate-limiter.js +36 -0
package/dist/reporter.d.ts +8 -0
package/dist/reporter.js +163 -0
package/dist/runner.d.ts +57 -0
package/dist/runner.js +339 -0
package/dist/server.d.ts +22 -0
package/dist/server.js +583 -0
package/dist/tools/html_report.d.ts +8 -0
package/dist/tools/html_report.js +188 -0
package/dist/tools/manage.d.ts +11 -0
package/dist/tools/manage.js +41 -0
package/dist/tools/report.d.ts +12 -0
package/dist/tools/report.js +120 -0
package/dist/tools/run.d.ts +20 -0
package/dist/tools/run.js +166 -0
package/dist/tools/scaffold.d.ts +11 -0
package/dist/tools/scaffold.js +90 -0
package/evals/reference/mcp-fetch.yaml +46 -0
package/evals/reference/mcp-filesystem.yaml +63 -0
package/evals/reference/mcp-memory.yaml +70 -0
package/evals/reference/step-piping-example.yaml +25 -0
package/evals/smoke.yaml +12 -0
package/package.json +67 -0

package/dist/tools/html_report.js ADDED Viewed

@@ -0,0 +1,188 @@
+/**
+ * generate_html_report tool implementation.
+ *
+ * Generates a full single-file HTML report for a given run_id.
+ * All styles are inlined — no external CDN required.
+ */
+function escapeHtml(s) {
+    return s
+        .replace(/&/g, "&amp;")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;")
+        .replace(/"/g, "&quot;");
+}
+function statusBadge(status) {
+    const color = status === "pass" ? "#22c55e" : status === "fail" ? "#ef4444" : "#f59e0b";
+    return `<span style="display:inline-block;padding:2px 8px;border-radius:4px;background:${color};color:#fff;font-weight:bold;font-size:.85em">${escapeHtml(status)}</span>`;
+}
+function assertionRows(assertions) {
+    if (assertions.length === 0) {
+        return `<tr><td colspan="3" style="color:#888;font-style:italic">no assertions</td></tr>`;
+    }
+    return assertions
+        .map((a) => {
+        const color = a.passed ? "#22c55e" : "#ef4444";
+        return `<tr>
+        <td style="color:${color}">${a.passed ? "✓" : "✗"}</td>
+        <td><code>${escapeHtml(a.type)}</code></td>
+        <td style="color:${color}">${escapeHtml(a.message)}</td>
+      </tr>`;
+    })
+        .join("\n");
+}
+export function generateHtmlReportTool(runId, db) {
+    const run = db.getRunById(runId);
+    if (!run) {
+        throw new Error(`Run not found: ${runId}`);
+    }
+    const cases = db.getCaseResultsForRun(runId);
+    const duration = run.ended_at !== null ? `${run.ended_at - run.started_at}ms` : "—";
+    const overallColor = run.failed === 0 ? "#22c55e" : "#ef4444";
+    const caseRows = cases
+        .map((c) => {
+        const caseColor = c.status === "pass" ? "#22c55e" : c.status === "fail" ? "#ef4444" : "#f59e0b";
+        // Parse assertions_json — it is an array of AssertionResult[][]
+        // (one array of assertion results per step)
+        let stepsAssertions = [];
+        try {
+            if (c.assertions_json) {
+                stepsAssertions = JSON.parse(c.assertions_json);
+            }
+        }
+        catch {
+            // ignore parse errors
+        }
+        const assertionSections = stepsAssertions
+            .map((stepAssertions, idx) => {
+            if (stepAssertions.length === 0)
+                return "";
+            return `<details style="margin-top:8px">
+            <summary style="cursor:pointer;color:#aaa">Step ${idx + 1} assertions (${stepAssertions.length})</summary>
+            <table style="width:100%;margin-top:4px;border-collapse:collapse">
+              <thead><tr>
+                <th style="width:2em;text-align:left"></th>
+                <th style="text-align:left;color:#aaa">Type</th>
+                <th style="text-align:left;color:#aaa">Message</th>
+              </tr></thead>
+              <tbody>${assertionRows(stepAssertions)}</tbody>
+            </table>
+          </details>`;
+        })
+            .join("");
+        return `<tr>
+        <td style="padding:12px 8px;font-weight:bold;color:${caseColor}">${escapeHtml(c.case_name)}</td>
+        <td style="padding:12px 8px">${statusBadge(c.status)}</td>
+        <td style="padding:12px 8px;color:#888">${c.duration_ms !== null ? `${c.duration_ms}ms` : "—"}</td>
+        <td style="padding:12px 8px">
+          ${c.error_message ? `<span style="color:#ef4444">${escapeHtml(c.error_message)}</span>` : ""}
+          ${assertionSections}
+        </td>
+      </tr>`;
+    })
+        .join("\n");
+    return `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>MCP Eval Report — ${escapeHtml(run.suite_name)}</title>
+  <style>
+    *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
+    body {
+      font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
+      background: #0f1117;
+      color: #e2e8f0;
+      padding: 2rem;
+      line-height: 1.6;
+    }
+    h1 { font-size: 1.5rem; margin-bottom: .5rem; }
+    h2 { font-size: 1.1rem; margin: 1.5rem 0 .75rem; color: #94a3b8; }
+    .meta { color: #64748b; font-size: .85rem; margin-bottom: 1.5rem; }
+    .summary-grid {
+      display: grid;
+      grid-template-columns: repeat(auto-fill, minmax(140px, 1fr));
+      gap: 1rem;
+      margin-bottom: 2rem;
+    }
+    .stat-card {
+      background: #1e2433;
+      border-radius: 8px;
+      padding: 1rem;
+      text-align: center;
+    }
+    .stat-card .value {
+      font-size: 2rem;
+      font-weight: bold;
+      display: block;
+    }
+    .stat-card .label { font-size: .75rem; color: #64748b; }
+    table {
+      width: 100%;
+      border-collapse: collapse;
+      background: #1e2433;
+      border-radius: 8px;
+      overflow: hidden;
+    }
+    thead tr { background: #252d3d; }
+    th {
+      padding: 10px 8px;
+      text-align: left;
+      font-size: .8rem;
+      color: #94a3b8;
+      text-transform: uppercase;
+      letter-spacing: .05em;
+    }
+    tbody tr:hover { background: #252d3d; }
+    tbody tr + tr td { border-top: 1px solid #2d3748; }
+    td { vertical-align: top; }
+    code { background: #2d3748; padding: 1px 4px; border-radius: 3px; font-size: .85em; }
+    details summary { list-style: none; }
+    details summary::-webkit-details-marker { display: none; }
+    details[open] summary { margin-bottom: 4px; }
+  </style>
+</head>
+<body>
+  <h1>MCP Eval Runner — Report</h1>
+  <p class="meta">
+    Suite: <strong>${escapeHtml(run.suite_name)}</strong> &nbsp;|&nbsp;
+    Run ID: <code>${escapeHtml(run.id)}</code> &nbsp;|&nbsp;
+    Started: ${new Date(run.started_at).toISOString()} &nbsp;|&nbsp;
+    Duration: ${duration}
+  </p>
+  <div class="summary-grid">
+    <div class="stat-card">
+      <span class="value" style="color:#94a3b8">${run.total_cases}</span>
+      <span class="label">Total Cases</span>
+    </div>
+    <div class="stat-card">
+      <span class="value" style="color:#22c55e">${run.passed}</span>
+      <span class="label">Passed</span>
+    </div>
+    <div class="stat-card">
+      <span class="value" style="color:#ef4444">${run.failed}</span>
+      <span class="label">Failed</span>
+    </div>
+    <div class="stat-card">
+      <span class="value" style="color:${overallColor}">${run.failed === 0 ? "PASS" : "FAIL"}</span>
+      <span class="label">Overall</span>
+    </div>
+  </div>
+  <h2>Test Cases</h2>
+  <table>
+    <thead>
+      <tr>
+        <th>Case</th>
+        <th>Status</th>
+        <th>Duration</th>
+        <th>Details</th>
+      </tr>
+    </thead>
+    <tbody>
+      ${cases.length > 0 ? caseRows : `<tr><td colspan="4" style="padding:16px;text-align:center;color:#64748b">No cases found for this run.</td></tr>`}
+    </tbody>
+  </table>
+</body>
+</html>`;
+}

package/dist/tools/manage.d.ts ADDED Viewed

@@ -0,0 +1,11 @@
+/**
+ * list_cases and create_test_case MCP tool implementations.
+ */
+/**
+ * list_cases — enumerate available fixtures with their step counts.
+ */
+export declare function listCasesTool(fixturesDir: string): string;
+/**
+ * create_test_case — create a new YAML fixture file.
+ */
+export declare function createTestCaseTool(name: string, steps: unknown[], fixturesDir: string): string;

package/dist/tools/manage.js ADDED Viewed

@@ -0,0 +1,41 @@
+/**
+ * list_cases and create_test_case MCP tool implementations.
+ */
+import { loadFixturesFromDir, writeFixture, validateFixture } from "../fixture.js";
+/**
+ * list_cases — enumerate available fixtures with their step counts.
+ */
+export function listCasesTool(fixturesDir) {
+    const fixtures = loadFixturesFromDir(fixturesDir);
+    if (fixtures.length === 0) {
+        return [
+            "No fixtures found in: " + fixturesDir,
+            "",
+            "Use create_test_case to add your first test case.",
+        ].join("\n");
+    }
+    const lines = [`Found ${fixtures.length} fixture(s) in: ${fixturesDir}`, ""];
+    for (const f of fixtures) {
+        lines.push(`  • ${f.name} — ${f.steps.length} step(s)${f.description ? `  (${f.description})` : ""}`);
+        for (const step of f.steps) {
+            lines.push(`      - [${step.id}] tool: ${step.tool}${step.description ? `  — ${step.description}` : ""}`);
+        }
+    }
+    return lines.join("\n");
+}
+/**
+ * create_test_case — create a new YAML fixture file.
+ */
+export function createTestCaseTool(name, steps, fixturesDir) {
+    // Validate the input by constructing a fixture object and running it through validateFixture
+    const raw = { name, steps };
+    const fixture = validateFixture(raw, "create_test_case input");
+    const filePath = writeFixture(fixturesDir, fixture);
+    return [
+        `Created fixture: ${filePath}`,
+        `  Name: ${fixture.name}`,
+        `  Steps: ${fixture.steps.length}`,
+        "",
+        "Use run_case to execute it, or run_suite to run all fixtures.",
+    ].join("\n");
+}

package/dist/tools/report.d.ts ADDED Viewed

@@ -0,0 +1,12 @@
+/**
+ * regression_report and compare_results MCP tool implementations.
+ */
+import type { EvalDb } from "../db.js";
+/**
+ * regression_report — compare current state to the last run; returns what changed.
+ */
+export declare function regressionReportTool(fixturesDir: string, db: EvalDb): Promise<string>;
+/**
+ * compare_results — diff two named run results by run ID.
+ */
+export declare function compareResultsTool(runIdA: string, runIdB: string, db: EvalDb): Promise<string>;

package/dist/tools/report.js ADDED Viewed

@@ -0,0 +1,120 @@
+/**
+ * regression_report and compare_results MCP tool implementations.
+ */
+/**
+ * regression_report — compare current state to the last run; returns what changed.
+ */
+export async function regressionReportTool(fixturesDir, db) {
+    const runs = db.getAllRuns(2);
+    if (runs.length === 0) {
+        return ["No run history found.", "", "Run run_suite first to establish a baseline."].join("\n");
+    }
+    if (runs.length === 1) {
+        const [latest] = runs;
+        const cases = db.getCaseResultsForRun(latest.id);
+        const lines = [
+            `Only one run found (Run ID: ${latest.id}).`,
+            `Started: ${new Date(latest.started_at).toISOString()}`,
+            `Results: ${latest.passed} passed, ${latest.failed} failed`,
+            "",
+            "Run again after making changes to see regressions.",
+            "",
+            "Cases:",
+        ];
+        for (const c of cases) {
+            lines.push(`  ${c.status === "pass" ? "✓" : "✗"} ${c.case_name} — ${c.status}`);
+        }
+        return lines.join("\n");
+    }
+    const [latest, previous] = runs;
+    return diffRuns(previous, latest, db);
+}
+/**
+ * compare_results — diff two named run results by run ID.
+ */
+export async function compareResultsTool(runIdA, runIdB, db) {
+    const runA = db.getRunById(runIdA);
+    const runB = db.getRunById(runIdB);
+    if (!runA) {
+        throw new Error(`Run not found: ${runIdA}`);
+    }
+    if (!runB) {
+        throw new Error(`Run not found: ${runIdB}`);
+    }
+    return diffRuns(runA, runB, db);
+}
+function diffRuns(previous, latest, db) {
+    const prevCases = db.getCaseResultsForRun(previous.id);
+    const latestCases = db.getCaseResultsForRun(latest.id);
+    const prevMap = new Map(prevCases.map((c) => [c.case_name, c]));
+    const latestMap = new Map(latestCases.map((c) => [c.case_name, c]));
+    const regressions = [];
+    const fixes = [];
+    const newCases = [];
+    const removed = [];
+    const unchanged = [];
+    // Check for regressions and fixes
+    for (const [name, latestCase] of latestMap) {
+        const prevCase = prevMap.get(name);
+        if (!prevCase) {
+            newCases.push(`  + ${name} (new, ${latestCase.status})`);
+        }
+        else if (prevCase.status === "pass" && latestCase.status !== "pass") {
+            regressions.push(`  ✗ ${name}: ${prevCase.status} → ${latestCase.status}`);
+        }
+        else if (prevCase.status !== "pass" && latestCase.status === "pass") {
+            fixes.push(`  ✓ ${name}: ${prevCase.status} → ${latestCase.status}`);
+        }
+        else {
+            unchanged.push(`  = ${name}: ${latestCase.status}`);
+        }
+    }
+    // Check for removed cases
+    for (const [name] of prevMap) {
+        if (!latestMap.has(name)) {
+            removed.push(`  - ${name} (removed)`);
+        }
+    }
+    const lines = [
+        "=== Regression Report ===",
+        "",
+        `Previous run: ${previous.id}  (${new Date(previous.started_at).toISOString()})`,
+        `  ${previous.passed}/${previous.total_cases} passed`,
+        "",
+        `Latest run:   ${latest.id}  (${new Date(latest.started_at).toISOString()})`,
+        `  ${latest.passed}/${latest.total_cases} passed`,
+        "",
+    ];
+    if (regressions.length > 0) {
+        lines.push(`REGRESSIONS (${regressions.length}):`);
+        lines.push(...regressions);
+        lines.push("");
+    }
+    if (fixes.length > 0) {
+        lines.push(`FIXED (${fixes.length}):`);
+        lines.push(...fixes);
+        lines.push("");
+    }
+    if (newCases.length > 0) {
+        lines.push(`NEW CASES (${newCases.length}):`);
+        lines.push(...newCases);
+        lines.push("");
+    }
+    if (removed.length > 0) {
+        lines.push(`REMOVED (${removed.length}):`);
+        lines.push(...removed);
+        lines.push("");
+    }
+    if (unchanged.length > 0) {
+        lines.push(`UNCHANGED (${unchanged.length}):`);
+        lines.push(...unchanged);
+        lines.push("");
+    }
+    if (regressions.length === 0 &&
+        fixes.length === 0 &&
+        newCases.length === 0 &&
+        removed.length === 0) {
+        lines.push("No changes detected between runs.");
+    }
+    return lines.join("\n");
+}

package/dist/tools/run.d.ts ADDED Viewed

@@ -0,0 +1,20 @@
+/**
+ * run_suite and run_case MCP tool implementations.
+ */
+import type { EvalDb } from "../db.js";
+import type { RunnerOptions } from "../runner.js";
+import type { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+export interface RunToolOptions {
+    fixturesDir: string;
+    db: EvalDb;
+    runnerOptions: RunnerOptions;
+    server?: McpServer;
+}
+/**
+ * run_suite — execute all fixtures in the fixtures directory.
+ */
+export declare function runSuiteTool(opts: RunToolOptions): Promise<string>;
+/**
+ * run_case — run a single named test case.
+ */
+export declare function runCaseTool(name: string, opts: RunToolOptions): Promise<string>;

package/dist/tools/run.js ADDED Viewed

@@ -0,0 +1,166 @@
+/**
+ * run_suite and run_case MCP tool implementations.
+ */
+import { loadFixturesFromDir, loadFixture } from "../fixture.js";
+import { runSuite, runCase } from "../runner.js";
+import { formatResult } from "../reporter.js";
+import path from "path";
+import fs from "fs";
+import crypto from "crypto";
+/**
+ * Send a progress notification for the current suite run.
+ */
+async function sendProgress(server, suiteId, progress, total) {
+    if (!server)
+        return;
+    try {
+        await server.server.notification({
+            method: "notifications/progress",
+            params: {
+                progressToken: `eval-run-${suiteId}`,
+                progress,
+                total,
+            },
+        });
+    }
+    catch {
+        // Notifications are best-effort; ignore send errors
+    }
+}
+/**
+ * Send an MCP logging notification for an assertion result.
+ */
+async function sendAssertionLog(server, assertionType, caseId, passed) {
+    if (!server)
+        return;
+    try {
+        await server.server.notification({
+            method: "notifications/message",
+            params: {
+                level: "info",
+                logger: "eval-runner",
+                data: `Assertion ${assertionType} on case ${caseId}: ${passed ? "PASS" : "FAIL"}`,
+            },
+        });
+    }
+    catch {
+        // Notifications are best-effort; ignore send errors
+    }
+}
+/**
+ * run_suite — execute all fixtures in the fixtures directory.
+ */
+export async function runSuiteTool(opts) {
+    const fixtures = loadFixturesFromDir(opts.fixturesDir);
+    if (fixtures.length === 0) {
+        return [
+            "No fixtures found in: " + opts.fixturesDir,
+            "",
+            "NOTE: This runs in simulation mode (Phase 1).",
+            "Create fixtures with create_test_case or add YAML files to the fixtures directory.",
+        ].join("\n");
+    }
+    const total = fixtures.length;
+    const suiteId = crypto.randomUUID();
+    let completed = 0;
+    // Run fixtures one at a time, emitting progress + assertion logs per case
+    const caseResultsAcc = [];
+    for (const fixture of fixtures) {
+        const caseResult = await runCase(fixture, opts.runnerOptions);
+        caseResultsAcc.push(caseResult);
+        // Emit assertion-level logging notifications
+        for (const step of caseResult.steps) {
+            for (const assertion of step.assertions) {
+                await sendAssertionLog(opts.server, assertion.type, fixture.name, assertion.passed);
+            }
+        }
+        completed++;
+        await sendProgress(opts.server, suiteId, completed, total);
+    }
+    // Persist results through runSuite (which also recalculates pass/fail totals)
+    const result = await runSuite(fixtures, "default", opts.runnerOptions, opts.db);
+    const formatted = formatResult(result, opts.runnerOptions.format);
+    return formatted;
+}
+/**
+ * run_case — run a single named test case.
+ */
+export async function runCaseTool(name, opts) {
+    // Find the fixture file by name
+    const fixtures = loadFixturesFromDir(opts.fixturesDir);
+    const fixture = fixtures.find((f) => f.name === name);
+    if (!fixture) {
+        // Try to find by filename as well
+        const candidates = [
+            path.join(opts.fixturesDir, `${name}.yaml`),
+            path.join(opts.fixturesDir, `${name}.yml`),
+            path.join(opts.fixturesDir, `${name}.json`),
+        ];
+        for (const c of candidates) {
+            if (fs.existsSync(c)) {
+                const loaded = loadFixture(c);
+                const caseResult = await runCase(loaded, opts.runnerOptions);
+                // Emit assertion-level logging notifications
+                for (const step of caseResult.steps) {
+                    for (const assertion of step.assertions) {
+                        await sendAssertionLog(opts.server, assertion.type, loaded.name, assertion.passed);
+                    }
+                }
+                const suiteResult = {
+                    run_id: crypto.randomUUID(),
+                    suite_name: name,
+                    started_at: Date.now() - caseResult.duration_ms,
+                    ended_at: Date.now(),
+                    total_cases: 1,
+                    passed: caseResult.status === "pass" ? 1 : 0,
+                    failed: caseResult.status !== "pass" ? 1 : 0,
+                    cases: [caseResult],
+                };
+                return formatResult(suiteResult, opts.runnerOptions.format);
+            }
+        }
+        throw new Error(`No fixture named "${name}" found in ${opts.fixturesDir}`);
+    }
+    const caseResult = await runCase(fixture, opts.runnerOptions);
+    // Emit assertion-level logging notifications
+    for (const step of caseResult.steps) {
+        for (const assertion of step.assertions) {
+            await sendAssertionLog(opts.server, assertion.type, fixture.name, assertion.passed);
+        }
+    }
+    // Persist to DB — run record must exist before case_result (foreign key)
+    const runId = crypto.randomUUID();
+    const now = Date.now();
+    opts.db.insertRun({
+        id: runId,
+        suite_name: name,
+        started_at: now - caseResult.duration_ms,
+        ended_at: now,
+        total_cases: 1,
+        passed: caseResult.status === "pass" ? 1 : 0,
+        failed: caseResult.status !== "pass" ? 1 : 0,
+        format: opts.runnerOptions.format ?? "console",
+    });
+    opts.db.insertCaseResult({
+        id: crypto.randomUUID(),
+        run_id: runId,
+        case_name: caseResult.case_name,
+        status: caseResult.status,
+        duration_ms: caseResult.duration_ms,
+        error_message: caseResult.error ?? null,
+        assertions_json: JSON.stringify(caseResult.steps.map((s) => s.assertions)),
+        created_at: now,
+    });
+    const suiteResult = {
+        run_id: runId,
+        suite_name: name,
+        started_at: now - caseResult.duration_ms,
+        ended_at: now,
+        total_cases: 1,
+        passed: caseResult.status === "pass" ? 1 : 0,
+        failed: caseResult.status !== "pass" ? 1 : 0,
+        cases: [caseResult],
+    };
+    const formatted = formatResult(suiteResult, opts.runnerOptions.format);
+    return formatted;
+}

package/dist/tools/scaffold.d.ts ADDED Viewed

@@ -0,0 +1,11 @@
+/**
+ * scaffold_fixture tool implementation.
+ *
+ * Generates a boilerplate YAML fixture file given a name and a list of tool names.
+ * Each tool name becomes a documented step with placeholder input and assertions.
+ */
+/**
+ * Scaffold a YAML fixture file and write it to the fixtures directory.
+ * Returns the path to the created file.
+ */
+export declare function scaffoldFixtureTool(name: string, toolNames: string[], fixturesDir: string): string;

package/dist/tools/scaffold.js ADDED Viewed

@@ -0,0 +1,90 @@
+/**
+ * scaffold_fixture tool implementation.
+ *
+ * Generates a boilerplate YAML fixture file given a name and a list of tool names.
+ * Each tool name becomes a documented step with placeholder input and assertions.
+ */
+import fs from "fs";
+import path from "path";
+/**
+ * Build a YAML fixture template string from a name and list of tool names.
+ */
+function buildFixtureYaml(name, toolNames) {
+    const safeId = (s) => s.replace(/[^a-z0-9_]/gi, "_");
+    const steps = toolNames.map((tool, idx) => {
+        const prevId = idx > 0 ? `step_${idx}_${safeId(toolNames[idx - 1])}` : null;
+        const id = `step_${idx + 1}_${safeId(tool)}`;
+        const pipingComment = prevId
+            ? `      # Pipe previous step output: "{{steps.${prevId}.output}}"\n`
+            : "";
+        return [
+            `  - id: ${id}`,
+            `    description: "Call ${tool} and verify the response"`,
+            `    tool: ${tool}`,
+            `    input:`,
+            `      # TODO: fill in tool-specific input parameters`,
+            `      example_param: "example_value"`,
+            pipingComment.trimEnd(),
+            `    expected_output: "TODO: paste or describe expected output here"`,
+            `    expect:`,
+            `      output_contains: "TODO: expected substring"`,
+            `      # output_not_contains: "error"`,
+            `      # output_matches: "^result:"`,
+            `      tool_called: ${tool}`,
+            `      latency_under: 5000`,
+            `      # schema_match:`,
+            `      #   type: object`,
+            `      #   required: [result]`,
+            `      #   properties:`,
+            `      #     result:`,
+            `      #       type: string`,
+        ]
+            .filter((l) => l !== "")
+            .join("\n");
+    });
+    return [
+        `# Fixture: ${name}`,
+        `# Generated by mcp-eval-runner scaffold_fixture`,
+        `# Edit the TODO fields before running.`,
+        `#`,
+        `# Live execution: add a 'server' block to run against a real MCP server:`,
+        `# server:`,
+        `#   command: node`,
+        `#   args: ["/path/to/your/server.js"]`,
+        `#   env:`,
+        `#     MY_ENV_VAR: "value"`,
+        ``,
+        `name: ${name}`,
+        `description: "Auto-scaffolded fixture for tools: ${toolNames.join(", ")}"`,
+        ``,
+        `# server:`,
+        `#   command: node`,
+        `#   args: ["/path/to/server.js"]`,
+        ``,
+        `steps:`,
+        steps.join("\n\n"),
+        ``,
+    ].join("\n");
+}
+/**
+ * Scaffold a YAML fixture file and write it to the fixtures directory.
+ * Returns the path to the created file.
+ */
+export function scaffoldFixtureTool(name, toolNames, fixturesDir) {
+    if (!name || name.trim() === "") {
+        throw new Error('"name" must be a non-empty string');
+    }
+    if (!toolNames || toolNames.length === 0) {
+        throw new Error('"tool_names" must be a non-empty array');
+    }
+    fs.mkdirSync(fixturesDir, { recursive: true });
+    const safeName = name.replace(/[^a-z0-9_-]/gi, "_");
+    const fileName = `${safeName}.yaml`;
+    const filePath = path.join(fixturesDir, fileName);
+    if (fs.existsSync(filePath)) {
+        throw new Error(`Fixture already exists: ${filePath}`);
+    }
+    const content = buildFixtureYaml(name, toolNames);
+    fs.writeFileSync(filePath, content, "utf-8");
+    return filePath;
+}