@speakeasy-api/docs-mcp-eval 0.1.0

package/dist/runner.js

@@ -0,0 +1,311 @@
+import { Client } from "@modelcontextprotocol/sdk/client/index.js";
+import { StdioClientTransport } from "@modelcontextprotocol/sdk/client/stdio.js";
+import { execFile, spawn } from "node:child_process";
+import { performance } from "node:perf_hooks";
+import { promisify } from "node:util";
+import { summarizeCases } from "./metrics.js";
+const execFileAsync = promisify(execFile);
+export function runEvaluation(input) {
+    const summary = summarizeCases(input.cases, input.timings);
+    return {
+        summary,
+        metadata: {
+            deterministic: input.deterministic ?? true,
+            provider: input.model?.provider ?? null,
+            model: input.model?.model ?? null
+        }
+    };
+}
+export async function runEvaluationAgainstServer(input) {
+    const buildTimeMs = input.build ? await runBuildStep(input.build) : 0;
+    const serverParams = {
+        command: input.server.command
+    };
+    if (input.server.args !== undefined) {
+        serverParams.args = input.server.args;
+    }
+    if (input.server.cwd !== undefined) {
+        serverParams.cwd = input.server.cwd;
+    }
+    if (input.server.env !== undefined) {
+        serverParams.env = input.server.env;
+    }
+    const transport = new StdioClientTransport(serverParams);
+    const client = new Client({
+        name: "@speakeasy-api/docs-mcp-eval",
+        version: "0.1.0"
+    }, {
+        capabilities: {}
+    });
+    await client.connect(transport);
+    const rssSampler = createRssSampler(transport.pid);
+    await rssSampler.start();
+    try {
+        await client.listTools();
+        await warmupServer(client, input.cases, input.warmupQueries ?? 0);
+        const rankedCases = [];
+        const searchLatenciesMs = [];
+        const getDocLatenciesMs = [];
+        for (const testCase of input.cases) {
+            const executed = await executeCase(client, testCase);
+            rankedCases.push(executed.rankedCase);
+            searchLatenciesMs.push(...executed.searchLatenciesMs);
+            getDocLatenciesMs.push(...executed.getDocLatenciesMs);
+        }
+        const peakRssMb = await rssSampler.stop();
+        const output = runEvaluation({
+            cases: rankedCases,
+            timings: {
+                searchLatenciesMs,
+                getDocLatenciesMs,
+                buildTimeMs,
+                peakRssMb
+            },
+            ...(input.model ? { model: input.model } : {}),
+            ...(input.deterministic !== undefined ? { deterministic: input.deterministic } : {})
+        });
+        return {
+            ...output,
+            rankedCases,
+            stats: {
+                searchLatenciesMs,
+                getDocLatenciesMs,
+                buildTimeMs,
+                peakRssMb
+            }
+        };
+    }
+    finally {
+        await rssSampler.stop();
+        await transport.close();
+    }
+}
+async function executeCase(client, testCase) {
+    const rankedChunkIds = [];
+    const seen = new Set();
+    const searchLatenciesMs = [];
+    const getDocLatenciesMs = [];
+    let rounds = 0;
+    let cursor;
+    const maxRounds = testCase.maxRounds ?? 3;
+    while (rounds < maxRounds) {
+        rounds += 1;
+        const args = {
+            query: testCase.query,
+            limit: testCase.limit ?? 5,
+            ...(testCase.filters ?? {})
+        };
+        if (cursor) {
+            args.cursor = cursor;
+        }
+        const searchStart = performance.now();
+        const toolResult = await client.callTool({
+            name: "search_docs",
+            arguments: args
+        });
+        searchLatenciesMs.push(performance.now() - searchStart);
+        if ("toolResult" in toolResult) {
+            throw new Error("Unexpected compatibility tool result shape from server");
+        }
+        if (toolResult.isError) {
+            throw new Error(readTextContent(toolResult.content) || "search_docs returned an unknown error");
+        }
+        const payload = parseSearchResultText(readTextContent(toolResult.content));
+        for (const hit of payload.hits) {
+            if (!seen.has(hit.chunk_id)) {
+                rankedChunkIds.push(hit.chunk_id);
+                seen.add(hit.chunk_id);
+            }
+        }
+        const targetHit = payload.hits.find((hit) => hit.chunk_id === testCase.expectedChunkId);
+        if (targetHit) {
+            const getDocStart = performance.now();
+            const getDocResult = await client.callTool({
+                name: "get_doc",
+                arguments: {
+                    chunk_id: targetHit.chunk_id,
+                    context: 0
+                }
+            });
+            getDocLatenciesMs.push(performance.now() - getDocStart);
+            if ("toolResult" in getDocResult) {
+                throw new Error("Unexpected compatibility tool result shape from server");
+            }
+            if (getDocResult.isError) {
+                throw new Error(readTextContent(getDocResult.content) || "get_doc returned an unknown error");
+            }
+            break;
+        }
+        cursor = payload.next_cursor ?? undefined;
+        if (!cursor) {
+            break;
+        }
+    }
+    const roundsToRightDoc = computeRoundsToRightDoc({
+        found: rankedChunkIds.includes(testCase.expectedChunkId),
+        roundsExecuted: rounds,
+        maxRounds
+    });
+    return {
+        rankedCase: {
+            expectedChunkId: testCase.expectedChunkId,
+            rankedChunkIds,
+            roundsToRightDoc
+        },
+        searchLatenciesMs,
+        getDocLatenciesMs
+    };
+}
+export function computeRoundsToRightDoc(input) {
+    return input.found ? input.roundsExecuted : input.maxRounds + 1;
+}
+async function warmupServer(client, cases, warmupQueries) {
+    if (warmupQueries <= 0 || cases.length === 0) {
+        return;
+    }
+    for (let i = 0; i < warmupQueries; i += 1) {
+        const testCase = cases[i % cases.length];
+        if (!testCase) {
+            continue;
+        }
+        const args = {
+            query: testCase.query,
+            limit: Math.min(5, testCase.limit ?? 5),
+            ...(testCase.filters ?? {})
+        };
+        try {
+            await client.callTool({
+                name: "search_docs",
+                arguments: args
+            });
+        }
+        catch {
+            // Warmup is best-effort.
+        }
+    }
+}
+function readTextContent(content) {
+    return content
+        .filter((entry) => entry.type === "text")
+        .map((entry) => entry.text ?? "")
+        .join("\n")
+        .trim();
+}
+function parseSearchResultText(text) {
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch {
+        throw new Error("search_docs result was not valid JSON text");
+    }
+    if (!parsed || typeof parsed !== "object") {
+        throw new Error("search_docs result was not an object");
+    }
+    const result = parsed;
+    if (!Array.isArray(result.hits)) {
+        throw new Error("search_docs result is missing hits[]");
+    }
+    const hits = [];
+    for (const entry of result.hits) {
+        if (!entry || typeof entry !== "object") {
+            continue;
+        }
+        const chunkId = entry.chunk_id;
+        if (typeof chunkId === "string" && chunkId) {
+            hits.push({ chunk_id: chunkId });
+        }
+    }
+    const nextCursorRaw = result.next_cursor;
+    const nextCursor = nextCursorRaw === null || typeof nextCursorRaw === "string"
+        ? nextCursorRaw
+        : null;
+    return {
+        hits,
+        next_cursor: nextCursor
+    };
+}
+async function runBuildStep(config) {
+    const startedAt = performance.now();
+    await new Promise((resolve, reject) => {
+        const child = spawn(config.command, config.args ?? [], {
+            cwd: config.cwd,
+            env: {
+                ...process.env,
+                ...config.env
+            },
+            stdio: "inherit",
+            shell: false
+        });
+        child.on("error", reject);
+        child.on("exit", (code, signal) => {
+            if (code === 0) {
+                resolve();
+                return;
+            }
+            reject(new Error(`Build command failed with ${signal ? `signal ${signal}` : `exit code ${String(code)}`}`));
+        });
+    });
+    return performance.now() - startedAt;
+}
+function createRssSampler(pid) {
+    if (!pid) {
+        return {
+            async start() {
+                // noop
+            },
+            async stop() {
+                return 0;
+            }
+        };
+    }
+    let interval;
+    let peakRssMb = 0;
+    let stopped = false;
+    const sample = async () => {
+        try {
+            const rssMb = await readProcessRssMb(pid);
+            if (rssMb > peakRssMb) {
+                peakRssMb = rssMb;
+            }
+        }
+        catch {
+            // RSS sampling is best-effort.
+        }
+    };
+    return {
+        async start() {
+            if (stopped || interval) {
+                return;
+            }
+            await sample();
+            interval = setInterval(() => {
+                void sample();
+            }, 200);
+            interval.unref();
+        },
+        async stop() {
+            if (stopped) {
+                return peakRssMb;
+            }
+            stopped = true;
+            if (interval) {
+                clearInterval(interval);
+                interval = undefined;
+            }
+            await sample();
+            return Number(peakRssMb.toFixed(6));
+        }
+    };
+}
+async function readProcessRssMb(pid) {
+    const { stdout } = await execFileAsync("ps", ["-o", "rss=", "-p", String(pid)], {
+        windowsHide: true
+    });
+    const kb = Number.parseFloat(stdout.trim());
+    if (!Number.isFinite(kb) || kb <= 0) {
+        return 0;
+    }
+    return kb / 1024;
+}
+//# sourceMappingURL=runner.js.map

package/dist/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../src/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,2CAA2C,CAAC;AACnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,2CAA2C,CAAC;AACjF,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,MAAM,aAAa,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AAsE1C,MAAM,UAAU,aAAa,CAAC,KAAmB;IAC/C,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAC3D,OAAO;QACL,OAAO;QACP,QAAQ,EAAE;YACR,aAAa,EAAE,KAAK,CAAC,aAAa,IAAI,IAAI;YAC1C,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,IAAI,IAAI;YACvC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,IAAI,IAAI;SAClC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,0BAA0B,CAC9C,KAAuB;IAEvB,MAAM,WAAW,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtE,MAAM,YAAY,GAKd;QACF,OAAO,EAAE,KAAK,CAAC,MAAM,CAAC,OAAO;KAC9B,CAAC;IAEF,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACpC,YAAY,CAAC,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC;IACxC,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;QACnC,YAAY,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC;IACtC,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;QACnC,YAAY,CAAC,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC;IACtC,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,oBAAoB,CAAC,YAAY,CAAC,CAAC;IAEzD,MAAM,MAAM,GAAG,IAAI,MAAM,CACvB;QACE,IAAI,EAAE,8BAA8B;QACpC,OAAO,EAAE,OAAO;KACjB,EACD;QACE,YAAY,EAAE,EAAE;KACjB,CACF,CAAC;IAEF,MAAM,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAChC,MAAM,UAAU,GAAG,gBAAgB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACnD,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;IAEzB,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;QACzB,MAAM,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC,CAAC;QAElE,MAAM,WAAW,GAAiB,EAAE,CAAC;QACrC,MAAM,iBAAiB,GAAa,EAAE,CAAC;QACvC,MAAM,iBAAiB,GAAa,EAAE,CAAC;QAEvC,KAAK,MAAM,QAAQ,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;YACrD,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACtC,iBAAiB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;YACtD,iBAAiB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QACxD,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QAE1C,MAAM,MAAM,GAAG,aAAa,CAAC;YAC3B,KAAK,EAAE,WAAW;YAClB,OAAO,EAAE;gBACP,iBAAiB;gBACjB,iBAAiB;gBACjB,WAAW;gBACX,SAAS;aACV;YACD,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9C,GAAG,CAAC,KAAK,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACrF,CAAC,CAAC;QAEH,OAAO;YACL,GAAG,MAAM;YACT,WAAW;YACX,KAAK,EAAE;gBACL,iBAAiB;gBACjB,iBAAiB;gBACjB,WAAW;gBACX,SAAS;aACV;SACF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,SAAS,CAAC,KAAK,EAAE,CAAC;IAC1B,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,MAAc,EACd,QAAuB;IAMvB,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,iBAAiB,GAAa,EAAE,CAAC;IACvC,MAAM,iBAAiB,GAAa,EAAE,CAAC;IAEvC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,MAA0B,CAAC;IAC/B,MAAM,SAAS,GAAG,QAAQ,CAAC,SAAS,IAAI,CAAC,CAAC;IAE1C,OAAO,MAAM,GAAG,SAAS,EAAE,CAAC;QAC1B,MAAM,IAAI,CAAC,CAAC;QAEZ,MAAM,IAAI,GAA4B;YACpC,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,KAAK,EAAE,QAAQ,CAAC,KAAK,IAAI,CAAC;YAC1B,GAAG,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;SAC5B,CAAC;QAEF,IAAI,MAAM,EAAE,CAAC;YACX,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACvB,CAAC;QAED,MAAM,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACtC,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;YACvC,IAAI,EAAE,aAAa;YACnB,SAAS,EAAE,IAAI;SAChB,CAAC,CAAC;QACH,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC,CAAC;QAExD,IAAI,YAAY,IAAI,UAAU,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;QAC5E,CAAC;QAED,IAAI,UAAU,CAAC,OAAO,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,uCAAuC,CAAC,CAAC;QAClG,CAAC;QAED,MAAM,OAAO,GAAG,qBAAqB,CAAC,eAAe,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3E,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC5B,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAClC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;QAED,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,eAAe,CAAC,CAAC;QACxF,IAAI,SAAS,EAAE,CAAC;YACd,MAAM,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YACtC,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC;gBACzC,IAAI,EAAE,SAAS;gBACf,SAAS,EAAE;oBACT,QAAQ,EAAE,SAAS,CAAC,QAAQ;oBAC5B,OAAO,EAAE,CAAC;iBACX;aACF,CAAC,CAAC;YACH,iBAAiB,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,WAAW,CAAC,CAAC;YAExD,IAAI,YAAY,IAAI,YAAY,EAAE,CAAC;gBACjC,MAAM,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC;YAC5E,CAAC;YACD,IAAI,YAAY,CAAC,OAAO,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,eAAe,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,mCAAmC,CAAC,CAAC;YAChG,CAAC;YAED,MAAM;QACR,CAAC;QAED,MAAM,GAAG,OAAO,CAAC,WAAW,IAAI,SAAS,CAAC;QAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,MAAM;QACR,CAAC;IACH,CAAC;IAED,MAAM,gBAAgB,GAAG,uBAAuB,CAAC;QAC/C,KAAK,EAAE,cAAc,CAAC,QAAQ,CAAC,QAAQ,CAAC,eAAe,CAAC;QACxD,cAAc,EAAE,MAAM;QACtB,SAAS;KACV,CAAC,CAAC;IAEH,OAAO;QACL,UAAU,EAAE;YACV,eAAe,EAAE,QAAQ,CAAC,eAAe;YACzC,cAAc;YACd,gBAAgB;SACjB;QACD,iBAAiB;QACjB,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,KAIvC;IACC,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,GAAG,CAAC,CAAC;AAClE,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,MAAc,EACd,KAAsB,EACtB,aAAqB;IAErB,IAAI,aAAa,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7C,OAAO;IACT,CAAC;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QAC1C,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAA4B;YACpC,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,IAAI,CAAC,CAAC;YACvC,GAAG,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;SAC5B,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,QAAQ,CAAC;gBACpB,IAAI,EAAE,aAAa;gBACnB,SAAS,EAAE,IAAI;aAChB,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,OAA+C;IACtE,OAAO,OAAO;SACX,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC;SACxC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC;SAChC,IAAI,CAAC,IAAI,CAAC;SACV,IAAI,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY;IAIzC,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,MAAM,GAAG,MAAiC,CAAC;IACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,IAAI,GAAgC,EAAE,CAAC;IAC7C,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YACxC,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAI,KAAiC,CAAC,QAAQ,CAAC;QAC5D,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,EAAE,CAAC;YAC3C,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,MAAM,CAAC,WAAW,CAAC;IACzC,MAAM,UAAU,GAAG,aAAa,KAAK,IAAI,IAAI,OAAO,aAAa,KAAK,QAAQ;QAC5E,CAAC,CAAC,aAAa;QACf,CAAC,CAAC,IAAI,CAAC;IAET,OAAO;QACL,IAAI;QACJ,WAAW,EAAE,UAAU;KACxB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,MAAuB;IACjD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;IAEpC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC1C,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,EAAE;YACrD,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,GAAG,EAAE;gBACH,GAAG,OAAO,CAAC,GAAG;gBACd,GAAG,MAAM,CAAC,GAAG;aACd;YACD,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;QAEH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QAC1B,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;YAChC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,OAAO,EAAE,CAAC;gBACV,OAAO;YACT,CAAC;YACD,MAAM,CACJ,IAAI,KAAK,CACP,6BAA6B,MAAM,CAAC,CAAC,CAAC,UAAU,MAAM,EAAE,CAAC,CAAC,CAAC,aAAa,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,CACzF,CACF,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;AACvC,CAAC;AAED,SAAS,gBAAgB,CAAC,GAAkB;IAI1C,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO;YACL,KAAK,CAAC,KAAK;gBACT,OAAO;YACT,CAAC;YACD,KAAK,CAAC,IAAI;gBACR,OAAO,CAAC,CAAC;YACX,CAAC;SACF,CAAC;IACJ,CAAC;IAED,IAAI,QAAoD,CAAC;IACzD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,OAAO,GAAG,KAAK,CAAC;IAEpB,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACvC,IAAI,CAAC;YACH,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,GAAG,CAAC,CAAC;YAC1C,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;gBACtB,SAAS,GAAG,KAAK,CAAC;YACpB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;QACjC,CAAC;IACH,CAAC,CAAC;IAEF,OAAO;QACL,KAAK,CAAC,KAAK;YACT,IAAI,OAAO,IAAI,QAAQ,EAAE,CAAC;gBACxB,OAAO;YACT,CAAC;YACD,MAAM,MAAM,EAAE,CAAC;YACf,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;gBAC1B,KAAK,MAAM,EAAE,CAAC;YAChB,CAAC,EAAE,GAAG,CAAC,CAAC;YACR,QAAQ,CAAC,KAAK,EAAE,CAAC;QACnB,CAAC;QACD,KAAK,CAAC,IAAI;YACR,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,SAAS,CAAC;YACnB,CAAC;YAED,OAAO,GAAG,IAAI,CAAC;YACf,IAAI,QAAQ,EAAE,CAAC;gBACb,aAAa,CAAC,QAAQ,CAAC,CAAC;gBACxB,QAAQ,GAAG,SAAS,CAAC;YACvB,CAAC;YACD,MAAM,MAAM,EAAE,CAAC;YACf,OAAO,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAW;IACzC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE;QAC9E,WAAW,EAAE,IAAI;KAClB,CAAC,CAAC;IACH,MAAM,EAAE,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IAC5C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;QACpC,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,EAAE,GAAG,IAAI,CAAC;AACnB,CAAC"}

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "@speakeasy-api/docs-mcp-eval",
+  "version": "0.1.0",
+  "description": "Evaluation and benchmarking harness for docs-mcp search quality metrics",
+  "license": "AGPL-3.0-only",
+  "type": "module",
+  "author": "Speakeasy <info@speakeasy.com> (https://speakeasy.com)",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/speakeasy-api/docs-mcp.git",
+    "directory": "packages/eval"
+  },
+  "homepage": "https://github.com/speakeasy-api/docs-mcp",
+  "bugs": {
+    "url": "https://github.com/speakeasy-api/docs-mcp/issues"
+  },
+  "files": [
+    "dist"
+  ],
+  "bin": {
+    "docs-mcp-eval": "dist/bin.js"
+  },
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "engines": {
+    "node": ">=22.0.0"
+  },
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.26.0",
+    "commander": "^13.1.0"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "vitest run",
+    "lint": "eslint src test"
+  }
+}