nodebench-mcp 3.0.0 → 3.0.2

package/dist/__tests__/gaiaCapabilityFilesEval.test.js

@@ -1,914 +0,0 @@
-/**
- * GAIA file-backed capability/accuracy benchmark: LLM-only vs LLM+NodeBench MCP local file tools.
- *
- * This lane targets GAIA tasks that include attachments (PDF / XLSX / CSV).
- * We provide deterministic local parsing via NodeBench MCP tools and score answers against
- * the ground-truth "Final answer" (stored locally under `.cache/gaia`, gitignored).
- *
- * Safety:
- * - GAIA is gated. Do not commit fixtures that contain prompts/answers.
- * - This test logs only task IDs and aggregate metrics (no prompt/answer text).
- *
- * Disabled by default (cost + rate limits). Run with:
- *   NODEBENCH_RUN_GAIA_CAPABILITY=1 npm --prefix packages/mcp-local run test
- */
-import { describe, expect, it } from "vitest";
-import { existsSync, readFileSync } from "node:fs";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-import { performance } from "node:perf_hooks";
-import { localFileTools } from "../tools/localFileTools.js";
-import { createTextLlmClient, generateTextFromHistory, } from "./helpers/textLlm.js";
-import { answersMatchWithJudge, autoDiscoverJudge } from "./helpers/answerMatch.js";
-const shouldRun = process.env.NODEBENCH_RUN_GAIA_CAPABILITY === "1";
-const shouldWriteReport = process.env.NODEBENCH_WRITE_GAIA_REPORT === "1";
-async function safeWriteJson(filePath, payload) {
-    try {
-        await mkdir(path.dirname(filePath), { recursive: true });
-        await writeFile(filePath, JSON.stringify(payload, null, 2) + "\n", "utf8");
-    }
-    catch (err) {
-        console.warn(`[gaia-capability-files] report write failed: ${err?.message ?? String(err)}`);
-    }
-}
-function resolveRepoRoot() {
-    const testDir = path.dirname(fileURLToPath(import.meta.url));
-    return path.resolve(testDir, "../../../..");
-}
-function resolveCapabilityFilesFixturePath() {
-    const override = process.env.NODEBENCH_GAIA_CAPABILITY_FILES_FIXTURE_PATH;
-    if (override) {
-        if (path.isAbsolute(override))
-            return override;
-        const repoRoot = resolveRepoRoot();
-        return path.resolve(repoRoot, override);
-    }
-    const config = process.env.NODEBENCH_GAIA_CAPABILITY_CONFIG ?? "2023_all";
-    const split = process.env.NODEBENCH_GAIA_CAPABILITY_SPLIT ?? "validation";
-    const repoRoot = resolveRepoRoot();
-    return path.join(repoRoot, ".cache", "gaia", `gaia_capability_files_${config}_${split}.sample.json`);
-}
-function loadDotEnvLocalIfPresent() {
-    const repoRoot = resolveRepoRoot();
-    const envPath = path.join(repoRoot, ".env.local");
-    if (!existsSync(envPath))
-        return;
-    const text = readFileSync(envPath, "utf8");
-    for (const rawLine of text.split(/\r?\n/)) {
-        const line = rawLine.trim();
-        if (!line || line.startsWith("#"))
-            continue;
-        const idx = line.indexOf("=");
-        if (idx <= 0)
-            continue;
-        const key = line.slice(0, idx).trim();
-        let value = line.slice(idx + 1).trim();
-        if ((value.startsWith("\"") && value.endsWith("\"")) ||
-            (value.startsWith("'") && value.endsWith("'"))) {
-            value = value.slice(1, -1);
-        }
-        if (!process.env[key])
-            process.env[key] = value;
-    }
-}
-function toIntegerOrNullLoose(value) {
-    if (value === null || value === undefined)
-        return null;
-    if (typeof value === "number" && Number.isFinite(value))
-        return Math.trunc(value);
-    const s = String(value).trim();
-    if (!s)
-        return null;
-    const m = s.match(/-?\d+/);
-    if (!m)
-        return null;
-    const n = Number.parseInt(m[0], 10);
-    return Number.isFinite(n) ? n : null;
-}
-function deriveAddressParityIfRelevant(taskPrompt, extract) {
-    const wantsParity = /\bodd\b|\beven\b|odd-?numbered|even-?numbered|parity/i.test(String(taskPrompt ?? ""));
-    if (!wantsParity)
-        return null;
-    const headers = Array.isArray(extract?.headers) ? extract.headers.map((h) => String(h ?? "")) : [];
-    const rows = Array.isArray(extract?.rows) ? extract.rows : [];
-    if (headers.length === 0 || rows.length === 0)
-        return null;
-    const addrIdx = headers.findIndex((h) => /address/i.test(h));
-    if (addrIdx < 0)
-        return null;
-    let integerCount = 0;
-    let evenCount = 0;
-    let oddCount = 0;
-    for (const r of rows) {
-        const n = toIntegerOrNullLoose(Array.isArray(r) ? r[addrIdx] : null);
-        if (n === null)
-            continue;
-        integerCount++;
-        if (Math.abs(n) % 2 === 0)
-            evenCount++;
-        else
-            oddCount++;
-    }
-    return {
-        column: headers[addrIdx],
-        columnIndex: addrIdx,
-        integerCount,
-        evenCount,
-        oddCount,
-    };
-}
-function inferAnswerFromAddressParityIfPossible(taskPrompt, parity) {
-    const p = String(taskPrompt ?? "").toLowerCase();
-    if (!p)
-        return null;
-    const oddEast = /odd[^.]*east/.test(p);
-    const oddWest = /odd[^.]*west/.test(p);
-    const evenEast = /even[^.]*east/.test(p);
-    const evenWest = /even[^.]*west/.test(p);
-    const lastSunrise = p.lastIndexOf("sunrise");
-    const lastSunset = p.lastIndexOf("sunset");
-    if (lastSunrise === -1 && lastSunset === -1)
-        return null;
-    // If both appear, assume the one mentioned last is what the question asks for.
-    const wantsSunrise = lastSunrise > lastSunset;
-    const wantsSunset = lastSunset > lastSunrise;
-    let desiredDirection = wantsSunrise ? "east" : wantsSunset ? "west" : null;
-    if (!desiredDirection)
-        return null;
-    // Some tasks specify the awning is for the *back* of the house, while the
-    // prompt gives the facing direction for the street address (front). In that
-    // case, invert the facing direction.
-    const mentionsBackOfHouse = /\bback\b/.test(p) && /\bhouse\b/.test(p);
-    if (mentionsBackOfHouse) {
-        desiredDirection = desiredDirection === "east" ? "west" : "east";
-    }
-    // Map parity -> facing direction when explicitly stated.
-    const oddFaces = oddEast ? "east" : oddWest ? "west" : null;
-    const evenFaces = evenEast ? "east" : evenWest ? "west" : null;
-    if (!oddFaces || !evenFaces)
-        return null;
-    if (evenFaces === desiredDirection)
-        return String(parity.evenCount);
-    if (oddFaces === desiredDirection)
-        return String(parity.oddCount);
-    return null;
-}
-async function llmGenerateText(llm, history) {
-    const temperature = Number.parseFloat(process.env.NODEBENCH_GAIA_CAPABILITY_TEMPERATURE ?? "0");
-    return generateTextFromHistory(llm, history, {
-        temperature: Number.isFinite(temperature) ? temperature : 0,
-        maxOutputTokens: 1024,
-    });
-}
-async function baselineAnswer(llm, task) {
-    const contents = [
-        {
-            role: "user",
-            parts: [
-                {
-                    text: `Answer the question using your existing knowledge only. Do not browse the web.\n\nReturn ONLY the final answer, no explanation.\n\nQuestion:\n${task.prompt}`,
-                },
-            ],
-        },
-    ];
-    return llmGenerateText(llm, contents);
-}
-function buildToolIndex() {
-    const byName = new Map();
-    for (const tool of localFileTools)
-        byName.set(tool.name, tool);
-    return byName;
-}
-function extractJsonObject(text) {
-    const trimmed = text.trim();
-    const fenceMatch = trimmed.match(/```json\s*([\s\S]*?)\s*```/i);
-    const candidate = fenceMatch ? fenceMatch[1] : trimmed;
-    const start = candidate.indexOf("{");
-    const end = candidate.lastIndexOf("}");
-    if (start === -1 || end === -1 || end <= start)
-        return null;
-    const slice = candidate.slice(start, end + 1);
-    try {
-        return JSON.parse(slice);
-    }
-    catch {
-        return null;
-    }
-}
-function resolveTaskLocalFilePath(task) {
-    const repoRoot = resolveRepoRoot();
-    const rel = String(task.localFilePath ?? "").trim();
-    if (rel)
-        return path.resolve(repoRoot, rel);
-    // Fallback to the standard cache layout used by the fixture generator.
-    const filePath = String(task.filePath ?? "").trim();
-    if (!filePath)
-        throw new Error("Task missing filePath/localFilePath");
-    return path.join(repoRoot, ".cache", "gaia", "data", filePath);
-}
-async function toolAugmentedAnswerFromFile(llm, task, opts) {
-    const toolIndex = buildToolIndex();
-    const toolsMode = (process.env.NODEBENCH_GAIA_CAPABILITY_TOOLS_MODE ?? "rag").toLowerCase();
-    const localPath = resolveTaskLocalFilePath(task);
-    if (!existsSync(localPath)) {
-        throw new Error(`Missing attachment on disk. Expected at ${localPath}. Refresh with dataset:gaia:capability:files:refresh`);
-    }
-    const ext = String(task.fileExt ?? "").trim().toLowerCase() ||
-        path.extname(task.fileName || task.filePath || "").toLowerCase().replace(/^\./, "");
-    // "rag" mode: single deterministic file extract -> answer (more stable than agent loops).
-    if (toolsMode === "rag") {
-        let extract;
-        if (ext === "csv") {
-            const tool = toolIndex.get("read_csv_file");
-            if (!tool)
-                throw new Error("Missing tool: read_csv_file");
-            extract = await tool.handler({
-                path: localPath,
-                hasHeader: true,
-                maxRows: 500,
-                maxCols: 80,
-                maxCellChars: 2000,
-            });
-        }
-        else if (ext === "xlsx") {
-            const tool = toolIndex.get("read_xlsx_file");
-            if (!tool)
-                throw new Error("Missing tool: read_xlsx_file");
-            extract = await tool.handler({
-                path: localPath,
-                headerRow: 1,
-                maxRows: 500,
-                maxCols: 80,
-                maxCellChars: 2000,
-            });
-        }
-        else if (ext === "pdf") {
-            const tool = toolIndex.get("read_pdf_text");
-            if (!tool)
-                throw new Error("Missing tool: read_pdf_text");
-            extract = await tool.handler({
-                path: localPath,
-                pageStart: 1,
-                pageEnd: 12,
-                maxChars: 40000,
-            });
-        }
-        else if (ext === "docx") {
-            const tool = toolIndex.get("read_docx_text");
-            if (!tool)
-                throw new Error("Missing tool: read_docx_text");
-            extract = await tool.handler({
-                path: localPath,
-                maxChars: 40000,
-            });
-        }
-        else if (ext === "pptx") {
-            const tool = toolIndex.get("read_pptx_text");
-            if (!tool)
-                throw new Error("Missing tool: read_pptx_text");
-            extract = await tool.handler({
-                path: localPath,
-                maxChars: 40000,
-            });
-        }
-        else if (ext === "json") {
-            const tool = toolIndex.get("read_json_file");
-            if (!tool)
-                throw new Error("Missing tool: read_json_file");
-            extract = await tool.handler({
-                path: localPath,
-                maxDepth: 10,
-                maxItems: 300,
-                maxStringChars: 2000,
-            });
-        }
-        else if (ext === "jsonl") {
-            const tool = toolIndex.get("read_jsonl_file");
-            if (!tool)
-                throw new Error("Missing tool: read_jsonl_file");
-            extract = await tool.handler({
-                path: localPath,
-                offsetLines: 0,
-                limitLines: 200,
-                parseJson: true,
-                maxDepth: 8,
-                maxItems: 200,
-                maxStringChars: 1000,
-            });
-        }
-        else if (ext === "txt" || ext === "md" || ext === "xml") {
-            const tool = toolIndex.get("read_text_file");
-            if (!tool)
-                throw new Error("Missing tool: read_text_file");
-            extract = await tool.handler({
-                path: localPath,
-                startChar: 0,
-                maxChars: 40000,
-            });
-        }
-        else if (ext === "zip") {
-            throw new Error('ZIP attachments are only supported in toolsMode="agent" (requires multi-step extraction).');
-        }
-        else {
-            throw new Error(`Unsupported attachment type: ${ext || "(unknown)"}`);
-        }
-        // Keep the model input bounded. The tools already return bounded previews,
-        // but JSON stringification can still be large on wide tables.
-        const derivedParity = deriveAddressParityIfRelevant(task.prompt, extract);
-        const inferredFromParity = derivedParity && (ext === "csv" || ext === "xlsx")
-            ? inferAnswerFromAddressParityIfPossible(task.prompt, derivedParity)
-            : null;
-        if (inferredFromParity) {
-            return { answer: inferredFromParity, toolCalls: 1 };
-        }
-        const enrichedExtract = derivedParity && (ext === "csv" || ext === "xlsx")
-            ? { ...extract, derivedParity: { address: derivedParity } }
-            : extract;
-        const extractText = JSON.stringify(enrichedExtract).slice(0, 40000);
-        const contents = [
-            {
-                role: "user",
-                parts: [
-                    {
-                        text: "Answer the question using the provided file extract plus general reasoning. " +
-                            "Do not browse the web and do not read any other files. " +
-                            "Reminder: the sun rises in the east and sets in the west. " +
-                            "If FILE_EXTRACT_JSON contains derivedParity, prefer it over recounting. " +
-                            "If the extract is insufficient, make the best supported guess.\n\n" +
-                            "Return ONLY the final answer, no explanation.\n\n" +
-                            `TASK_ID: ${task.id}\n` +
-                            `FILE_TYPE: ${ext}\n` +
-                            `LOCAL_FILE_PATH: ${localPath}\n` +
-                            `QUESTION:\n${task.prompt}\n\n` +
-                            `FILE_EXTRACT_JSON:\n${extractText}`,
-                    },
-                ],
-            },
-        ];
-        const answer = await llmGenerateText(llm, contents);
-        return { answer, toolCalls: 1 };
-    }
-    // "agent" mode: small tool loop. This is more realistic but higher variance.
-    const toolUsageSummary = [
-        "You have access to deterministic local file tools:",
-        "- where ops: eq, ne, contains, starts_with, ends_with, matches_regex, gt, gte, lt, lte, is_empty, not_empty, is_even, is_odd",
-        "- Prefer deterministic aggregations (csv_aggregate/xlsx_aggregate) over mental math. For parity rules, use where op is_even/is_odd.",
-        "- read_csv_file({path,hasHeader,delimiter,encoding,maxRows,maxCols,maxCellChars})",
-        "- csv_select_rows({path,hasHeader,delimiter,encoding,where,returnColumns,offset,limit,maxScanRows,maxCols,maxCellChars})",
-        "- csv_aggregate({path,hasHeader,delimiter,encoding,where,operation,value,ignoreNonNumeric,returnRow,returnColumns,maxScanRows,maxCols,maxCellChars})",
-        "- read_xlsx_file({path,sheetName,headerRow,rangeA1,maxRows,maxCols,maxCellChars})",
-        "- xlsx_select_rows({path,sheetName,headerRow,rangeA1,where,returnColumns,offset,limit,maxScanRows,maxCols,maxCellChars})",
-        "- xlsx_aggregate({path,sheetName,headerRow,rangeA1,where,operation,value,ignoreNonNumeric,returnRow,returnColumns,maxScanRows,maxCols,maxCellChars})",
-        "- read_pdf_text({path,pageStart,pageEnd,pageNumbers,maxChars})",
-        "- pdf_search_text({path,query,caseSensitive,pageStart,pageEnd,pageNumbers,maxMatches,snippetChars})",
-        "- read_text_file({path,encoding,startChar,maxChars})",
-        "- read_json_file({path,maxDepth,maxItems,maxStringChars})",
-        "- json_select({path,pointer,maxDepth,maxItems,maxStringChars})",
-        "- read_jsonl_file({path,encoding,offsetLines,limitLines,parseJson,maxLineChars,maxDepth,maxItems,maxStringChars})",
-        "- zip_list_files({path,maxEntries})",
-        "- zip_read_text_file({path,innerPath,caseSensitive,encoding,maxChars,maxBytes})",
-        "- zip_extract_file({path,innerPath,caseSensitive,outputDir,overwrite,maxBytes})",
-        "- read_docx_text({path,maxChars,maxBytes})",
-        "- read_pptx_text({path,maxChars,maxSlides,maxBytesPerSlide})",
-        "",
-        "When using tools, respond with a single JSON object only:",
-        "{\"action\":\"tool\",\"name\":\"read_pdf_text\",\"arguments\":{\"pageStart\":1,\"pageEnd\":5}}",
-        "When done, respond with:",
-        "{\"action\":\"final\",\"answer\":\"...\"}",
-        "",
-        "Rules:",
-        "- Do NOT use any external knowledge or web browsing.",
-        "- Always use the provided LOCAL_FILE_PATH; you may not read any other files.",
-        "- Keep tool results bounded (limit<=200, maxRows<=500, maxCols<=80, maxCellChars<=2000, maxChars<=40000, maxMatches<=50).",
-        "- Do NOT include any explanation. Final answer must match the requested formatting.",
-    ].join("\n");
-    const contents = [
-        {
-            role: "user",
-            parts: [
-                {
-                    text: `${toolUsageSummary}\n\nTASK_ID: ${task.id}\nFILE_TYPE: ${ext}\nLOCAL_FILE_PATH: ${localPath}\nQUESTION:\n${task.prompt}`,
-                },
-            ],
-        },
-    ];
-    let toolCalls = 0;
-    for (let step = 0; step < opts.maxSteps; step++) {
-        const out = await llmGenerateText(llm, contents);
-        contents.push({ role: "model", parts: [{ text: out }] });
-        const parsed = extractJsonObject(out);
-        if (!parsed || typeof parsed !== "object") {
-            contents.push({
-                role: "user",
-                parts: [{ text: "Invalid format. Return JSON only with action tool|final." }],
-            });
-            continue;
-        }
-        if (parsed.action === "final") {
-            const answer = String(parsed.answer ?? "").trim();
-            return { answer, toolCalls };
-        }
-        if (parsed.action !== "tool") {
-            contents.push({
-                role: "user",
-                parts: [{ text: "Invalid action. Return JSON only with action tool|final." }],
-            });
-            continue;
-        }
-        if (toolCalls >= opts.maxToolCalls) {
-            contents.push({
-                role: "user",
-                parts: [{ text: "Tool call budget exceeded. Return final answer now." }],
-            });
-            continue;
-        }
-        const name = String(parsed.name ?? "");
-        const tool = toolIndex.get(name);
-        if (!tool) {
-            contents.push({
-                role: "user",
-                parts: [
-                    {
-                        text: `Unknown tool "${name}". Use only read_csv_file, csv_select_rows, csv_aggregate, read_xlsx_file, xlsx_select_rows, xlsx_aggregate, read_pdf_text, or pdf_search_text.`,
-                    },
-                ],
-            });
-            continue;
-        }
-        const args = (parsed.arguments ?? {});
-        // Security: enforce file access restrictions.
-        // Default: force the path to the known GAIA attachment.
-        // ZIP: allow tools to operate on extracted children under a per-task extracted dir.
-        const extractedRoot = path.join(resolveRepoRoot(), ".cache", "gaia", "extracted", task.id);
-        const isZip = ext === "zip";
-        const isZipTool = ["zip_list_files", "zip_read_text_file", "zip_extract_file"].includes(name);
-        if (!isZip) {
-            args.path = localPath;
-        }
-        else if (isZipTool) {
-            args.path = localPath;
-            // Force deterministic extracted root to keep gated data under .cache/gaia (gitignored).
-            if (name === "zip_extract_file") {
-                args.outputDir = extractedRoot;
-            }
-        }
-        else {
-            const requested = String(args.path ?? "").trim();
-            if (!requested) {
-                contents.push({
-                    role: "user",
-                    parts: [
-                        {
-                            text: "ZIP workflow: first call zip_list_files, then zip_extract_file(innerPath=...), " +
-                                "then call a reader tool on the extractedPath returned by zip_extract_file.",
-                        },
-                    ],
-                });
-                continue;
-            }
-            const requestedAbs = path.isAbsolute(requested)
-                ? requested
-                : path.resolve(path.dirname(localPath), requested);
-            const extractedAbs = path.resolve(extractedRoot);
-            const reqResolved = path.resolve(requestedAbs);
-            if (!reqResolved.startsWith(extractedAbs + path.sep) && reqResolved !== extractedAbs) {
-                contents.push({
-                    role: "user",
-                    parts: [{ text: `Refusing to read path outside extractedRoot: ${reqResolved}` }],
-                });
-                continue;
-            }
-            args.path = reqResolved;
-        }
-        // Hard limits for safety and stable prompts.
-        if (name === "read_csv_file") {
-            if (args.hasHeader === undefined)
-                args.hasHeader = true;
-            if (typeof args.maxRows !== "number")
-                args.maxRows = 200;
-            if (typeof args.maxCols !== "number")
-                args.maxCols = 50;
-            if (typeof args.maxCellChars !== "number")
-                args.maxCellChars = 2000;
-            args.maxRows = Math.min(Number(args.maxRows) || 200, 500);
-            args.maxCols = Math.min(Number(args.maxCols) || 50, 80);
-            args.maxCellChars = Math.min(Number(args.maxCellChars) || 2000, 2000);
-        }
-        else if (name === "csv_select_rows") {
-            if (args.hasHeader === undefined)
-                args.hasHeader = true;
-            if (typeof args.offset !== "number")
-                args.offset = 0;
-            if (typeof args.limit !== "number")
-                args.limit = 50;
-            if (typeof args.maxScanRows !== "number")
-                args.maxScanRows = 50000;
-            if (typeof args.maxCols !== "number")
-                args.maxCols = 80;
-            if (typeof args.maxCellChars !== "number")
-                args.maxCellChars = 2000;
-            args.offset = Math.max(0, Number(args.offset) || 0);
-            args.limit = Math.min(Math.max(1, Number(args.limit) || 50), 200);
-            args.maxScanRows = Math.min(Math.max(1, Number(args.maxScanRows) || 50000), 50000);
-            args.maxCols = Math.min(Math.max(1, Number(args.maxCols) || 80), 80);
-            args.maxCellChars = Math.min(Math.max(20, Number(args.maxCellChars) || 2000), 2000);
-            if (Array.isArray(args.where))
-                args.where = args.where.slice(0, 10);
-            if (Array.isArray(args.returnColumns))
-                args.returnColumns = args.returnColumns.slice(0, 30);
-        }
-        else if (name === "csv_aggregate") {
-            if (args.hasHeader === undefined)
-                args.hasHeader = true;
-            if (typeof args.maxScanRows !== "number")
-                args.maxScanRows = 50000;
-            if (typeof args.maxCols !== "number")
-                args.maxCols = 200;
-            if (typeof args.maxCellChars !== "number")
-                args.maxCellChars = 2000;
-            args.maxScanRows = Math.min(Math.max(1, Number(args.maxScanRows) || 50000), 50000);
-            args.maxCols = Math.min(Math.max(1, Number(args.maxCols) || 200), 200);
-            args.maxCellChars = Math.min(Math.max(20, Number(args.maxCellChars) || 2000), 2000);
-            if (Array.isArray(args.where))
-                args.where = args.where.slice(0, 10);
-            if (Array.isArray(args.returnColumns))
-                args.returnColumns = args.returnColumns.slice(0, 30);
-        }
-        else if (name === "read_xlsx_file") {
-            if (typeof args.headerRow !== "number")
-                args.headerRow = 1;
-            if (typeof args.maxRows !== "number")
-                args.maxRows = 200;
-            if (typeof args.maxCols !== "number")
-                args.maxCols = 50;
-            if (typeof args.maxCellChars !== "number")
-                args.maxCellChars = 2000;
-            args.maxRows = Math.min(Number(args.maxRows) || 200, 500);
-            args.maxCols = Math.min(Number(args.maxCols) || 50, 80);
-            args.maxCellChars = Math.min(Number(args.maxCellChars) || 2000, 2000);
-        }
-        else if (name === "xlsx_select_rows") {
-            if (typeof args.headerRow !== "number")
-                args.headerRow = 1;
-            if (typeof args.offset !== "number")
-                args.offset = 0;
-            if (typeof args.limit !== "number")
-                args.limit = 50;
-            if (typeof args.maxScanRows !== "number")
-                args.maxScanRows = 50000;
-            if (typeof args.maxCols !== "number")
-                args.maxCols = 80;
-            if (typeof args.maxCellChars !== "number")
-                args.maxCellChars = 2000;
-            args.headerRow = Math.max(0, Math.min(Number(args.headerRow) || 1, 1000));
-            args.offset = Math.max(0, Number(args.offset) || 0);
-            args.limit = Math.min(Math.max(1, Number(args.limit) || 50), 200);
-            args.maxScanRows = Math.min(Math.max(1, Number(args.maxScanRows) || 50000), 50000);
-            args.maxCols = Math.min(Math.max(1, Number(args.maxCols) || 80), 80);
-            args.maxCellChars = Math.min(Math.max(20, Number(args.maxCellChars) || 2000), 2000);
-            if (Array.isArray(args.where))
-                args.where = args.where.slice(0, 10);
-            if (Array.isArray(args.returnColumns))
-                args.returnColumns = args.returnColumns.slice(0, 30);
-        }
-        else if (name === "xlsx_aggregate") {
-            if (typeof args.headerRow !== "number")
-                args.headerRow = 1;
-            if (typeof args.maxScanRows !== "number")
-                args.maxScanRows = 50000;
-            if (typeof args.maxCols !== "number")
-                args.maxCols = 200;
-            if (typeof args.maxCellChars !== "number")
-                args.maxCellChars = 2000;
-            args.headerRow = Math.max(0, Math.min(Number(args.headerRow) || 1, 1000));
-            args.maxScanRows = Math.min(Math.max(1, Number(args.maxScanRows) || 50000), 50000);
-            args.maxCols = Math.min(Math.max(1, Number(args.maxCols) || 200), 200);
-            args.maxCellChars = Math.min(Math.max(20, Number(args.maxCellChars) || 2000), 2000);
-            if (Array.isArray(args.where))
-                args.where = args.where.slice(0, 10);
-            if (Array.isArray(args.returnColumns))
-                args.returnColumns = args.returnColumns.slice(0, 30);
-        }
-        else if (name === "read_pdf_text") {
-            if (typeof args.pageStart !== "number")
-                args.pageStart = 1;
-            if (typeof args.pageEnd !== "number")
-                args.pageEnd = 3;
-            if (typeof args.maxChars !== "number")
-                args.maxChars = 12000;
-            args.pageStart = Math.max(1, Math.min(Number(args.pageStart) || 1, 500));
-            args.pageEnd = Math.max(1, Math.min(Number(args.pageEnd) || 3, 500));
-            args.maxChars = Math.min(Number(args.maxChars) || 12000, 40000);
-            if (Array.isArray(args.pageNumbers)) {
-                // Keep explicit page lists short to avoid huge extracts.
-                args.pageNumbers = args.pageNumbers
-                    .map((n) => Number(n))
-                    .filter((n) => Number.isFinite(n) && n > 0)
-                    .slice(0, 20);
-            }
-        }
-        else if (name === "pdf_search_text") {
-            if (typeof args.query !== "string")
-                args.query = "";
-            if (typeof args.pageStart !== "number")
-                args.pageStart = 1;
-            if (typeof args.pageEnd !== "number")
-                args.pageEnd = 25;
-            if (typeof args.maxMatches !== "number")
-                args.maxMatches = 25;
-            if (typeof args.snippetChars !== "number")
-                args.snippetChars = 180;
-            args.pageStart = Math.max(1, Math.min(Number(args.pageStart) || 1, 500));
-            args.pageEnd = Math.max(1, Math.min(Number(args.pageEnd) || 25, 500));
-            args.maxMatches = Math.min(Math.max(1, Number(args.maxMatches) || 25), 50);
-            args.snippetChars = Math.min(Math.max(40, Number(args.snippetChars) || 180), 400);
-            if (Array.isArray(args.pageNumbers)) {
-                args.pageNumbers = args.pageNumbers
-                    .map((n) => Number(n))
-                    .filter((n) => Number.isFinite(n) && n > 0)
-                    .slice(0, 20);
-            }
-        }
-        else if (name === "read_text_file") {
-            if (typeof args.startChar !== "number")
-                args.startChar = 0;
-            if (typeof args.maxChars !== "number")
-                args.maxChars = 12000;
-            args.startChar = Math.max(0, Number(args.startChar) || 0);
-            args.maxChars = Math.min(Math.max(1, Number(args.maxChars) || 12000), 40000);
-        }
-        else if (name === "read_json_file" || name === "json_select") {
-            if (typeof args.maxDepth !== "number")
-                args.maxDepth = 8;
-            if (typeof args.maxItems !== "number")
-                args.maxItems = 200;
-            if (typeof args.maxStringChars !== "number")
-                args.maxStringChars = 2000;
-            args.maxDepth = Math.min(Math.max(1, Number(args.maxDepth) || 8), 12);
-            args.maxItems = Math.min(Math.max(1, Number(args.maxItems) || 200), 500);
-            args.maxStringChars = Math.min(Math.max(20, Number(args.maxStringChars) || 2000), 2000);
-            if (name === "json_select" && typeof args.pointer !== "string")
-                args.pointer = "";
-        }
-        else if (name === "read_jsonl_file") {
-            if (typeof args.offsetLines !== "number")
-                args.offsetLines = 0;
-            if (typeof args.limitLines !== "number")
-                args.limitLines = 200;
-            if (typeof args.maxLineChars !== "number")
-                args.maxLineChars = 4000;
-            if (typeof args.maxDepth !== "number")
-                args.maxDepth = 6;
-            if (typeof args.maxItems !== "number")
-                args.maxItems = 100;
-            if (typeof args.maxStringChars !== "number")
-                args.maxStringChars = 1000;
-            args.offsetLines = Math.max(0, Number(args.offsetLines) || 0);
-            args.limitLines = Math.min(Math.max(1, Number(args.limitLines) || 200), 500);
-            args.maxLineChars = Math.min(Math.max(200, Number(args.maxLineChars) || 4000), 10000);
-            args.maxDepth = Math.min(Math.max(1, Number(args.maxDepth) || 6), 10);
-            args.maxItems = Math.min(Math.max(1, Number(args.maxItems) || 100), 300);
-            args.maxStringChars = Math.min(Math.max(20, Number(args.maxStringChars) || 1000), 2000);
-        }
-        else if (name === "zip_list_files") {
-            if (typeof args.maxEntries !== "number")
-                args.maxEntries = 200;
-            args.maxEntries = Math.min(Math.max(1, Number(args.maxEntries) || 200), 500);
-        }
-        else if (name === "zip_read_text_file") {
-            if (typeof args.innerPath !== "string")
-                args.innerPath = "";
-            if (typeof args.maxChars !== "number")
-                args.maxChars = 12000;
-            if (typeof args.maxBytes !== "number")
-                args.maxBytes = 5000000;
-            args.maxChars = Math.min(Math.max(200, Number(args.maxChars) || 12000), 20000);
-            args.maxBytes = Math.min(Math.max(1000, Number(args.maxBytes) || 5000000), 20000000);
-        }
-        else if (name === "zip_extract_file") {
-            if (typeof args.innerPath !== "string")
-                args.innerPath = "";
-            if (typeof args.maxBytes !== "number")
-                args.maxBytes = 25000000;
-            args.maxBytes = Math.min(Math.max(1000, Number(args.maxBytes) || 25000000), 50000000);
-            args.overwrite = false;
-        }
-        else if (name === "read_docx_text") {
-            if (typeof args.maxChars !== "number")
-                args.maxChars = 12000;
-            args.maxChars = Math.min(Math.max(200, Number(args.maxChars) || 12000), 40000);
-        }
-        else if (name === "read_pptx_text") {
-            if (typeof args.maxChars !== "number")
-                args.maxChars = 12000;
-            if (typeof args.maxSlides !== "number")
-                args.maxSlides = 60;
-            args.maxChars = Math.min(Math.max(200, Number(args.maxChars) || 12000), 40000);
-            args.maxSlides = Math.min(Math.max(1, Number(args.maxSlides) || 60), 120);
-        }
-        // Reduce model confusion: enforce tool matches the attachment type.
-        const allowedByExt = (ext === "csv" && ["read_csv_file", "csv_select_rows", "csv_aggregate"].includes(name)) ||
-            (ext === "xlsx" && ["read_xlsx_file", "xlsx_select_rows", "xlsx_aggregate"].includes(name)) ||
-            (ext === "pdf" && ["read_pdf_text", "pdf_search_text"].includes(name)) ||
-            (ext === "docx" && ["read_docx_text"].includes(name)) ||
-            (ext === "pptx" && ["read_pptx_text"].includes(name)) ||
-            ((ext === "txt" || ext === "md" || ext === "xml") && ["read_text_file"].includes(name)) ||
-            (ext === "json" && ["read_json_file", "json_select", "read_text_file"].includes(name)) ||
-            (ext === "jsonl" && ["read_jsonl_file", "read_text_file"].includes(name)) ||
-            (ext === "zip" &&
-                [
-                    "zip_list_files",
-                    "zip_read_text_file",
-                    "zip_extract_file",
-                    "read_csv_file",
-                    "csv_select_rows",
-                    "csv_aggregate",
-                    "read_xlsx_file",
-                    "xlsx_select_rows",
-                    "xlsx_aggregate",
-                    "read_pdf_text",
-                    "pdf_search_text",
-                    "read_text_file",
-                    "read_json_file",
-                    "json_select",
-                    "read_jsonl_file",
-                    "read_docx_text",
-                    "read_pptx_text",
-                ].includes(name));
-        if (!allowedByExt) {
-            contents.push({
-                role: "user",
-                parts: [{ text: `Wrong tool for FILE_TYPE=${ext}. Use a tool that matches the file type.` }],
-            });
-            continue;
-        }
-        toolCalls++;
-        const toolResult = await tool.handler(args);
-        // Provide a bounded JSON summary to the model. Avoid dumping large content.
-        const toolResultText = JSON.stringify(toolResult).slice(0, 12000);
-        contents.push({
-            role: "user",
-            parts: [{ text: `TOOL_RESULT ${name}:\n${toolResultText}\n\nContinue. Return JSON only.` }],
-        });
-    }
-    // If we ran out of steps, force a final answer.
-    contents.push({
-        role: "user",
-        parts: [{ text: "Out of steps. Return final answer now as JSON." }],
-    });
-    const out = await llmGenerateText(llm, contents);
-    const parsed = extractJsonObject(out);
-    if (parsed?.action === "final") {
-        return { answer: String(parsed.answer ?? "").trim(), toolCalls };
-    }
-    return { answer: String(out ?? "").trim(), toolCalls };
-}
-async function loadFixture(fixturePath) {
-    const raw = await readFile(fixturePath, "utf8");
-    const parsed = JSON.parse(raw);
-    if (!parsed || !Array.isArray(parsed.tasks))
-        throw new Error("Invalid GAIA capability files fixture");
-    return parsed;
-}
-describe("Capability: GAIA accuracy (file-backed) (LLM-only vs LLM+local tools)", () => {
-    const testFn = shouldRun ? it : it.skip;
-    testFn("should measure accuracy delta on a small GAIA file-backed subset", async () => {
-        loadDotEnvLocalIfPresent();
-        const fixturePath = resolveCapabilityFilesFixturePath();
-        if (!existsSync(fixturePath)) {
-            throw new Error(`Missing GAIA capability files fixture at ${fixturePath}. Generate it with: python packages/mcp-local/src/__tests__/fixtures/generateGaiaCapabilityFilesFixture.py`);
-        }
-        const baselineModel = process.env.NODEBENCH_GAIA_BASELINE_MODEL ?? "gemini-3-flash-preview";
-        const toolsModel = process.env.NODEBENCH_GAIA_TOOLS_MODEL ?? baselineModel;
-        const baselineLlm = await createTextLlmClient({ model: baselineModel });
-        const toolsLlm = await createTextLlmClient({ model: toolsModel });
-        const baselineModelLabel = `${baselineLlm.provider}:${baselineLlm.model}`;
-        const toolsModelLabel = `${toolsLlm.provider}:${toolsLlm.model}`;
-        const fixture = await loadFixture(fixturePath);
-        expect(Array.isArray(fixture.tasks)).toBe(true);
-        expect(fixture.tasks.length).toBeGreaterThan(0);
-        const requestedLimit = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_TASK_LIMIT ?? "6", 10);
-        const taskLimit = Math.max(1, Math.min(fixture.tasks.length, Number.isFinite(requestedLimit) ? requestedLimit : 6));
-        const tasks = fixture.tasks.slice(0, taskLimit);
-        const requestedConcurrency = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_CONCURRENCY ?? "1", 10);
-        const concurrency = Math.max(1, Math.min(tasks.length, Number.isFinite(requestedConcurrency) ? requestedConcurrency : 1));
-        const maxSteps = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_MAX_STEPS ?? "7", 10);
-        const maxToolCalls = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_MAX_TOOL_CALLS ?? "5", 10);
-        // Auto-discover judge (free OpenRouter → paid LLM → deterministic-only)
-        const judge = await autoDiscoverJudge(toolsLlm);
-        const results = new Array(tasks.length);
-        let nextIndex = 0;
-        const workers = Array.from({ length: concurrency }, () => (async () => {
-            while (true) {
-                const idx = nextIndex++;
-                if (idx >= tasks.length)
-                    return;
-                const task = tasks[idx];
-                try {
-                    const baseStart = performance.now();
-                    const base = await baselineAnswer(baselineLlm, task);
-                    const baseMs = performance.now() - baseStart;
-                    const toolsStart = performance.now();
-                    const tools = await toolAugmentedAnswerFromFile(toolsLlm, task, { maxSteps, maxToolCalls });
-                    const toolsMs = performance.now() - toolsStart;
-                    const baseJudge = await answersMatchWithJudge(task.expectedAnswer, base, judge);
-                    const toolsJudge = await answersMatchWithJudge(task.expectedAnswer, tools.answer, judge);
-                    results[idx] = {
-                        taskId: task.id,
-                        baselineCorrect: baseJudge.match,
-                        toolsCorrect: toolsJudge.match,
-                        baselineMs: baseMs,
-                        toolsMs,
-                        toolCalls: tools.toolCalls,
-                        judgeProvider: toolsJudge.judgeProvider,
-                        judgeInvoked: toolsJudge.judgeInvoked,
-                    };
-                }
-                catch (err) {
-                    results[idx] = {
-                        taskId: task.id,
-                        baselineCorrect: false,
-                        toolsCorrect: false,
-                        baselineMs: 0,
-                        toolsMs: 0,
-                        toolCalls: 0,
-                        error: err?.message ?? String(err),
-                    };
-                }
-            }
-        })());
-        await Promise.all(workers);
-        const baselineCorrect = results.filter((r) => r.baselineCorrect).length;
-        const toolsCorrect = results.filter((r) => r.toolsCorrect).length;
-        const improved = results.filter((r) => !r.baselineCorrect && r.toolsCorrect).length;
-        const regressions = results.filter((r) => r.baselineCorrect && !r.toolsCorrect).length;
-        const avg = (values) => values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;
-        const avgBaseMs = avg(results.map((r) => r.baselineMs).filter((n) => n > 0));
-        const avgToolsMs = avg(results.map((r) => r.toolsMs).filter((n) => n > 0));
-        const avgToolCalls = avg(results.map((r) => r.toolCalls));
-        console.log(`[gaia-capability-files] config=${fixture.config} split=${fixture.split} tasks=${tasks.length} concurrency=${concurrency} baseline=${baselineCorrect}/${tasks.length} tools=${toolsCorrect}/${tasks.length} improved=${improved} regressions=${regressions} avgBaselineMs=${avgBaseMs.toFixed(0)} avgToolsMs=${avgToolsMs.toFixed(0)} avgToolCalls=${avgToolCalls.toFixed(2)}`);
-        console.log(`[gaia-capability-files] perTask: ${results
-            .map((r) => `${r.taskId}:B${r.baselineCorrect ? "1" : "0"}T${r.toolsCorrect ? "1" : "0"}${r.error ? "E" : ""}`)
-            .join(" ")}`);
-        if (shouldWriteReport) {
-            const repoRoot = resolveRepoRoot();
-            const generatedAtIso = new Date().toISOString();
-            const stamp = generatedAtIso.replace(/[:.]/g, "-");
-            const toolsMode = (process.env.NODEBENCH_GAIA_CAPABILITY_TOOLS_MODE ?? "rag").toLowerCase();
-            const publicSummary = {
-                suiteId: "gaia_capability_files",
-                lane: "files",
-                generatedAtIso,
-                config: fixture.config,
-                split: fixture.split,
-                taskCount: tasks.length,
-                concurrency,
-                baseline: {
-                    model: baselineModelLabel,
-                    correct: baselineCorrect,
-                    passRatePct: tasks.length === 0 ? 0 : (baselineCorrect / tasks.length) * 100,
-                    avgMs: avgBaseMs,
-                },
-                tools: {
-                    model: toolsModelLabel,
-                    mode: toolsMode,
-                    correct: toolsCorrect,
-                    passRatePct: tasks.length === 0 ? 0 : (toolsCorrect / tasks.length) * 100,
-                    avgMs: avgToolsMs,
-                    avgToolCalls: avgToolCalls,
-                },
-                improved,
-                regressions,
-                notes: "GAIA is gated. This file contains only aggregate metrics (no prompt/answer text). Detailed per-task report is written under .cache/gaia/reports (gitignored).",
-            };
-            await safeWriteJson(path.join(repoRoot, "public", "evals", "gaia_capability_files_latest.json"), publicSummary);
-            await safeWriteJson(path.join(repoRoot, ".cache", "gaia", "reports", `gaia_capability_files_${fixture.config}_${fixture.split}_${stamp}.json`), {
-                ...publicSummary,
-                perTask: results.map((r) => ({
-                    taskId: r.taskId,
-                    baselineCorrect: r.baselineCorrect,
-                    toolsCorrect: r.toolsCorrect,
-                    baselineMs: r.baselineMs,
-                    toolsMs: r.toolsMs,
-                    toolCalls: r.toolCalls,
-                    error: r.error ?? null,
-                })),
-            });
-        }
-        const enforce = process.env.NODEBENCH_GAIA_CAPABILITY_ENFORCE === "1";
-        if (enforce) {
-            // For file-backed tasks, the baseline is expected to be low. We still enforce that
-            // tool-augmented performance is not worse than baseline and has at least one improvement.
-            const allowedRegression = Math.max(1, Math.floor(tasks.length * 0.2));
-            expect(improved).toBeGreaterThanOrEqual(1);
-            expect(toolsCorrect).toBeGreaterThanOrEqual(baselineCorrect - allowedRegression);
-            expect(toolsCorrect).toBeGreaterThanOrEqual(1);
-        }
-        else {
-            expect(results.length).toBe(tasks.length);
-            expect(results.some((r) => r.error)).toBe(false);
-        }
-    }, 300000);
-});
-//# sourceMappingURL=gaiaCapabilityFilesEval.test.js.map