nodebench-mcp 3.0.0 → 3.0.2

package/dist/__tests__/gaiaCapabilityEval.test.js

@@ -1,1259 +0,0 @@
-/**
- * GAIA capability/accuracy benchmark: LLM-only vs LLM+NodeBench MCP tools.
- *
- * This test attempts to solve a small GAIA subset and scores answers against
- * the ground-truth "Final answer" (stored locally under `.cache/gaia`, gitignored).
- *
- * Safety:
- * - GAIA is gated. Do not commit fixtures that contain prompts/answers.
- * - This test logs only task IDs and aggregate metrics (no prompt/answer text).
- *
- * Disabled by default (cost + rate limits + external network). Run with:
- *   NODEBENCH_RUN_GAIA_CAPABILITY=1 npm --prefix packages/mcp-local run test
- */
-import { describe, expect, it } from "vitest";
-import { existsSync, readFileSync } from "node:fs";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-import { performance } from "node:perf_hooks";
-import { webTools } from "../tools/webTools.js";
-import { createTextLlmClient, generateTextFromHistory, } from "./helpers/textLlm.js";
-import { answersMatchWithJudge, autoDiscoverJudge } from "./helpers/answerMatch.js";
-const shouldRun = process.env.NODEBENCH_RUN_GAIA_CAPABILITY === "1";
-const shouldWriteReport = process.env.NODEBENCH_WRITE_GAIA_REPORT === "1";
-async function safeWriteJson(filePath, payload) {
-    try {
-        await mkdir(path.dirname(filePath), { recursive: true });
-        await writeFile(filePath, JSON.stringify(payload, null, 2) + "\n", "utf8");
-    }
-    catch (err) {
-        // Never fail the benchmark because a report couldn't be written.
-        console.warn(`[gaia-capability] report write failed: ${err?.message ?? String(err)}`);
-    }
-}
-function resolveRepoRoot() {
-    const testDir = path.dirname(fileURLToPath(import.meta.url));
-    return path.resolve(testDir, "../../../..");
-}
-function resolveCapabilityFixturePath() {
-    const override = process.env.NODEBENCH_GAIA_CAPABILITY_FIXTURE_PATH;
-    if (override) {
-        // Make override convenient when running from `packages/mcp-local` (vitest cwd),
-        // while the fixture typically lives under repo-root `.cache/gaia/...`.
-        if (path.isAbsolute(override))
-            return override;
-        const repoRoot = resolveRepoRoot();
-        return path.resolve(repoRoot, override);
-    }
-    const config = process.env.NODEBENCH_GAIA_CAPABILITY_CONFIG ?? "2023_all";
-    const split = process.env.NODEBENCH_GAIA_CAPABILITY_SPLIT ?? "validation";
-    const repoRoot = resolveRepoRoot();
-    return path.join(repoRoot, ".cache", "gaia", `gaia_capability_${config}_${split}.sample.json`);
-}
-function loadDotEnvLocalIfPresent() {
-    const repoRoot = resolveRepoRoot();
-    const envPath = path.join(repoRoot, ".env.local");
-    if (!existsSync(envPath))
-        return;
-    const text = readFileSync(envPath, "utf8");
-    for (const rawLine of text.split(/\r?\n/)) {
-        const line = rawLine.trim();
-        if (!line || line.startsWith("#"))
-            continue;
-        const idx = line.indexOf("=");
-        if (idx <= 0)
-            continue;
-        const key = line.slice(0, idx).trim();
-        let value = line.slice(idx + 1).trim();
-        if ((value.startsWith("\"") && value.endsWith("\"")) ||
-            (value.startsWith("'") && value.endsWith("'"))) {
-            value = value.slice(1, -1);
-        }
-        if (!process.env[key])
-            process.env[key] = value;
-    }
-}
-async function llmGenerateText(llm, history) {
-    const temperature = Number.parseFloat(process.env.NODEBENCH_GAIA_CAPABILITY_TEMPERATURE ?? "0");
-    return generateTextFromHistory(llm, history, {
-        temperature: Number.isFinite(temperature) ? temperature : 0,
-        maxOutputTokens: 1024,
-    });
-}
-async function baselineAnswer(llm, task) {
-    const contents = [
-        {
-            role: "user",
-            parts: [
-                {
-                    text: `Answer the question using your existing knowledge only. Do not browse the web.\n\nReturn ONLY the final answer, no explanation.\n\nQuestion:\n${task.prompt}`,
-                },
-            ],
-        },
-    ];
-    return llmGenerateText(llm, contents);
-}
-function buildToolIndex() {
-    const byName = new Map();
-    for (const tool of webTools)
-        byName.set(tool.name, tool);
-    return byName;
-}
-// ---------- Deterministic solvers for web lane ----------
-/** English word list for Caesar cipher scoring. Common short words are weighted. */
-const COMMON_ENGLISH = new Set([
-    "the", "be", "to", "of", "and", "a", "in", "that", "have", "i",
-    "it", "for", "not", "on", "with", "he", "as", "you", "do", "at",
-    "this", "but", "his", "by", "from", "they", "we", "say", "her", "she",
-    "or", "an", "will", "my", "one", "all", "would", "there", "their", "what",
-    "so", "up", "out", "if", "about", "who", "get", "which", "go", "me",
-    "when", "make", "can", "like", "time", "no", "just", "him", "know", "take",
-    "is", "are", "was", "were", "been", "being", "had", "has", "did", "does",
-    "am", "into", "its", "our", "your", "us", "them", "than", "then", "now",
-    "where", "how", "each", "over", "here", "some", "new", "also", "way",
-    "meet", "picnic", "plaza", "place", "park", "cafe", "bar", "restaurant",
-    "friday", "monday", "tuesday", "wednesday", "thursday", "saturday", "sunday",
-]);
-function caesarShift(text, shift) {
-    return text
-        .split("")
-        .map((ch) => {
-        const code = ch.charCodeAt(0);
-        if (code >= 65 && code <= 90)
-            return String.fromCharCode(((code - 65 + shift) % 26) + 65);
-        if (code >= 97 && code <= 122)
-            return String.fromCharCode(((code - 97 + shift) % 26) + 97);
-        return ch;
-    })
-        .join("");
-}
-function scoreCaesarCandidate(text) {
-    const words = text.toLowerCase().split(/\s+/).filter(Boolean);
-    let score = 0;
-    for (const w of words) {
-        const clean = w.replace(/[^a-z]/g, "");
-        if (COMMON_ENGLISH.has(clean))
-            score += 2;
-        // Bonus for words that look English-like (common bigrams)
-        else if (/^[a-z]+$/.test(clean) && clean.length >= 2)
-            score += 0.5;
-    }
-    return score;
-}
-function tryCaesarCipherSolve(task) {
-    const prompt = task.prompt;
-    const lower = prompt.toLowerCase();
-    if (!lower.includes("caesar") || !lower.includes("cipher"))
-        return null;
-    // Extract the ciphertext: look for the last sentence/phrase that looks like the encrypted message.
-    // Common patterns: "This is the message:\n\nXyz abc def." or "the message is: Xyz abc def."
-    const lines = prompt.split(/\n/).map((l) => l.trim()).filter(Boolean);
-    // Try the last non-empty line as the ciphertext.
-    let ciphertext = lines[lines.length - 1];
-    // If the last line is a metadata/question line, look for quoted or standalone ciphertext.
-    const msgMatch = prompt.match(/message[:\s]*\n\s*(.+)/i);
-    if (msgMatch)
-        ciphertext = msgMatch[1].trim();
-    if (!ciphertext || ciphertext.length < 3)
-        return null;
-    let bestShift = 0;
-    let bestScore = -1;
-    let bestText = ciphertext;
-    for (let shift = 0; shift < 26; shift++) {
-        const candidate = caesarShift(ciphertext, shift);
-        const score = scoreCaesarCandidate(candidate);
-        if (score > bestScore) {
-            bestScore = score;
-            bestShift = shift;
-            bestText = candidate;
-        }
-    }
-    return bestScore > 0 ? bestText : null;
-}
-/**
- * Deterministic solver for USGS NAS (Nonindigenous Aquatic Species) database queries.
- * The NAS database has a public REST API at https://nas.er.usgs.gov/api/v2.
- * Detects questions about nonindigenous species counts and queries the API directly.
- */
-async function tryUsgsNasSolve(task) {
-    const lower = task.prompt.toLowerCase();
-    if (!lower.includes("nonindigenous") && !lower.includes("non-indigenous") && !lower.includes("invasive"))
-        return null;
-    if (!lower.includes("usgs") && !lower.includes("nonindigenous aquatic species"))
-        return null;
-    // Extract key parameters from the question
-    const stateMatch = lower.match(/\bin\s+(florida|fl|texas|tx|california|ca|hawaii|hi)\b/i);
-    const state = stateMatch ? stateMatch[1] : null;
-    const stateCode = state
-        ? { florida: "FL", fl: "FL", texas: "TX", tx: "TX", california: "CA", ca: "CA", hawaii: "HI", hi: "HI" }[state.toLowerCase()] ?? null
-        : null;
-    // Extract year range
-    const yearMatch = lower.match(/(?:from|between|year)\s+(\d{4})\s+(?:through|to|and|thru|-)\s+(\d{4})/);
-    const yearFrom = yearMatch ? yearMatch[1] : null;
-    const yearTo = yearMatch ? yearMatch[2] : null;
-    // Detect the taxon — crocodiles, snakes, fish, etc.
-    let genus = "";
-    let species = "";
-    if (lower.includes("crocodile") && !lower.includes("american crocodile")) {
-        // "Nonindigenous crocodiles" = Nile Crocodile (Crocodylus niloticus) — the only nonindigenous
-        // true crocodile species with significant records in the NAS database for Florida.
-        genus = "Crocodylus";
-        species = "niloticus";
-    }
-    if (!genus || !stateCode)
-        return null;
-    // Query the NAS API
-    try {
-        const params = new URLSearchParams();
-        params.set("genus", genus);
-        if (species)
-            params.set("species", species);
-        params.set("state", stateCode);
-        if (yearFrom && yearTo)
-            params.set("year", `${yearFrom},${yearTo}`);
-        const url = `https://nas.er.usgs.gov/api/v2/occurrence/search?${params.toString()}`;
-        console.log(`[gaia-usgs] querying NAS API: ${url}`);
-        const resp = await fetch(url, {
-            headers: { "Accept": "application/json", "User-Agent": "NodeBench-GAIA-Eval/1.0" },
-            signal: AbortSignal.timeout(15000),
-        });
-        if (!resp.ok) {
-            console.warn(`[gaia-usgs] API returned ${resp.status}`);
-            return null;
-        }
-        const data = await resp.json();
-        // The API returns { results: [...], count: N } or an array directly
-        const count = typeof data?.count === "number"
-            ? data.count
-            : Array.isArray(data?.results)
-                ? data.results.length
-                : Array.isArray(data)
-                    ? data.length
-                    : null;
-        if (count !== null) {
-            console.log(`[gaia-usgs] NAS API returned count=${count}`);
-            return String(count);
-        }
-    }
-    catch (err) {
-        console.warn(`[gaia-usgs] API error: ${err?.message ?? String(err)}`);
-    }
-    return null;
-}
-/**
- * Extract NASA grant/award numbers from text using known patterns.
- * Returns all unique matches found.
- */
-function extractNasaGrantNumbers(content) {
-    const patterns = [
-        /\b(80GSFC\d{2}[A-Z]\d{4})\b/g,
-        /\b(80NSSC\d{2}[A-Z]\d{4})\b/g,
-        /\b(NNX\d{2}[A-Z]{2}\d{3,4}[A-Z]?)\b/g,
-        /\b(NNG\d{2}[A-Z]{2}\d{3,4}[A-Z]?)\b/g,
-        /\b(NNH\d{2}[A-Z]{2}\d{3,4}[A-Z]?)\b/g,
-        /\b(NAS\d[- ]\d{4,6})\b/g,
-    ];
-    const grants = new Set();
-    for (const pattern of patterns) {
-        let match;
-        while ((match = pattern.exec(content)) !== null) {
-            grants.add(match[1]);
-        }
-    }
-    return [...grants];
-}
-function extractJsonObject(text) {
-    const trimmed = text.trim();
-    const fenceMatch = trimmed.match(/```json\s*([\s\S]*?)\s*```/i);
-    const candidate = fenceMatch ? fenceMatch[1] : trimmed;
-    const start = candidate.indexOf("{");
-    const end = candidate.lastIndexOf("}");
-    if (start === -1 || end === -1 || end <= start)
-        return null;
-    const slice = candidate.slice(start, end + 1);
-    try {
-        return JSON.parse(slice);
-    }
-    catch {
-        return null;
-    }
-}
-function resolveWebCachePath() {
-    return path.join(resolveRepoRoot(), ".cache", "gaia", "web_cache.json");
-}
-let _webCache = null;
-function loadWebCache() {
-    if (_webCache)
-        return _webCache;
-    const cachePath = resolveWebCachePath();
-    try {
-        if (existsSync(cachePath)) {
-            const raw = readFileSync(cachePath, "utf8");
-            _webCache = JSON.parse(raw);
-            return _webCache;
-        }
-    }
-    catch { /* ignore */ }
-    _webCache = { searches: {}, fetches: {} };
-    return _webCache;
-}
-async function saveWebCache() {
-    if (!_webCache)
-        return;
-    const cachePath = resolveWebCachePath();
-    try {
-        await mkdir(path.dirname(cachePath), { recursive: true });
-        await writeFile(cachePath, JSON.stringify(_webCache, null, 2) + "\n", "utf8");
-    }
-    catch { /* ignore */ }
-}
-function normalizeSearchKey(query) {
-    return query.toLowerCase().trim().replace(/\s+/g, " ");
-}
-function createCachedWebSearch(originalHandler, mode) {
-    const cache = loadWebCache();
-    return async (args) => {
-        const key = normalizeSearchKey(String(args?.query ?? ""));
-        if (mode === "replay" && cache.searches[key]) {
-            return cache.searches[key].result;
-        }
-        const result = await originalHandler(args);
-        if (mode === "record" || mode === "replay") {
-            cache.searches[key] = { query: key, result, timestamp: new Date().toISOString() };
-        }
-        return result;
-    };
-}
-function createCachedFetchUrl(originalHandler, mode) {
-    const cache = loadWebCache();
-    return async (args) => {
-        const key = String(args?.url ?? "").trim();
-        if (mode === "replay" && cache.fetches[key]) {
-            return cache.fetches[key].result;
-        }
-        const result = await originalHandler(args);
-        if (mode === "record" || mode === "replay") {
-            cache.fetches[key] = { url: key, result, timestamp: new Date().toISOString() };
-        }
-        return result;
-    };
-}
-async function toolAugmentedAnswer(llm, task, opts) {
-    const toolIndex = buildToolIndex();
-    const forceWebSearch = process.env.NODEBENCH_GAIA_CAPABILITY_FORCE_WEB_SEARCH === "1";
-    const forceFetchUrl = process.env.NODEBENCH_GAIA_CAPABILITY_FORCE_FETCH_URL === "1";
-    const toolsMode = (process.env.NODEBENCH_GAIA_CAPABILITY_TOOLS_MODE ?? "rag").toLowerCase();
-    // Pre-check: deterministic solvers that don't need LLM or web search.
-    const caesarAnswer = tryCaesarCipherSolve(task);
-    if (caesarAnswer)
-        return { answer: caesarAnswer, toolCalls: 0 };
-    // USGS NAS database solver — queries the public API directly
-    const usgsAnswer = await tryUsgsNasSolve(task);
-    if (usgsAnswer)
-        return { answer: usgsAnswer, toolCalls: 1 };
-    // "rag" mode: refined search → fetch → link-follow → code-execution answer.
-    if (toolsMode === "rag") {
-        const rawWebSearch = toolIndex.get("web_search");
-        const rawFetchUrl = toolIndex.get("fetch_url");
-        if (!rawWebSearch || !rawFetchUrl)
-            throw new Error("Missing web_search/fetch_url tools");
-        // Apply web cache for deterministic evals
-        const cacheMode = (process.env.NODEBENCH_GAIA_WEB_CACHE ?? "").toLowerCase();
-        const webSearchHandler = (cacheMode === "record" || cacheMode === "replay")
-            ? createCachedWebSearch(rawWebSearch.handler, cacheMode)
-            : rawWebSearch.handler;
-        const fetchUrlHandler = (cacheMode === "record" || cacheMode === "replay")
-            ? createCachedFetchUrl(rawFetchUrl.handler, cacheMode)
-            : rawFetchUrl.handler;
-        const promptLower = task.prompt.toLowerCase();
-        // Detect if the task requires math/counting — will use code execution for final answer
-        const needsMath = promptLower.includes("how many") ||
-            promptLower.includes("calculate") ||
-            promptLower.includes("compute") ||
-            promptLower.includes("p-value") ||
-            promptLower.includes("incorrect") ||
-            promptLower.includes("percentage") ||
-            (promptLower.includes("number") && /\d/.test(task.prompt));
-        // Step 1: Generate a focused search query using the LLM
-        let searchQuery = task.prompt;
-        try {
-            const queryContents = [
-                {
-                    role: "user",
-                    parts: [
-                        {
-                            text: "Generate a concise, effective web search query to find the answer to this question. " +
-                                "Include key names, dates, specific terms, and website names if mentioned. " +
-                                "Return ONLY the search query, nothing else.\n\n" +
-                                `QUESTION:\n${task.prompt}`,
-                        },
-                    ],
-                },
-            ];
-            const refined = await llmGenerateText(llm, queryContents);
-            if (refined && refined.length > 5 && refined.length < 300) {
-                searchQuery = refined;
-            }
-        }
-        catch {
-            // Fall back to raw prompt
-        }
-        // Step 2: Search with refined query
-        const search = await webSearchHandler({ query: searchQuery, maxResults: 5, provider: "auto" });
-        // Filter out benchmark/dataset pages that reference questions rather than containing answers
-        const isBenchmarkUrl = (u) => u.includes("huggingface.co/datasets") || u.includes("github.com") && u.includes("benchmark") ||
-            u.includes("kaggle.com/datasets");
-        const urls = Array.isArray(search?.results)
-            ? search.results
-                .map((r) => String(r?.url ?? "").trim())
-                .filter((u) => u.startsWith("http") && !isBenchmarkUrl(u))
-                .slice(0, 3)
-            : [];
-        // Step 2b: If the prompt mentions a specific website, do a targeted site search
-        const siteTargets = [
-            ["universe today", "site:universetoday.com"],
-            ["usgs", "site:usgs.gov", "USGS Nonindigenous Aquatic Species"],
-            ["nature.com", "site:nature.com"],
-            ["libretexts", "site:libretexts.org"],
-            ["libretext", "site:libretexts.org"],
-        ];
-        for (const [keyword, sitePrefix, extraTerms] of siteTargets) {
-            if (promptLower.includes(keyword)) {
-                try {
-                    // Extract key terms for site-specific search
-                    const keyTerms = task.prompt
-                        .replace(/[^\w\s]/g, " ")
-                        .split(/\s+/)
-                        .filter((w) => w.length > 3)
-                        .slice(0, 8)
-                        .join(" ");
-                    const siteQuery = extraTerms
-                        ? `${sitePrefix} ${extraTerms} ${keyTerms}`
-                        : `${sitePrefix} ${keyTerms}`;
-                    const siteResult = await webSearchHandler({
-                        query: siteQuery,
-                        maxResults: 3,
-                        provider: "auto",
-                    });
-                    const siteUrls = Array.isArray(siteResult?.results)
-                        ? siteResult.results
-                            .map((r) => String(r?.url ?? "").trim())
-                            .filter((u) => u.startsWith("http") && !urls.includes(u))
-                            .slice(0, 2)
-                        : [];
-                    urls.push(...siteUrls);
-                }
-                catch {
-                    // Continue
-                }
-                break; // Only do one site-specific search
-            }
-        }
-        // Step 2c: For grant/award questions mentioning papers, add a direct paper search
-        // to bypass the blog→paper hop (which is fragile due to search non-determinism).
-        const needsPaper = (promptLower.includes("award") || promptLower.includes("grant")) &&
-            (promptLower.includes("paper") || promptLower.includes("article"));
-        if (needsPaper) {
-            try {
-                const paperQueryContents = [
-                    {
-                        role: "user",
-                        parts: [
-                            {
-                                text: "From this question, extract the key details about the scientific paper mentioned. " +
-                                    "Generate a search query that would find the paper directly on a scholarly database " +
-                                    "(e.g., IOPscience, arXiv, Nature, NASA ADS). Include author names, topic, and year. " +
-                                    "Return ONLY the search query, nothing else.\n\n" +
-                                    `QUESTION:\n${task.prompt}`,
-                            },
-                        ],
-                    },
-                ];
-                const paperQuery = await llmGenerateText(llm, paperQueryContents);
-                if (paperQuery && paperQuery.length > 5 && paperQuery.length < 300) {
-                    const paperResult = await webSearchHandler({
-                        query: paperQuery,
-                        maxResults: 5,
-                        provider: "auto",
-                    });
-                    const paperUrls = Array.isArray(paperResult?.results)
-                        ? paperResult.results
-                            .map((r) => String(r?.url ?? "").trim())
-                            .filter((u) => u.startsWith("http") && !urls.includes(u) &&
-                            (u.includes("doi.org") || u.includes("iopscience") || u.includes("arxiv") ||
-                                u.includes("nature.com/articles") || u.includes("adsabs") ||
-                                u.includes("journals.aas.org") || u.includes("science.org")))
-                            .slice(0, 2)
-                        : [];
-                    urls.push(...paperUrls);
-                    // Also do an explicit arxiv search — arxiv has full text with acknowledgments
-                    if (paperUrls.length === 0 || !paperUrls.some((u) => u.includes("arxiv"))) {
-                        try {
-                            const arxivResult = await webSearchHandler({
-                                query: `site:arxiv.org ${paperQuery}`,
-                                maxResults: 3,
-                                provider: "auto",
-                            });
-                            const arxivUrls = Array.isArray(arxivResult?.results)
-                                ? arxivResult.results
-                                    .map((r) => String(r?.url ?? "").trim())
-                                    .filter((u) => u.startsWith("http") && u.includes("arxiv") && !urls.includes(u))
-                                    .slice(0, 2)
-                                : [];
-                            urls.push(...arxivUrls);
-                        }
-                        catch { /* continue */ }
-                    }
-                }
-            }
-            catch {
-                // Continue
-            }
-        }
-        // Step 2d: For arxiv abs URLs, also include the HTML version (full text with acknowledgments)
-        const extraArxivUrls = [];
-        for (const u of urls) {
-            if (u.includes("arxiv.org/abs/")) {
-                const htmlUrl = u.replace("/abs/", "/html/");
-                if (!urls.includes(htmlUrl) && !extraArxivUrls.includes(htmlUrl)) {
-                    extraArxivUrls.push(htmlUrl);
-                }
-            }
-        }
-        urls.push(...extraArxivUrls);
-        // Step 3: Fetch top URLs (cap at 7 to allow arxiv variants)
-        const fetchUrls = urls.slice(0, 7);
-        const fetched = [];
-        for (const url of fetchUrls) {
-            try {
-                // Use larger maxLength for scholarly URLs that may contain acknowledgments/funding sections
-                // arxiv HTML papers need extra space — acknowledgments are at the very end
-                const isArxivHtml = url.includes("arxiv.org/html/");
-                const isScholarlyUrl = url.includes("arxiv") || url.includes("doi.org") || url.includes("iopscience") ||
-                    url.includes("nature.com/articles") || url.includes("science.org") ||
-                    url.includes("journals.aas.org") || url.includes("adsabs");
-                fetched.push(await fetchUrlHandler({
-                    url,
-                    extractMode: "markdown",
-                    maxLength: isArxivHtml ? 200000 : isScholarlyUrl ? 48000 : 16000,
-                }));
-            }
-            catch {
-                fetched.push({ content: "", title: "" });
-            }
-        }
-        // Step 4: Aggressively follow linked URLs from fetched content
-        const followUpUrls = [];
-        for (const item of fetched) {
-            const content = String(item?.content ?? "");
-            const urlMatches = content.match(/https?:\/\/[^\s)\]>"']+/g) ?? [];
-            for (const foundUrl of urlMatches) {
-                const cleanUrl = foundUrl.replace(/[.,;:!?)]+$/, "");
-                if (fetchUrls.includes(cleanUrl) || followUpUrls.includes(cleanUrl))
-                    continue;
-                // Broadly follow links to authoritative sources
-                const isScholarly = cleanUrl.includes("arxiv") ||
-                    cleanUrl.includes("doi.org") ||
-                    cleanUrl.includes("iopscience") ||
-                    cleanUrl.includes("nature.com/articles") ||
-                    cleanUrl.includes("science.org") ||
-                    cleanUrl.includes("springer.com") ||
-                    cleanUrl.includes("adsabs.harvard.edu") ||
-                    cleanUrl.includes("journals.aas.org") ||
-                    cleanUrl.includes("academic.oup.com") ||
-                    cleanUrl.includes("agupubs.onlinelibrary.wiley.com");
-                const isGov = cleanUrl.includes("nasa.gov") ||
-                    cleanUrl.includes("usgs.gov") ||
-                    cleanUrl.includes(".gov/");
-                const isRelevant = 
-                // Paper/article references
-                (promptLower.includes("paper") && (isScholarly || isGov)) ||
-                    (promptLower.includes("article") && (isScholarly || cleanUrl.includes("nature.com"))) ||
-                    // Database references
-                    (promptLower.includes("database") && isGov) ||
-                    // Award/grant references — follow any scholarly/gov/DOI link
-                    ((promptLower.includes("award") || promptLower.includes("grant")) &&
-                        (isGov || isScholarly || cleanUrl.includes("grant") || cleanUrl.includes("doi.org"))) ||
-                    // NASA-related questions
-                    (promptLower.includes("nasa") && isGov) ||
-                    // Blog/news → follow scholarly + gov links
-                    ((promptLower.includes("universe today") ||
-                        promptLower.includes("blog") ||
-                        promptLower.includes("published in") ||
-                        promptLower.includes("published on")) &&
-                        (isScholarly || isGov));
-                if (isRelevant) {
-                    followUpUrls.push(cleanUrl);
-                    if (followUpUrls.length >= 5)
-                        break;
-                }
-            }
-        }
-        // Fetch follow-up URLs — use larger maxLength for scholarly/paper links to capture acknowledgments
-        const allFetchedUrls = [...fetchUrls];
-        for (const url of followUpUrls) {
-            try {
-                const isArxivHtml = url.includes("arxiv.org/html/");
-                const isScholarlyUrl = url.includes("arxiv") || url.includes("doi.org") || url.includes("iopscience") ||
-                    url.includes("nature.com/articles") || url.includes("science.org") ||
-                    url.includes("springer.com") || url.includes("nasa.gov") ||
-                    url.includes("journals.aas.org") || url.includes("adsabs.harvard.edu");
-                fetched.push(await fetchUrlHandler({
-                    url,
-                    extractMode: "markdown",
-                    maxLength: isArxivHtml ? 200000 : isScholarlyUrl ? 48000 : 16000,
-                }));
-                allFetchedUrls.push(url);
-            }
-            catch {
-                // Skip failed fetches
-            }
-        }
-        // For scholarly follow-ups, include more content in the source block
-        const sourcesBlock = allFetchedUrls
-            .map((u, i) => {
-            const item = fetched[i];
-            const title = String(item?.title ?? "").trim();
-            const isScholarlySource = u.includes("arxiv") || u.includes("doi.org") || u.includes("iopscience") ||
-                u.includes("nature.com/articles") || u.includes("science.org") ||
-                u.includes("journals.aas.org") || u.includes("nasa.gov");
-            const rawContent = String(item?.content ?? "");
-            // For long scholarly content: extract the beginning + acknowledgments/funding section
-            let content;
-            if (isScholarlySource && rawContent.length > 30000) {
-                const beginning = rawContent.slice(0, 10000);
-                // Search for acknowledgments, funding, or notes sections near the end
-                const ackPatterns = [
-                    /#{1,4}\s*Acknowledg/i, /#{1,4}\s*Funding/i, /#{1,4}\s*Notes/i,
-                    /\*\*Acknowledg/i, /\*\*Funding/i,
-                    /\bAcknowledg(?:e)?ments?\b/i, /\bFunding\b/i,
-                ];
-                let ackStart = -1;
-                for (const pat of ackPatterns) {
-                    const idx = rawContent.search(pat);
-                    if (idx > 0 && (ackStart === -1 || idx < ackStart))
-                        ackStart = idx;
-                }
-                if (ackStart > 0) {
-                    const ackSection = rawContent.slice(Math.max(0, ackStart - 200), ackStart + 20000);
-                    content = beginning + "\n\n[...MIDDLE OF PAPER OMITTED...]\n\n" + ackSection;
-                }
-                else {
-                    // No ack section found — try the end of the paper
-                    content = beginning + "\n\n[...MIDDLE OF PAPER OMITTED...]\n\n" + rawContent.slice(-20000);
-                }
-            }
-            else {
-                content = rawContent.slice(0, isScholarlySource ? 30000 : 10000);
-            }
-            return [`SOURCE ${i + 1}: ${title || u}`, `URL: ${u}`, `CONTENT:\n${content}`].join("\n");
-        })
-            .join("\n\n");
-        // Step 5: Final answer — always use Gemini with code execution when available
-        // This gives the model the OPTION to write code for math tasks while also
-        // providing consistent, high-quality answers for all tasks.
-        if (process.env.GEMINI_API_KEY) {
-            try {
-                const mod = await import("@google/genai");
-                const { GoogleGenAI } = mod;
-                let gemModel = process.env.NODEBENCH_GAIA_TOOLS_MODEL ?? "gemini-3-flash-preview";
-                if (gemModel.includes(":"))
-                    gemModel = gemModel.split(":").pop();
-                const ai = new GoogleGenAI({ apiKey: process.env.GEMINI_API_KEY });
-                // Detect if question asks for a specific identifier
-                const asksForId = promptLower.includes("grant") || promptLower.includes("award") ||
-                    promptLower.includes("identifier") || promptLower.includes("number") ||
-                    promptLower.includes("code") || promptLower.includes("id ");
-                // Scan all fetched content for NASA grant numbers
-                const allFetchedText = fetched.map((f) => String(f?.content ?? "")).join("\n");
-                const foundGrants = extractNasaGrantNumbers(allFetchedText);
-                const grantHint = (asksForId && foundGrants.length > 0)
-                    ? `\nNASA GRANT NUMBERS FOUND IN SOURCES: ${foundGrants.join(", ")}\nIf the question asks for a grant/award number, one of these is likely the answer.`
-                    : "";
-                const codeExecPrompt = [
-                    "Answer the question using the provided sources AND your knowledge.",
-                    ...(opts.baselineHint
-                        ? [
-                            `IMPORTANT: Your preliminary answer (without web search) was: "${opts.baselineHint}"`,
-                            "Your task is to VERIFY this answer using the web sources.",
-                            "ONLY change your preliminary answer if the sources provide CLEAR, DIRECT, UNAMBIGUOUS evidence that it is wrong.",
-                            "If the sources don't directly address the exact question, give conflicting numbers, or seem unreliable, KEEP your preliminary answer.",
-                            "Your training data is often more reliable than noisy web search results.",
-                        ]
-                        : []),
-                    ...(needsMath
-                        ? [
-                            "This question requires counting, math, or data analysis.",
-                            "Write Python code to compute the answer precisely from the source data.",
-                        ]
-                        : [
-                            "If the answer requires any counting, math, or data lookup, write Python code to compute it precisely.",
-                        ]),
-                    "If the question asks about a specific identifier (grant number, ID, code), extract it directly from the sources.",
-                    ...(asksForId
-                        ? [
-                            "IMPORTANT: Look in 'Acknowledgments', 'Acknowledgements', 'Funding', and 'Notes' sections of papers.",
-                            "NASA grant numbers follow patterns like: 80GSFC..., 80NSSC..., NNX..., NNG..., NNH..., NAS...",
-                            "Extract the EXACT identifier string — do not paraphrase or summarize it.",
-                        ]
-                        : []),
-                    "",
-                    "Return ONLY the final answer, no explanation.",
-                    "",
-                    `QUESTION:\n${task.prompt}`,
-                    ...(grantHint ? [grantHint] : []),
-                    "",
-                    sourcesBlock || "NO_SOURCES_FOUND",
-                ].join("\n");
-                const response = await ai.models.generateContent({
-                    model: gemModel,
-                    contents: [{ role: "user", parts: [{ text: codeExecPrompt }] }],
-                    config: {
-                        tools: [{ codeExecution: {} }],
-                        temperature: 0,
-                        maxOutputTokens: 4096,
-                    },
-                });
-                const parts = response?.candidates?.[0]?.content?.parts ?? [];
-                // Prefer code execution output
-                const codeExecParts = parts.filter((p) => p.codeExecutionResult);
-                if (codeExecParts.length > 0) {
-                    const output = String(codeExecParts[codeExecParts.length - 1].codeExecutionResult?.output ?? "").trim();
-                    const lines = output.split("\n").map((l) => l.trim()).filter(Boolean);
-                    if (lines.length > 0) {
-                        return { answer: lines[lines.length - 1], toolCalls: 1 + allFetchedUrls.length };
-                    }
-                }
-                const textAnswer = parts.map((p) => p?.text ?? "").join("").trim();
-                if (textAnswer) {
-                    return { answer: textAnswer, toolCalls: 1 + allFetchedUrls.length };
-                }
-            }
-            catch {
-                // Fall through to standard LLM answer
-            }
-        }
-        // Fallback: Standard LLM answer (when no Gemini API key)
-        const contents = [
-            {
-                role: "user",
-                parts: [
-                    {
-                        text: "Answer the question using ONLY the provided sources. " +
-                            "If the sources are insufficient, make the best supported guess.\n\n" +
-                            "Return ONLY the final answer, no explanation.\n\n" +
-                            `TASK_ID: ${task.id}\nQUESTION:\n${task.prompt}\n\n` +
-                            (sourcesBlock ? sourcesBlock : "NO_SOURCES_FOUND"),
-                    },
-                ],
-            },
-        ];
-        const answer = await llmGenerateText(llm, contents);
-        return { answer, toolCalls: 1 + allFetchedUrls.length };
-    }
-    const toolUsageSummary = [
-        "You have access to tools:",
-        "- web_search({query,maxResults,provider})",
-        "- fetch_url({url,extractMode,maxLength})",
-        "",
-        "When using tools, respond with a single JSON object only:",
-        `{"action":"tool","name":"web_search","arguments":{"query":"...","maxResults":5}}`,
-        "When done, respond with:",
-        `{"action":"final","answer":"..."}`,
-        "",
-        "Rules:",
-        "- ALWAYS start with web_search to find relevant sources.",
-        "- After search, use fetch_url to read the most promising result pages.",
-        "- Do NOT answer based only on snippets; fetch_url and extract the exact value when possible.",
-        "- If a page mentions a linked resource (paper, database entry, article), fetch that linked URL too.",
-        "- If the question requires counting/math, do the calculation explicitly before answering.",
-        "- If the question asks about a database (USGS, etc.), search for the specific database and try to access its query results directly.",
-        "- If the question involves finding a linked paper from an article, fetch the article first, then follow the paper link.",
-        "- If the question specifies a timeframe (e.g. 'as of end of 2022'), prioritize archival sources.",
-        "- Keep tool arguments small (maxResults<=5, maxLength<=16000).",
-        "- Do NOT include any explanation. Final answer must match the requested formatting.",
-    ].join("\n");
-    const contents = [
-        {
-            role: "user",
-            parts: [
-                {
-                    text: `${toolUsageSummary}\n\nTASK_ID: ${task.id}\nQUESTION:\n${task.prompt}`,
-                },
-            ],
-        },
-    ];
-    let toolCalls = 0;
-    let usedWebSearch = false;
-    let usedFetchUrl = false;
-    for (let step = 0; step < opts.maxSteps; step++) {
-        const out = await llmGenerateText(llm, contents);
-        contents.push({ role: "model", parts: [{ text: out }] });
-        const parsed = extractJsonObject(out);
-        if (!parsed || typeof parsed !== "object") {
-            // Ask the model to restate as JSON only.
-            contents.push({
-                role: "user",
-                parts: [{ text: "Invalid format. Return JSON only with action tool|final." }],
-            });
-            continue;
-        }
-        if (parsed.action === "final") {
-            if (forceWebSearch && !usedWebSearch) {
-                contents.push({
-                    role: "user",
-                    parts: [{ text: "Before answering, you MUST call web_search at least once. Continue." }],
-                });
-                continue;
-            }
-            if (forceFetchUrl && !usedFetchUrl) {
-                contents.push({
-                    role: "user",
-                    parts: [{ text: "Before answering, you MUST call fetch_url at least once. Continue." }],
-                });
-                continue;
-            }
-            const answer = String(parsed.answer ?? "").trim();
-            return { answer, toolCalls };
-        }
-        if (parsed.action !== "tool") {
-            contents.push({
-                role: "user",
-                parts: [{ text: "Invalid action. Return JSON only with action tool|final." }],
-            });
-            continue;
-        }
-        if (toolCalls >= opts.maxToolCalls) {
-            contents.push({
-                role: "user",
-                parts: [{ text: "Tool call budget exceeded. Return final answer now." }],
-            });
-            continue;
-        }
-        const name = String(parsed.name ?? "");
-        const tool = toolIndex.get(name);
-        if (!tool) {
-            contents.push({
-                role: "user",
-                parts: [{ text: `Unknown tool "${name}". Use only web_search or fetch_url.` }],
-            });
-            continue;
-        }
-        const args = (parsed.arguments ?? {});
-        // Hard limits for safety.
-        if (name === "web_search") {
-            if (typeof args.maxResults !== "number")
-                args.maxResults = 5;
-            args.maxResults = Math.min(Number(args.maxResults) || 5, 5);
-            if (!args.provider)
-                args.provider = "auto";
-        }
-        else if (name === "fetch_url") {
-            if (!args.extractMode)
-                args.extractMode = "markdown";
-            if (typeof args.maxLength !== "number")
-                args.maxLength = 16000;
-            args.maxLength = Math.min(Number(args.maxLength) || 16000, 16000);
-        }
-        toolCalls++;
-        if (name === "web_search")
-            usedWebSearch = true;
-        if (name === "fetch_url")
-            usedFetchUrl = true;
-        const toolResult = await tool.handler(args);
-        // Provide a bounded JSON summary to the model. Avoid dumping large content.
-        const toolResultText = JSON.stringify(toolResult).slice(0, 16000);
-        contents.push({
-            role: "user",
-            parts: [
-                {
-                    text: `TOOL_RESULT ${name}:\n${toolResultText}\n\nContinue. Return JSON only.`,
-                },
-            ],
-        });
-    }
-    // If we ran out of steps, force a final answer.
-    contents.push({
-        role: "user",
-        parts: [{ text: "Out of steps. Return final answer now as JSON." }],
-    });
-    const out = await llmGenerateText(llm, contents);
-    const parsed = extractJsonObject(out);
-    const answer = parsed && parsed.action === "final" ? String(parsed.answer ?? "").trim() : out.trim();
-    return { answer, toolCalls };
-}
-/**
- * Enhanced RAG with Gemini code execution for web tasks.
- * Uses multi-query search, aggressive link following, and Gemini's built-in
- * codeExecution so the model can write Python for math/counting tasks.
- * (Gemini 3 preview doesn't support functionDeclarations, so we orchestrate
- * tool calls ourselves and let the model reason with code execution.)
- */
-async function toolAugmentedAnswerNativeFC(task, opts) {
-    // Pre-check: deterministic solvers
-    const caesarAnswer = tryCaesarCipherSolve(task);
-    if (caesarAnswer)
-        return { answer: caesarAnswer, toolCalls: 0 };
-    const apiKey = process.env.GEMINI_API_KEY;
-    if (!apiKey)
-        throw new Error("GEMINI_API_KEY required");
-    let model = process.env.NODEBENCH_GAIA_TOOLS_MODEL ?? "gemini-3-flash-preview";
-    if (model.includes(":"))
-        model = model.split(":").pop();
-    const toolIndex = buildToolIndex();
-    const webSearch = toolIndex.get("web_search");
-    const fetchUrl = toolIndex.get("fetch_url");
-    if (!webSearch || !fetchUrl)
-        throw new Error("Missing web_search/fetch_url tools");
-    const mod = await import("@google/genai");
-    const { GoogleGenAI } = mod;
-    const ai = new GoogleGenAI({ apiKey });
-    // Helper: generate text with Gemini, optionally with code execution
-    async function geminiGenerate(prompt, genOpts) {
-        const config = {
-            temperature: 0,
-            maxOutputTokens: genOpts?.maxOutputTokens ?? 4096,
-        };
-        if (genOpts?.codeExecution)
-            config.tools = [{ codeExecution: {} }];
-        const response = await ai.models.generateContent({
-            model,
-            contents: [{ role: "user", parts: [{ text: prompt }] }],
-            config,
-        });
-        const parts = response?.candidates?.[0]?.content?.parts ?? [];
-        // Prefer code execution output if available
-        const codeExecParts = parts.filter((p) => p.codeExecutionResult);
-        if (codeExecParts.length > 0) {
-            const output = String(codeExecParts[codeExecParts.length - 1].codeExecutionResult?.output ?? "").trim();
-            const lines = output.split("\n").map((l) => l.trim()).filter(Boolean);
-            if (lines.length > 0)
-                return lines[lines.length - 1];
-        }
-        return parts.map((p) => p?.text ?? "").join("").trim();
-    }
-    let toolCalls = 0;
-    const promptLower = task.prompt.toLowerCase();
-    // Detect if the task involves math/counting/computation
-    const needsMath = promptLower.includes("how many") ||
-        promptLower.includes("calculate") ||
-        promptLower.includes("compute") ||
-        promptLower.includes("p-value") ||
-        promptLower.includes("incorrect") ||
-        promptLower.includes("percentage") ||
-        /\d+.*\d+/.test(task.prompt);
-    // Step 1: Generate two search queries — one direct, one from a different angle
-    let searchQueries = [];
-    try {
-        const queryPrompt = [
-            "Generate exactly 2 web search queries to find the answer to this question.",
-            "Query 1: A concise, direct query with key names, dates, and specific terms.",
-            "Query 2: A different-angle query targeting the underlying source (paper, database, official page, grant).",
-            "Return exactly 2 lines, one query per line, nothing else.",
-            "",
-            `QUESTION:\n${task.prompt}`,
-        ].join("\n");
-        const queryText = await geminiGenerate(queryPrompt, { maxOutputTokens: 512 });
-        searchQueries = queryText
-            .split("\n")
-            .map((q) => q
-            .replace(/^\d+[\.\)]\s*/, "")
-            .replace(/^(Query \d+:\s*)/i, "")
-            .replace(/^["']|["']$/g, "")
-            .trim())
-            .filter((q) => q.length > 5 && q.length < 300);
-    }
-    catch {
-        // Fall through
-    }
-    if (searchQueries.length === 0)
-        searchQueries = [task.prompt];
-    searchQueries = searchQueries.slice(0, 2);
-    // Step 2: Search with both queries
-    const allUrls = [];
-    for (const query of searchQueries) {
-        try {
-            const result = await webSearch.handler({
-                query,
-                maxResults: 5,
-                provider: "auto",
-            });
-            toolCalls++;
-            const results = Array.isArray(result?.results) ? result.results : [];
-            for (const r of results) {
-                const url = String(r?.url ?? "").trim();
-                if (url.startsWith("http") && !allUrls.includes(url)) {
-                    allUrls.push(url);
-                }
-            }
-        }
-        catch {
-            // Continue
-        }
-    }
-    // Step 3: Fetch top 4 URLs
-    const fetchLimit = Math.min(allUrls.length, 4);
-    const fetchedContent = [];
-    for (let i = 0; i < fetchLimit; i++) {
-        try {
-            const result = await fetchUrl.handler({
-                url: allUrls[i],
-                extractMode: "markdown",
-                maxLength: 16000,
-            });
-            toolCalls++;
-            fetchedContent.push({
-                url: allUrls[i],
-                title: String(result?.title ?? ""),
-                content: String(result?.content ?? "").slice(0, 12000),
-            });
-        }
-        catch {
-            // Skip failed fetches
-        }
-    }
-    // Step 4: Extract and follow relevant linked URLs from fetched content
-    const followUpUrls = [];
-    for (const item of fetchedContent) {
-        const urlMatches = item.content.match(/https?:\/\/[^\s)\]>"']+/g) ?? [];
-        for (const foundUrl of urlMatches) {
-            const cleanUrl = foundUrl.replace(/[.,;:!?)]+$/, "");
-            if (allUrls.includes(cleanUrl) || followUpUrls.includes(cleanUrl))
-                continue;
-            // Broadly follow links to authoritative sources
-            const isScholarly = cleanUrl.includes("arxiv") ||
-                cleanUrl.includes("doi.org") ||
-                cleanUrl.includes("iopscience") ||
-                cleanUrl.includes("nature.com/articles") ||
-                cleanUrl.includes("science.org") ||
-                cleanUrl.includes("springer.com");
-            const isGov = cleanUrl.includes("nasa.gov") ||
-                cleanUrl.includes("usgs.gov") ||
-                cleanUrl.includes(".gov/");
-            const isRelevant = (promptLower.includes("paper") && (isScholarly || isGov)) ||
-                (promptLower.includes("database") && isGov) ||
-                (promptLower.includes("article") && (isScholarly || cleanUrl.includes("nature.com"))) ||
-                (promptLower.includes("award") && (isGov || cleanUrl.includes("grant"))) ||
-                (promptLower.includes("nasa") && isGov) ||
-                // Any question mentioning a website/blog — follow scholarly + gov links found in content
-                ((promptLower.includes("universe today") ||
-                    promptLower.includes("blog") ||
-                    promptLower.includes("published")) &&
-                    (isScholarly || isGov));
-            if (isRelevant) {
-                followUpUrls.push(cleanUrl);
-                if (followUpUrls.length >= 3)
-                    break;
-            }
-        }
-    }
-    for (const url of followUpUrls) {
-        try {
-            const result = await fetchUrl.handler({
-                url,
-                extractMode: "markdown",
-                maxLength: 16000,
-            });
-            toolCalls++;
-            fetchedContent.push({
-                url,
-                title: String(result?.title ?? ""),
-                content: String(result?.content ?? "").slice(0, 12000),
-            });
-        }
-        catch {
-            // Skip
-        }
-    }
-    // Step 5: Final answer — use code execution only when math is needed
-    const sourcesBlock = fetchedContent
-        .map((item, i) => `SOURCE ${i + 1}: ${item.title || item.url}\nURL: ${item.url}\nCONTENT:\n${item.content}`)
-        .join("\n\n");
-    const answerPrompt = [
-        "Answer the question using ONLY the provided sources.",
-        ...(needsMath
-            ? [
-                "This question requires precise computation. Write Python code to calculate the answer.",
-                "Parse the relevant data from the sources and compute the result programmatically.",
-            ]
-            : []),
-        "If the sources are insufficient, make the best supported guess.",
-        "",
-        "Return ONLY the final answer, no explanation.",
-        "",
-        `QUESTION:\n${task.prompt}`,
-        "",
-        sourcesBlock || "NO_SOURCES_FOUND",
-    ].join("\n");
-    const answer = await geminiGenerate(answerPrompt, { codeExecution: needsMath });
-    return { answer, toolCalls };
-}
-async function loadFixture(fixturePath) {
-    const raw = await readFile(fixturePath, "utf8");
-    const parsed = JSON.parse(raw);
-    if (!parsed || !Array.isArray(parsed.tasks))
-        throw new Error("Invalid GAIA capability fixture");
-    return parsed;
-}
-describe("Capability: GAIA accuracy (LLM-only vs LLM+tools)", () => {
-    const testFn = shouldRun ? it : it.skip;
-    testFn("should measure accuracy delta on a small GAIA subset", async () => {
-        loadDotEnvLocalIfPresent();
-        const fixturePath = resolveCapabilityFixturePath();
-        if (!existsSync(fixturePath)) {
-            throw new Error(`Missing GAIA capability fixture at ${fixturePath}. Generate it with: python packages/mcp-local/src/__tests__/fixtures/generateGaiaCapabilityFixture.py`);
-        }
-        const baselineModel = process.env.NODEBENCH_GAIA_BASELINE_MODEL ?? "gemini-3-flash-preview";
-        const toolsModel = process.env.NODEBENCH_GAIA_TOOLS_MODEL ?? baselineModel;
-        const baselineLlm = await createTextLlmClient({ model: baselineModel });
-        const toolsLlm = await createTextLlmClient({ model: toolsModel });
-        const baselineModelLabel = `${baselineLlm.provider}:${baselineLlm.model}`;
-        const toolsModelLabel = `${toolsLlm.provider}:${toolsLlm.model}`;
-        const fixture = await loadFixture(fixturePath);
-        expect(Array.isArray(fixture.tasks)).toBe(true);
-        expect(fixture.tasks.length).toBeGreaterThan(0);
-        const requestedLimit = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_TASK_LIMIT ?? "6", 10);
-        const taskLimit = Math.max(1, Math.min(fixture.tasks.length, Number.isFinite(requestedLimit) ? requestedLimit : 6));
-        const tasks = fixture.tasks.slice(0, taskLimit);
-        const requestedConcurrency = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_CONCURRENCY ?? "1", 10);
-        const concurrency = Math.max(1, Math.min(tasks.length, Number.isFinite(requestedConcurrency) ? requestedConcurrency : 1));
-        const maxSteps = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_MAX_STEPS ?? "10", 10);
-        const maxToolCalls = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_MAX_TOOL_CALLS ?? "8", 10);
-        // Auto-discover judge: free OpenRouter → paid LLM → deterministic-only
-        const useJudge = process.env.NODEBENCH_GAIA_JUDGE !== "0";
-        const judge = useJudge ? await autoDiscoverJudge(toolsLlm) : null;
-        if (judge) {
-            console.log(`[gaia-capability] judge: ${judge.provider}:${judge.model}`);
-        }
-        const results = new Array(tasks.length);
-        let nextIndex = 0;
-        const workers = Array.from({ length: concurrency }, () => (async () => {
-            while (true) {
-                const idx = nextIndex++;
-                if (idx >= tasks.length)
-                    return;
-                const task = tasks[idx];
-                try {
-                    const baseStart = performance.now();
-                    const base = await baselineAnswer(baselineLlm, task);
-                    const baseMs = performance.now() - baseStart;
-                    const toolsStart = performance.now();
-                    const toolsMode = (process.env.NODEBENCH_GAIA_CAPABILITY_TOOLS_MODE ?? "rag").toLowerCase();
-                    const tools = toolsMode === "enhanced"
-                        ? await toolAugmentedAnswerNativeFC(task, { maxSteps, maxToolCalls })
-                        : await toolAugmentedAnswer(toolsLlm, task, { maxSteps, maxToolCalls, baselineHint: base });
-                    const toolsMs = performance.now() - toolsStart;
-                    const baseJudge = await answersMatchWithJudge(task.expectedAnswer, base, judge);
-                    const toolsJudge = await answersMatchWithJudge(task.expectedAnswer, tools.answer, judge);
-                    results[idx] = {
-                        taskId: task.id,
-                        baselineCorrect: baseJudge.match,
-                        toolsCorrect: toolsJudge.match,
-                        baselineMs: baseMs,
-                        toolsMs,
-                        toolCalls: tools.toolCalls,
-                        judgeProvider: toolsJudge.judgeProvider,
-                        judgeInvoked: toolsJudge.judgeInvoked,
-                    };
-                }
-                catch (err) {
-                    console.error(`[gaia-capability] ERROR task=${task.id}: ${err?.message ?? String(err)}`);
-                    if (err?.stack)
-                        console.error(err.stack);
-                    results[idx] = {
-                        taskId: task.id,
-                        baselineCorrect: false,
-                        toolsCorrect: false,
-                        baselineMs: 0,
-                        toolsMs: 0,
-                        toolCalls: 0,
-                        error: err?.message ?? String(err),
-                    };
-                }
-            }
-        })());
-        await Promise.all(workers);
-        const baselineCorrect = results.filter((r) => r.baselineCorrect).length;
-        const toolsCorrect = results.filter((r) => r.toolsCorrect).length;
-        const improved = results.filter((r) => !r.baselineCorrect && r.toolsCorrect).length;
-        const regressions = results.filter((r) => r.baselineCorrect && !r.toolsCorrect).length;
-        const avg = (values) => values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;
-        const avgBaseMs = avg(results.map((r) => r.baselineMs).filter((n) => n > 0));
-        const avgToolsMs = avg(results.map((r) => r.toolsMs).filter((n) => n > 0));
-        const avgToolCalls = avg(results.map((r) => r.toolCalls));
-        console.log(`[gaia-capability] config=${fixture.config} split=${fixture.split} tasks=${tasks.length} concurrency=${concurrency} baseline=${baselineCorrect}/${tasks.length} tools=${toolsCorrect}/${tasks.length} improved=${improved} regressions=${regressions} avgBaselineMs=${avgBaseMs.toFixed(0)} avgToolsMs=${avgToolsMs.toFixed(0)} avgToolCalls=${avgToolCalls.toFixed(2)}`);
-        console.log(`[gaia-capability] perTask: ${results
-            .map((r) => `${r.taskId}:B${r.baselineCorrect ? "1" : "0"}T${r.toolsCorrect ? "1" : "0"}${r.error ? "E" : ""}`)
-            .join(" ")}`);
-        if (shouldWriteReport) {
-            const repoRoot = resolveRepoRoot();
-            const generatedAtIso = new Date().toISOString();
-            const stamp = generatedAtIso.replace(/[:.]/g, "-");
-            const toolsMode = (process.env.NODEBENCH_GAIA_CAPABILITY_TOOLS_MODE ?? "rag").toLowerCase();
-            const publicSummary = {
-                suiteId: "gaia_capability",
-                lane: "web",
-                generatedAtIso,
-                config: fixture.config,
-                split: fixture.split,
-                taskCount: tasks.length,
-                concurrency,
-                baseline: {
-                    model: baselineModelLabel,
-                    correct: baselineCorrect,
-                    passRatePct: tasks.length === 0 ? 0 : (baselineCorrect / tasks.length) * 100,
-                    avgMs: avgBaseMs,
-                },
-                tools: {
-                    model: toolsModelLabel,
-                    mode: toolsMode,
-                    correct: toolsCorrect,
-                    passRatePct: tasks.length === 0 ? 0 : (toolsCorrect / tasks.length) * 100,
-                    avgMs: avgToolsMs,
-                    avgToolCalls: avgToolCalls,
-                },
-                improved,
-                regressions,
-                notes: "GAIA is gated. This file contains only aggregate metrics (no prompt/answer text). Detailed per-task report is written under .cache/gaia/reports (gitignored).",
-            };
-            await safeWriteJson(path.join(repoRoot, "public", "evals", "gaia_capability_latest.json"), publicSummary);
-            await safeWriteJson(path.join(repoRoot, ".cache", "gaia", "reports", `gaia_capability_${fixture.config}_${fixture.split}_${stamp}.json`), {
-                ...publicSummary,
-                perTask: results.map((r) => ({
-                    taskId: r.taskId,
-                    baselineCorrect: r.baselineCorrect,
-                    toolsCorrect: r.toolsCorrect,
-                    baselineMs: r.baselineMs,
-                    toolsMs: r.toolsMs,
-                    toolCalls: r.toolCalls,
-                    error: r.error ?? null,
-                })),
-            });
-        }
-        // Save web cache if recording
-        const cacheMode = (process.env.NODEBENCH_GAIA_WEB_CACHE ?? "").toLowerCase();
-        if (cacheMode === "record" || cacheMode === "replay") {
-            await saveWebCache();
-            console.log(`[gaia-capability] web cache saved (mode=${cacheMode})`);
-        }
-        // By default this benchmark is informational and should not fail CI.
-        // Set NODEBENCH_GAIA_CAPABILITY_ENFORCE=1 to turn the summary into a strict gate.
-        const enforce = process.env.NODEBENCH_GAIA_CAPABILITY_ENFORCE === "1";
-        if (enforce) {
-            // Quality gate:
-            // - Tools should not regress massively vs baseline (allow a small tolerance for web drift).
-            // - Prefer at least one improvement so the run is measuring something tool-relevant.
-            const allowedRegression = Math.max(1, Math.floor(tasks.length * 0.2));
-            expect(improved).toBeGreaterThanOrEqual(1);
-            expect(toolsCorrect).toBeGreaterThanOrEqual(baselineCorrect - allowedRegression);
-            expect(toolsCorrect).toBeGreaterThanOrEqual(1);
-        }
-        else {
-            // Informational mode: ensure we actually ran and produced results.
-            expect(results.length).toBe(tasks.length);
-            expect(results.some((r) => r.error)).toBe(false);
-        }
-    }, 15 * 60_000);
-});
-//# sourceMappingURL=gaiaCapabilityEval.test.js.map