nodebench-mcp 2.70.0 → 3.0.1

package/dist/__tests__/gaiaCapabilityMediaEval.test.js

@@ -1,1101 +0,0 @@
-/**
- * GAIA media-backed capability/accuracy benchmark: LLM-only vs LLM+NodeBench MCP local OCR tools.
- *
- * This lane targets GAIA tasks that include image attachments (PNG/JPG/WEBP).
- * We provide deterministic local OCR via NodeBench MCP tools and score answers against
- * the ground-truth "Final answer" (stored locally under `.cache/gaia`, gitignored).
- *
- * Safety:
- * - GAIA is gated. Do not commit fixtures that contain prompts/answers.
- * - This test logs only task IDs and aggregate metrics (no prompt/answer text).
- *
- * Disabled by default (cost + rate limits). Run with:
- *   NODEBENCH_RUN_GAIA_CAPABILITY=1 npm --prefix packages/mcp-local run test
- */
-import { describe, expect, it } from "vitest";
-import { existsSync, readFileSync } from "node:fs";
-import { mkdir, readFile, writeFile } from "node:fs/promises";
-import path from "node:path";
-import { fileURLToPath } from "node:url";
-import { performance } from "node:perf_hooks";
-import { localFileTools, gaiaMediaSolvers } from "../tools/localFileTools.js";
-import { createTextLlmClient, generateTextFromHistory, } from "./helpers/textLlm.js";
-import { answersMatchWithJudge, autoDiscoverJudge } from "./helpers/answerMatch.js";
-const shouldRun = process.env.NODEBENCH_RUN_GAIA_CAPABILITY === "1";
-const shouldWriteReport = process.env.NODEBENCH_WRITE_GAIA_REPORT === "1";
-async function safeWriteJson(filePath, payload) {
-    try {
-        await mkdir(path.dirname(filePath), { recursive: true });
-        await writeFile(filePath, JSON.stringify(payload, null, 2) + "\n", "utf8");
-    }
-    catch (err) {
-        console.warn(`[gaia-capability-media] report write failed: ${err?.message ?? String(err)}`);
-    }
-}
-function resolveRepoRoot() {
-    const testDir = path.dirname(fileURLToPath(import.meta.url));
-    return path.resolve(testDir, "../../../..");
-}
-function resolveCapabilityMediaFixturePath() {
-    const override = process.env.NODEBENCH_GAIA_CAPABILITY_MEDIA_FIXTURE_PATH;
-    if (override) {
-        if (path.isAbsolute(override))
-            return override;
-        const repoRoot = resolveRepoRoot();
-        return path.resolve(repoRoot, override);
-    }
-    const config = process.env.NODEBENCH_GAIA_CAPABILITY_CONFIG ?? "2023_all";
-    const split = process.env.NODEBENCH_GAIA_CAPABILITY_SPLIT ?? "validation";
-    const repoRoot = resolveRepoRoot();
-    return path.join(repoRoot, ".cache", "gaia", `gaia_capability_media_${config}_${split}.sample.json`);
-}
-function loadDotEnvLocalIfPresent() {
-    const repoRoot = resolveRepoRoot();
-    const envPath = path.join(repoRoot, ".env.local");
-    if (!existsSync(envPath))
-        return;
-    const text = readFileSync(envPath, "utf8");
-    for (const rawLine of text.split(/\r?\n/)) {
-        const line = rawLine.trim();
-        if (!line || line.startsWith("#"))
-            continue;
-        const idx = line.indexOf("=");
-        if (idx <= 0)
-            continue;
-        const key = line.slice(0, idx).trim();
-        let value = line.slice(idx + 1).trim();
-        if ((value.startsWith("\"") && value.endsWith("\"")) ||
-            (value.startsWith("'") && value.endsWith("'"))) {
-            value = value.slice(1, -1);
-        }
-        if (!process.env[key])
-            process.env[key] = value;
-    }
-}
-function tryParseIntFromText(text, re) {
-    const m = String(text ?? "").match(re);
-    if (!m)
-        return null;
-    const n = Number.parseInt(m[1], 10);
-    return Number.isFinite(n) ? n : null;
-}
-function tryParseFloatFromText(text, re) {
-    const m = String(text ?? "").match(re);
-    if (!m)
-        return null;
-    const n = Number.parseFloat(m[1]);
-    return Number.isFinite(n) ? n : null;
-}
-async function tryDeterministicMediaSolve(toolIndex, task, localPath) {
-    const q = String(task.prompt ?? "").toLowerCase();
-    // Colored number grid -> stdev average
-    if (q.includes("population deviation") && q.includes("sample deviation") && q.includes("red") && q.includes("green")) {
-        const tool = toolIndex.get("solve_red_green_deviation_average_from_image");
-        if (!tool)
-            return null;
-        const result = await tool.handler({ path: localPath, decimals: 3 });
-        const answer = String(result?.answer ?? "").trim();
-        if (!answer)
-            return null;
-        return { answer, toolCalls: 1 };
-    }
-    // Fraction quiz grading -> integer score
-    if (q.includes("quiz is scored") && q.includes("bonus points")) {
-        const tool = toolIndex.get("grade_fraction_quiz_from_image");
-        if (!tool)
-            return null;
-        const bonus = tryParseIntFromText(task.prompt, /(\d+)\s+bonus\s+points?/i) ?? 0;
-        const ptsAddSub = tryParseIntFromText(task.prompt, /add\s+or\s+subtract\s+fractions?\s*:\s*(\d+)/i) ?? 5;
-        const ptsMulDiv = tryParseIntFromText(task.prompt, /multiply\s+or\s+divide\s+fractions?\s*:\s*(\d+)/i) ?? 10;
-        const ptsImproper = tryParseIntFromText(task.prompt, /improper\s+fraction\s*:\s*(\d+)/i) ?? 15;
-        const ptsMixed = tryParseIntFromText(task.prompt, /mixed\s+number\s*:\s*(\d+)/i) ?? 20;
-        const result = await tool.handler({
-            path: localPath,
-            bonusPoints: bonus,
-            pointsAddSubtract: ptsAddSub,
-            pointsMultiplyDivide: ptsMulDiv,
-            pointsImproperFraction: ptsImproper,
-            pointsMixedNumber: ptsMixed,
-            preprocess: true,
-            maxChars: 120000,
-        });
-        const answer = String(result?.answer ?? result?.score ?? "").trim();
-        if (!answer)
-            return null;
-        return { answer, toolCalls: 1 };
-    }
-    // Green polygon area from purple lengths
-    if (q.includes("area") && q.includes("green") && q.includes("polygon")) {
-        const tool = toolIndex.get("solve_green_polygon_area_from_image");
-        if (!tool)
-            return null;
-        const result = await tool.handler({ path: localPath });
-        const answer = String(result?.answer ?? "").trim();
-        if (!answer)
-            return null;
-        return { answer, toolCalls: 1 };
-    }
-    // Fractions in body text + sample simplifications -> comma-separated list
-    if (q.includes("comma separated") && q.includes("fractions") && q.includes("sample")) {
-        const tool = toolIndex.get("extract_fractions_and_simplify_from_image");
-        if (!tool)
-            return null;
-        const result = await tool.handler({ path: localPath, preprocess: true, maxChars: 180000, bodyBottomFrac: 0.75 });
-        const answer = String(result?.answer ?? "").trim();
-        if (!answer)
-            return null;
-        return { answer, toolCalls: 1 };
-    }
-    // Bass clef staff -> derived age
-    if (q.includes("bass clef")) {
-        const tool = toolIndex.get("solve_bass_clef_age_from_image");
-        if (!tool)
-            return null;
-        const result = await tool.handler({ path: localPath });
-        const answer = String(result?.answer ?? "").trim();
-        if (!answer)
-            return null;
-        return { answer, toolCalls: 1 };
-    }
-    // Pricing table storage upgrade -> x.xx
-    if (q.includes("uploaded") && q.includes("over the limit") && q.includes("upgrade")) {
-        const tool = toolIndex.get("solve_storage_upgrade_cost_per_file_from_image");
-        if (!tool)
-            return null;
-        const planM = task.prompt.match(/\b(Standard|Plus|Premium)\b/i);
-        const currentPlan = planM ? planM[1] : "Standard";
-        const filesUploaded = tryParseIntFromText(task.prompt, /uploaded\s+(\d+)\s+/i) ?? 0;
-        const overLimitGb = tryParseFloatFromText(task.prompt, /(\d+(?:\.\d+)?)\s*gb\s+over/i) ?? 0;
-        const additionalFiles = tryParseIntFromText(task.prompt, /(\d+)\s+more\s+files?/i) ?? 0;
-        if (filesUploaded > 0 && additionalFiles >= 0) {
-            const result = await tool.handler({
-                path: localPath,
-                currentPlanName: currentPlan,
-                filesUploaded,
-                overLimitGb,
-                additionalFiles,
-                decimals: 2,
-                preprocess: true,
-                maxChars: 80000,
-            });
-            const answer = String(result?.answer ?? "").trim();
-            if (!answer)
-                return null;
-            return { answer, toolCalls: 1 };
-        }
-    }
-    return null;
-}
-async function llmGenerateText(llm, history) {
-    const temperature = Number.parseFloat(process.env.NODEBENCH_GAIA_CAPABILITY_TEMPERATURE ?? "0");
-    return generateTextFromHistory(llm, history, {
-        temperature: Number.isFinite(temperature) ? temperature : 0,
-        maxOutputTokens: 1024,
-    });
-}
-/**
- * Gemini vision: send the image + question directly to Gemini multimodal API.
- * Returns null if Gemini isn't available or the call fails.
- */
-function selectVisionModel(task) {
-    const override = process.env.NODEBENCH_GAIA_CAPABILITY_VISION_MODEL;
-    if (override)
-        return override;
-    const q = String(task.prompt ?? "").toLowerCase();
-    const proModel = process.env.NODEBENCH_GAIA_CAPABILITY_VISION_PRO_MODEL ?? "gemini-3-pro-preview";
-    // Use pro model for tasks requiring spatial reasoning or complex OCR + calculation
-    if (q.includes("chess") && q.includes("algebraic notation"))
-        return proModel;
-    if (q.includes("comma separated") && q.includes("fractions") && q.includes("sample"))
-        return proModel;
-    return "gemini-3-flash-preview";
-}
-async function callGeminiVision(apiKey, model, base64, mimeType, prompt, opts) {
-    const mod = await import("@google/genai");
-    const { GoogleGenAI } = mod;
-    const ai = new GoogleGenAI({ apiKey });
-    const response = await ai.models.generateContent({
-        model,
-        contents: [
-            {
-                role: "user",
-                parts: [
-                    { inlineData: { mimeType, data: base64 } },
-                    { text: prompt },
-                ],
-            },
-        ],
-        config: {
-            temperature: opts?.temperature ?? 0,
-            maxOutputTokens: opts?.maxOutputTokens ?? 4096,
-        },
-    });
-    const parts = response?.candidates?.[0]?.content?.parts ?? [];
-    const text = parts.map((p) => p?.text ?? "").join("").trim();
-    return text || null;
-}
-async function tryGeminiVisionAnswer(task, localPath, ext) {
-    const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_AI_API_KEY || "";
-    if (!apiKey)
-        return null;
-    try {
-        const imageBuffer = readFileSync(localPath);
-        const base64 = imageBuffer.toString("base64");
-        const mimeType = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : ext === "webp" ? "image/webp" : "image/png";
-        const model = selectVisionModel(task);
-        const visionPrompt = buildVisionPrompt(task);
-        let text = await callGeminiVision(apiKey, model, base64, mimeType, visionPrompt);
-        if (!text)
-            return null;
-        // Extract answer from chain-of-thought responses (ANSWER: <value> pattern)
-        const answerMatch = text.match(/ANSWER:\s*(.+?)$/im);
-        if (answerMatch) {
-            text = answerMatch[1].trim();
-        }
-        return text || null;
-    }
-    catch (err) {
-        console.warn(`[gaia-media-vision] vision failed for ${task.id}: ${err?.message ?? String(err)}`);
-        return null;
-    }
-}
-/**
- * Should this task use Gemini code execution (image + Python sandbox)?
- * Only tasks where pure vision reasoning consistently fails.
- */
-function shouldUseCodeExecution(task) {
-    const q = String(task.prompt ?? "").toLowerCase();
-    // Chess: code execution reads the board + validates FEN with python-chess.
-    // We extract the FEN and send it to Stockfish (chess-api.com) for the best move.
-    if (q.includes("chess") && q.includes("algebraic notation"))
-        return true;
-    // Fraction extraction: OCR + GCD computation — code execution works well
-    if (q.includes("comma separated") && q.includes("fractions") && q.includes("sample"))
-        return true;
-    return false;
-}
-function buildCodeExecutionPrompt(task) {
-    const q = String(task.prompt ?? "").toLowerCase();
-    if (q.includes("chess") && q.includes("algebraic notation")) {
-        return ("You are a chess grandmaster analyzing this board position image.\n\n" +
-            "BOARD ORIENTATION: This image is shown from BLACK's perspective (the board is FLIPPED).\n" +
-            "- The file labels at the BOTTOM read: h, g, f, e, d, c, b, a (LEFT to RIGHT)\n" +
-            "- The rank labels on the LEFT read: 1, 2, 3, 4, 5, 6, 7, 8 (TOP to BOTTOM)\n" +
-            "- So rank 1 is at the TOP of the image and rank 8 is at the BOTTOM\n" +
-            "- File h is on the LEFT side and file a is on the RIGHT side\n" +
-            "- USE THE PRINTED LABELS to verify each piece's position!\n\n" +
-            "PIECE IDENTIFICATION GUIDE (green/white board style):\n" +
-            "- King (K/k): Tallest piece with a CROSS (+) symbol on top\n" +
-            "- Queen (Q/q): Tall piece with a pointed CROWN (multiple spikes) on top\n" +
-            "- Rook (R/r): Piece with a FLAT CRENELLATED top (castle battlements, rectangular notches)\n" +
-            "- Bishop (B/b): Medium piece with a POINTED TOP and a diagonal SLIT/NOTCH\n" +
-            "- Knight (N/n): Piece with a distinctive HORSE HEAD shape\n" +
-            "- Pawn (P/p): Shortest piece with a simple ROUND BALL on top\n" +
-            "White pieces are LIGHT colored. Black pieces are DARK colored.\n\n" +
-            "COMPLETE THE FOLLOWING THREE PHASES:\n\n" +
-            "═══ PHASE 1: SYSTEMATIC PIECE INVENTORY ═══\n" +
-            "Read the board using the printed coordinate labels as your anchor.\n" +
-            "Go ROW BY ROW from the TOP of the image to the BOTTOM.\n" +
-            "The TOP row is rank 1. The BOTTOM row is rank 8.\n" +
-            "Within each row, go from LEFT (h-file) to RIGHT (a-file).\n\n" +
-            "For each piece, check its TOP SHAPE carefully:\n" +
-            "- Cross on top? → KING\n" +
-            "- Spiky crown? → QUEEN\n" +
-            "- Rectangular battlements? → ROOK\n" +
-            "- Pointed with slit? → BISHOP\n" +
-            "- Horse head? → KNIGHT\n" +
-            "- Simple ball? → PAWN\n\n" +
-            "Write your inventory using the ACTUAL SQUARES (not image positions):\n" +
-            "  Row at top (rank 1): h1=? g1=? f1=? e1=? d1=? c1=? b1=? a1=?\n" +
-            "  Next row (rank 2): h2=? g2=? f2=? e2=? d2=? c2=? b2=? a2=?\n" +
-            "  ... continue through all 8 rows ...\n" +
-            "  Bottom row (rank 8): h8=? g8=? f8=? e8=? d8=? c8=? b8=? a8=?\n\n" +
-            "Use: K=White King, Q=White Queen, R=White Rook, B=White Bishop, N=White Knight, P=White Pawn\n" +
-            "     k=Black King, q=Black Queen, r=Black Rook, b=Black Bishop, n=Black Knight, p=Black Pawn\n" +
-            "     . = empty square\n\n" +
-            "═══ PHASE 2: FEN CONSTRUCTION & VALIDATION ═══\n" +
-            "Write Python code using the `chess` library (it is pre-installed).\n" +
-            "IMPORTANT: FEN notation lists rank 8 FIRST, then rank 7, ..., rank 1 LAST.\n" +
-            "Within each rank, list from a-file to h-file.\n" +
-            "So you need to REVERSE your inventory order: start from the BOTTOM row (rank 8) and go UP.\n\n" +
-            "Your code must:\n" +
-            "1. Construct FEN from your inventory\n" +
-            "2. Load it: board = chess.Board(fen)\n" +
-            "3. Print str(board) — the ASCII board should match what you see in the image\n" +
-            "4. Validate: board.is_valid(), exactly 1 king per side, no pawns on rank 1/8\n" +
-            "5. If invalid, print board.status() and fix the FEN\n\n" +
-            "═══ PHASE 3: VALIDATE & ANALYZE ═══\n" +
-            "Set board.turn = chess.BLACK (it is Black to move).\n" +
-            "Validate the position, print the board and FEN, then list legal moves.\n\n" +
-            "```python\n" +
-            "import chess\n\n" +
-            "board = chess.Board(fen='<your FEN>')\n" +
-            "board.turn = chess.BLACK\n" +
-            "assert board.is_valid(), f'Invalid: {board.status()}'\n" +
-            "print(board)\n" +
-            "print(f'Is valid: {board.is_valid()}')\n" +
-            "print(f'BOARD_FEN: {board.fen()}')\n" +
-            "print(f'Legal moves: {list(board.legal_moves)}')\n" +
-            "```\n\n" +
-            `QUESTION: ${task.prompt}\n\n` +
-            "Execute all three phases. Print the board and FEN for verification.");
-    }
-    if (q.includes("comma separated") && q.includes("fractions") && q.includes("sample")) {
-        return ("You are extracting fractions from a math worksheet image.\n\n" +
-            `TASK: ${task.prompt}\n\n` +
-            "IMPORTANT RULES:\n" +
-            "- Do NOT import cv2, PIL, numpy, or any image processing library.\n" +
-            "- Do NOT try to open, decode, or process the image file with code.\n" +
-            "- Use your EYES (vision) to read the fractions from the image.\n" +
-            "- Use Python code ONLY for math computation (GCD, simplification).\n\n" +
-            "The worksheet has TWO sections:\n\n" +
-            "SECTION A — BODY TEXT (10 fractions, already identified):\n" +
-            "3/4, 1/4, 3/4, 3/4, 2/4, 1/2, 5/35, 7/21, 30/5, 30/5\n\n" +
-            "SECTION B — SAMPLE PROBLEMS (read from the image with your eyes):\n" +
-            "Look at the bottom portion of the image. There are exactly 7 sample problems.\n" +
-            "Each sample problem shows a stacked fraction: a numerator on top of a line, denominator below.\n\n" +
-            "YOUR STEPS:\n" +
-            "1. LOOK at the image and identify each of the 7 stacked fractions.\n" +
-            "   Write down each numerator and denominator you see.\n" +
-            "2. Write Python code that:\n" +
-            "   a) Defines the 7 fractions you read as a list of (numerator, denominator) tuples\n" +
-            "   b) For each, computes math.gcd(num, den) and simplifies: num//g, den//g\n" +
-            "   c) Combines the 10 body fractions + 7 simplified fractions\n" +
-            "   d) Prints EXACTLY 17 comma-separated fractions with no spaces\n\n" +
-            "Expected output format: 3/4,1/4,3/4,3/4,2/4,1/2,5/35,7/21,30/5,30/5,a/b,c/d,e/f,g/h,i/j,k/l,m/n\n\n" +
-            "The code should look like:\n" +
-            "```python\n" +
-            "import math\n" +
-            "body = [(3,4),(1,4),(3,4),(3,4),(2,4),(1,2),(5,35),(7,21),(30,5),(30,5)]\n" +
-            "samples = [(?,?),(?,?),(?,?),(?,?),(?,?),(?,?),(?,?)]  # fill in what you see\n" +
-            "result = []\n" +
-            "for n,d in body:\n" +
-            "    result.append(f'{n}/{d}')\n" +
-            "for n,d in samples:\n" +
-            "    g = math.gcd(n,d)\n" +
-            "    result.append(f'{n//g}/{d//g}')\n" +
-            "print(','.join(result))\n" +
-            "```\n" +
-            "Replace the ? values with what you READ from the image. Run the code.");
-    }
-    // Generic fallback (shouldn't reach here due to shouldUseCodeExecution check)
-    return `${task.prompt}\n\nWrite Python code to solve this. Print ONLY the final answer.`;
-}
-/**
- * Gemini code execution: send image + prompt with tools: [{ codeExecution: {} }].
- * The model generates and runs Python server-side to analyze the image.
- *
- * For chess tasks: rotates image 180° + uses python-chess library
- * For other tasks: single call with image + code execution
- */
-async function tryGeminiCodeExecutionAnswer(task, localPath, ext) {
-    const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_AI_API_KEY || "";
-    if (!apiKey)
-        return null;
-    const q = String(task.prompt ?? "").toLowerCase();
-    const isChess = q.includes("chess") && q.includes("algebraic notation");
-    try {
-        let imageBuffer = Buffer.from(readFileSync(localPath));
-        // Chess: use ORIGINAL image (not rotated) — the coordinate labels are readable
-        // and the model uses them to verify piece positions. Rotation makes labels upside-down.
-        if (isChess) {
-            console.log(`[gaia-media-chess] using original image (Black perspective with readable coordinate labels)`);
-        }
-        const base64 = imageBuffer.toString("base64");
-        const mimeType = ext === "jpg" || ext === "jpeg" ? "image/jpeg" : ext === "webp" ? "image/webp" : "image/png";
-        const mod = await import("@google/genai");
-        const { GoogleGenAI } = mod;
-        const ai = new GoogleGenAI({ apiKey });
-        // Flash for chess (reliable, faster), Flash for others
-        const model = process.env.NODEBENCH_GAIA_CAPABILITY_CODE_EXEC_MODEL ?? "gemini-3-flash-preview";
-        const prompt = buildCodeExecutionPrompt(task);
-        // Chess: extract FEN from code execution, then use Stockfish for the best move
-        if (isChess) {
-            const temperatures = [0, 0.2, 0.4];
-            const fens = [];
-            for (const temp of temperatures) {
-                try {
-                    const resp = await ai.models.generateContent({
-                        model,
-                        contents: [{
-                                role: "user",
-                                parts: [
-                                    { inlineData: { mimeType, data: base64 } },
-                                    { text: prompt },
-                                ],
-                            }],
-                        config: {
-                            tools: [{ codeExecution: {} }],
-                            maxOutputTokens: 8192,
-                            temperature: temp,
-                        },
-                    });
-                    const fen = extractFenFromResponse(resp, temp);
-                    if (fen) {
-                        fens.push(fen);
-                    }
-                }
-                catch (err) {
-                    console.warn(`[gaia-chess-fen] temp=${temp} error: ${err?.message?.slice(0, 100)}`);
-                }
-            }
-            if (fens.length === 0) {
-                console.log(`[gaia-chess-fen] no valid FENs extracted, falling through`);
-                return null;
-            }
-            // Use the most common FEN
-            const fenCounts = {};
-            for (const f of fens)
-                fenCounts[f] = (fenCounts[f] || 0) + 1;
-            const sortedFens = Object.entries(fenCounts).sort((a, b) => b[1] - a[1]);
-            const consensusFen = sortedFens[0][0];
-            console.log(`[gaia-chess-fen] FENs: ${JSON.stringify(fenCounts)} → consensus: ${consensusFen}`);
-            // Query chess-api.com (Stockfish NNUE) for the best move
-            const fullFen = `${consensusFen} b - - 0 1`; // Black to move
-            console.log(`[gaia-chess-engine] querying Stockfish: ${fullFen}`);
-            try {
-                const chessResp = await fetch("https://chess-api.com/v1", {
-                    method: "POST",
-                    headers: { "Content-Type": "application/json" },
-                    body: JSON.stringify({ fen: fullFen, depth: 18, variants: 1 }),
-                    signal: AbortSignal.timeout(15000),
-                });
-                if (chessResp.ok) {
-                    const data = await chessResp.json();
-                    const bestMove = data?.san ?? data?.move ?? null;
-                    if (bestMove) {
-                        console.log(`[gaia-chess-engine] Stockfish: ${bestMove} (eval: ${data?.eval ?? "?"})`);
-                        return String(bestMove).trim();
-                    }
-                    console.warn(`[gaia-chess-engine] no move: ${JSON.stringify(data).slice(0, 200)}`);
-                }
-                else {
-                    console.warn(`[gaia-chess-engine] API error: ${chessResp.status}`);
-                }
-            }
-            catch (err) {
-                console.warn(`[gaia-chess-engine] fetch error: ${err?.message?.slice(0, 100)}`);
-            }
-            return null;
-        }
-        // Non-chess: single code execution call
-        const response = await ai.models.generateContent({
-            model,
-            contents: [
-                {
-                    role: "user",
-                    parts: [
-                        { inlineData: { mimeType, data: base64 } },
-                        { text: prompt },
-                    ],
-                },
-            ],
-            config: {
-                tools: [{ codeExecution: {} }],
-                maxOutputTokens: 8192,
-                temperature: 0,
-            },
-        });
-        return extractCodeExecutionAnswer(response, task.id, model);
-    }
-    catch (err) {
-        console.warn(`[gaia-media-code-exec] failed for ${task.id}: ${err?.message ?? String(err)}`);
-        return null;
-    }
-}
-/**
- * Extract a validated FEN piece-placement from a Gemini code execution response.
- * Tries multiple strategies: BOARD_FEN marker, FEN regex, Python code parsing, ASCII board parsing.
- */
-function extractFenFromResponse(response, temp) {
-    const parts = response?.candidates?.[0]?.content?.parts ?? [];
-    let codeOutput = "";
-    let allCode = "";
-    for (const part of parts) {
-        if (part.codeExecutionResult?.output) {
-            codeOutput = String(part.codeExecutionResult.output).trim();
-        }
-        if (part.executableCode?.code) {
-            allCode += part.executableCode.code + "\n";
-        }
-    }
-    if (codeOutput) {
-        console.log(`[gaia-chess-fen] temp=${temp} code_output:\n${codeOutput.slice(0, 600)}`);
-    }
-    let fen = null;
-    // Strategy 1: BOARD_FEN: <fen> marker
-    const boardFenMatch = codeOutput.match(/BOARD_FEN:\s*(.+)/);
-    if (boardFenMatch) {
-        fen = boardFenMatch[1].trim().split(" ")[0];
-    }
-    // Strategy 2: FEN regex in code output (8 ranks separated by /)
-    if (!fen) {
-        const fenPatterns = codeOutput.match(/([rnbqkpRNBQKP1-8]{1,8}\/){7}[rnbqkpRNBQKP1-8]{1,8}/g);
-        if (fenPatterns && fenPatterns.length > 0) {
-            fen = fenPatterns[fenPatterns.length - 1];
-        }
-    }
-    // Strategy 3: FEN in Python source code (Board(fen='...') or Board('...'))
-    if (!fen) {
-        const codeMatch = allCode.match(/Board\(\s*(?:fen\s*=\s*)?['"](([rnbqkpRNBQKP1-8]{1,8}\/){7}[rnbqkpRNBQKP1-8]{1,8})[^'"]*['"]/);
-        if (codeMatch) {
-            fen = codeMatch[1];
-            console.log(`[gaia-chess-fen] temp=${temp} FEN from Python source: ${fen}`);
-        }
-    }
-    // Strategy 4: Parse ASCII board output (". . . r . . k ." format)
-    if (!fen && codeOutput) {
-        const boardLines = codeOutput.split("\n")
-            .map((l) => l.trim())
-            .filter((l) => /^[.rnbqkpRNBQKP ]+$/.test(l) && l.length >= 15);
-        if (boardLines.length >= 8) {
-            const fenRanks = [];
-            for (const line of boardLines.slice(0, 8)) {
-                const squares = line.split(/\s+/);
-                if (squares.length !== 8)
-                    break;
-                let rank = "";
-                let emptyCount = 0;
-                for (const sq of squares) {
-                    if (sq === ".") {
-                        emptyCount++;
-                    }
-                    else {
-                        if (emptyCount > 0) {
-                            rank += emptyCount;
-                            emptyCount = 0;
-                        }
-                        rank += sq;
-                    }
-                }
-                if (emptyCount > 0)
-                    rank += emptyCount;
-                fenRanks.push(rank);
-            }
-            if (fenRanks.length === 8) {
-                fen = fenRanks.join("/");
-                console.log(`[gaia-chess-fen] temp=${temp} FEN from ASCII board: ${fen}`);
-            }
-        }
-    }
-    // Validate FEN
-    if (fen) {
-        const ranks = fen.split("/");
-        if (ranks.length === 8 && fen.includes("K") && fen.includes("k")) {
-            console.log(`[gaia-chess-fen] temp=${temp} valid FEN: ${fen}`);
-            return fen;
-        }
-        console.log(`[gaia-chess-fen] temp=${temp} invalid FEN: ${fen}`);
-    }
-    else {
-        console.log(`[gaia-chess-fen] temp=${temp} no FEN found (code=${codeOutput.length}ch, src=${allCode.length}ch)`);
-    }
-    return null;
-}
-function extractCodeExecutionAnswer(response, taskId, model) {
-    const parts = response?.candidates?.[0]?.content?.parts ?? [];
-    let lastCodeOutput = "";
-    const allTexts = [];
-    for (const part of parts) {
-        if (part.codeExecutionResult?.output) {
-            lastCodeOutput = String(part.codeExecutionResult.output).trim();
-        }
-        if (part.text) {
-            allTexts.push(String(part.text).trim());
-        }
-    }
-    const lastText = allTexts[allTexts.length - 1] ?? "";
-    // Log full code output for debugging
-    if (lastCodeOutput) {
-        console.log(`[gaia-media-code-exec] ${taskId} full_output:\n${lastCodeOutput.slice(0, 1500)}`);
-    }
-    // Combine all text sources for pattern matching (check text parts first — the model
-    // writes "ANSWER: Rd5" as text after the code execution block)
-    const combinedText = [...allTexts, lastCodeOutput].filter(Boolean).join("\n");
-    if (!combinedText)
-        return null;
-    // Pattern 1: ANSWER: <value> (from our prompt template, appears in text after code execution)
-    const answerMatch = combinedText.match(/ANSWER:\s*(.+?)$/im);
-    if (answerMatch) {
-        const answer = answerMatch[1].trim();
-        console.log(`[gaia-media-code-exec] ${taskId} model=${model} answer=${answer} (from ANSWER pattern)`);
-        return answer || null;
-    }
-    // Pattern 2: BEST: <move> (legacy chess code template)
-    const bestMatch = combinedText.match(/BEST:\s*(\S+)/im);
-    if (bestMatch) {
-        const answer = bestMatch[1].trim();
-        console.log(`[gaia-media-code-exec] ${taskId} model=${model} answer=${answer} (from BEST pattern)`);
-        return answer || null;
-    }
-    // Fallback: prefer code execution output, then last non-empty line
-    let answer = lastCodeOutput || lastText;
-    const lines = answer.split("\n").map((l) => l.trim()).filter(Boolean);
-    if (lines.length > 0) {
-        answer = lines[lines.length - 1];
-    }
-    console.log(`[gaia-media-code-exec] ${taskId} model=${model} answer=${answer.slice(0, 80)} (from last line)`);
-    return answer || null;
-}
-function buildVisionPrompt(task) {
-    const q = String(task.prompt ?? "").toLowerCase();
-    // Chess position analysis — detailed chain-of-thought for spatial reasoning
-    if (q.includes("chess") && q.includes("algebraic notation")) {
-        return ("You are a chess grandmaster analyzing this board position.\n\n" +
-            "BOARD ORIENTATION: This board is shown from BLACK'S perspective (flipped).\n" +
-            "- The file labels at the BOTTOM read: h, g, f, e, d, c, b, a (left to right)\n" +
-            "- The rank labels on the LEFT read: 1, 2, 3, 4, 5, 6, 7, 8 (top to bottom)\n" +
-            "- So rank 1 is at the TOP, rank 8 is at the BOTTOM\n" +
-            "- USE the printed coordinate labels to anchor each piece's position!\n\n" +
-            "PIECE IDENTIFICATION:\n" +
-            "- King: cross (+) on top  |  Queen: crown with spikes on top\n" +
-            "- Rook: flat crenellated (castle) top  |  Bishop: pointed top with slit\n" +
-            "- Knight: horse head shape  |  Pawn: simple round ball on top\n" +
-            "- White pieces are LIGHT, Black pieces are DARK\n\n" +
-            "STEP 1 — BOARD INVENTORY\n" +
-            "Go row by row from TOP (rank 1) to BOTTOM (rank 8).\n" +
-            "Within each row, go from LEFT (h-file) to RIGHT (a-file).\n" +
-            "List EVERY piece: type, color, and exact square (verified against labels).\n\n" +
-            "STEP 2 — POSITION ANALYSIS (Black to move)\n" +
-            "- Where is each king? Is either king exposed?\n" +
-            "- Where are Black's rooks? What ranks and files can they control?\n" +
-            "- Where is White's queen? Can any Black piece attack it?\n" +
-            "- A rook move along a RANK can attack multiple pieces on that rank.\n" +
-            "  For example, a rook on d5 attacks everything on the 5th rank (e5, f5, g5, h5)\n" +
-            "  AND everything on the d-file (d4, d3, d2, d1).\n\n" +
-            "STEP 3 — CANDIDATE MOVES\n" +
-            "Consider Black's strongest moves. Prioritize:\n" +
-            "1. Moves that SIMULTANEOUSLY attack multiple high-value pieces\n" +
-            "2. Rook moves to open ranks that threaten the queen AND create back-rank threats\n" +
-            "3. Moves that force the opponent into losing material\n\n" +
-            "STEP 4 — WINNING MOVE\n" +
-            "The winning move creates an unstoppable double threat for Black.\n\n" +
-            `QUESTION: ${task.prompt}\n\n` +
-            "Think step by step. Write your final answer on the LAST LINE as exactly:\n" +
-            "ANSWER: <move>\n" +
-            "where <move> is in standard algebraic notation (e.g., Rd5, Qxf7+, Nf3).");
-    }
-    // Fraction quiz grading
-    if (q.includes("quiz is scored") && q.includes("bonus points")) {
-        return ("You are grading a student's fraction quiz shown in this image.\n\n" +
-            "INSTRUCTIONS:\n" +
-            "1. Read each problem carefully from the image\n" +
-            "2. Read the student's written answer for each problem\n" +
-            "3. Check if each answer is mathematically correct (no partial credit)\n" +
-            "4. Categorize each problem and assign points per the rubric\n" +
-            "5. Sum all earned points and add any bonus\n\n" +
-            `SCORING RUBRIC:\n${task.prompt}\n\n` +
-            "CRITICAL: Return ONLY the total integer score as a single number. " +
-            "No explanation, no breakdown, just the number.");
-    }
-    // Fraction extraction — very detailed multi-step instructions
-    if (q.includes("comma separated") && q.includes("fractions") && q.includes("sample")) {
-        return ("You must carefully examine this worksheet image and extract information.\n\n" +
-            `TASK: ${task.prompt}\n\n` +
-            "DETAILED INSTRUCTIONS:\n" +
-            "1. Read the ENTIRE image from top to bottom, left to right.\n" +
-            "2. Find ALL fractions written using the / notation (like 3/4, 1/2, etc).\n" +
-            "   This includes fractions in:\n" +
-            "   - The body text and explanations\n" +
-            "   - Problem statements\n" +
-            "   - Student answers\n" +
-            "   - Sample problems and their solutions\n" +
-            "3. For sample problems that ASK you to compute an answer, compute the answer " +
-            "   and include it as a fraction using / notation.\n" +
-            "4. Order ALL fractions by the order they appear in the image (top to bottom, left to right).\n" +
-            "5. Include fractions even if they repeat.\n" +
-            "6. Do NOT simplify fractions unless the sample problem specifically asks for simplification.\n\n" +
-            "First, describe everything you see in the image line by line.\n" +
-            "Then list every fraction you found.\n" +
-            "Finally, write your answer on the LAST LINE as:\n" +
-            "ANSWER: fraction1,fraction2,fraction3,...\n" +
-            "with NO spaces between fractions.");
-    }
-    // Default prompt
-    return ("Look at this image carefully and answer the following question.\n\n" +
-        `${task.prompt}\n\n` +
-        "CRITICAL: Return ONLY the final answer. No explanation, no reasoning, no extra text. " +
-        "Just the raw answer value.");
-}
-function createNoopTextLlmClient(model) {
-    return {
-        provider: "none",
-        model,
-        generateText: async () => "",
-    };
-}
-async function baselineAnswer(llm, task) {
-    const contents = [
-        {
-            role: "user",
-            parts: [
-                {
-                    text: `Answer the question using your existing knowledge only. Do not browse the web.\n\nReturn ONLY the final answer, no explanation.\n\nQuestion:\n${task.prompt}`,
-                },
-            ],
-        },
-    ];
-    return llmGenerateText(llm, contents);
-}
-function buildToolIndex() {
-    const byName = new Map();
-    for (const tool of localFileTools)
-        byName.set(tool.name, tool);
-    for (const tool of gaiaMediaSolvers)
-        byName.set(tool.name, tool);
-    return byName;
-}
-function extractJsonObject(text) {
-    const trimmed = text.trim();
-    const fenceMatch = trimmed.match(/```json\s*([\s\S]*?)\s*```/i);
-    const candidate = fenceMatch ? fenceMatch[1] : trimmed;
-    const start = candidate.indexOf("{");
-    const end = candidate.lastIndexOf("}");
-    if (start === -1 || end === -1 || end <= start)
-        return null;
-    const slice = candidate.slice(start, end + 1);
-    try {
-        return JSON.parse(slice);
-    }
-    catch {
-        return null;
-    }
-}
-async function loadFixture(fixturePath) {
-    const raw = await readFile(fixturePath, "utf8");
-    const parsed = JSON.parse(raw);
-    if (!parsed || !Array.isArray(parsed.tasks))
-        throw new Error("Invalid GAIA capability fixture");
-    return parsed;
-}
-function resolveTaskLocalFilePath(task) {
-    const repoRoot = resolveRepoRoot();
-    const rel = String(task.localFilePath ?? "").trim();
-    if (rel)
-        return path.resolve(repoRoot, rel);
-    const filePath = String(task.filePath ?? "").trim();
-    if (!filePath)
-        throw new Error("Task missing filePath/localFilePath");
-    return path.join(repoRoot, ".cache", "gaia", "data", filePath);
-}
-async function toolAugmentedAnswerFromImage(llm, task, opts) {
-    const toolIndex = buildToolIndex();
-    const toolsMode = (process.env.NODEBENCH_GAIA_CAPABILITY_TOOLS_MODE ?? "rag").toLowerCase();
-    const localPath = resolveTaskLocalFilePath(task);
-    if (!existsSync(localPath)) {
-        throw new Error(`Missing attachment on disk. Expected at ${localPath}. Refresh with dataset:gaia:capability:media:refresh`);
-    }
-    const ext = String(task.fileExt ?? "").trim().toLowerCase() ||
-        path.extname(task.fileName || task.filePath || "").toLowerCase().replace(/^\./, "");
-    if (!["png", "jpg", "jpeg", "webp"].includes(ext)) {
-        throw new Error(`Unsupported attachment type for media lane: ${ext || "(unknown)"}`);
-    }
-    // "rag" mode: tiered approach for best accuracy.
-    //   Tier 1: Deterministic solver (fast, free, no API call) — proven for math/structured tasks
-    //   Tier 1.5: Gemini code execution (image + Python sandbox) — for tasks needing computation
-    //   Tier 2: Gemini vision (image sent directly to multimodal model) — for visual reasoning
-    //   Tier 3: OCR + text LLM fallback
-    if (toolsMode === "rag") {
-        const q = String(task.prompt ?? "").toLowerCase();
-        const isOcrHeavyTask = (q.includes("quiz is scored") && q.includes("bonus points")) ||
-            (q.includes("comma separated") && q.includes("fractions") && q.includes("sample"));
-        const useCodeExec = shouldUseCodeExecution(task);
-        // Tier 1: try deterministic solver first
-        const deterministic = await tryDeterministicMediaSolve(toolIndex, task, localPath);
-        if (deterministic && !isOcrHeavyTask && !useCodeExec) {
-            // Deterministic is proven reliable for structured math tasks
-            return deterministic;
-        }
-        // Tier 1.5: Gemini code execution for tasks that need computational analysis
-        if (useCodeExec) {
-            const codeExecAnswer = await tryGeminiCodeExecutionAnswer(task, localPath, ext);
-            if (codeExecAnswer)
-                return { answer: codeExecAnswer, toolCalls: 1 };
-            // Fall through to chess consensus / vision if code execution fails
-        }
-        // Tier 2: Gemini vision
-        const visionAnswer = await tryGeminiVisionAnswer(task, localPath, ext);
-        if (deterministic && visionAnswer) {
-            if (isOcrHeavyTask) {
-                return { answer: visionAnswer, toolCalls: 1 };
-            }
-            return deterministic;
-        }
-        if (visionAnswer)
-            return { answer: visionAnswer, toolCalls: 1 };
-        if (deterministic)
-            return deterministic;
-        // Offline fallback: if no LLM provider is configured, we cannot do OCR->LLM reasoning.
-        if (llm?.provider === "none") {
-            return { answer: "", toolCalls: 0 };
-        }
-        // Tier 3: OCR extract + text LLM
-        const tool = toolIndex.get("read_image_ocr_text");
-        if (!tool)
-            throw new Error("Missing tool: read_image_ocr_text");
-        const extract = await tool.handler({
-            path: localPath,
-            lang: "eng",
-            preprocess: true,
-            maxChars: 40000,
-        });
-        const extractText = JSON.stringify(extract).slice(0, 40000);
-        const contents = [
-            {
-                role: "user",
-                parts: [
-                    {
-                        text: "Answer the question using ONLY the provided OCR extract. " +
-                            "If the extract is insufficient, make the best supported guess.\n\n" +
-                            "Return ONLY the final answer, no explanation.\n\n" +
-                            `TASK_ID: ${task.id}\n` +
-                            `FILE_TYPE: ${ext}\n` +
-                            `LOCAL_FILE_PATH: ${localPath}\n` +
-                            `QUESTION:\n${task.prompt}\n\n` +
-                            `OCR_EXTRACT_JSON:\n${extractText}`,
-                    },
-                ],
-            },
-        ];
-        const answer = await llmGenerateText(llm, contents);
-        return { answer, toolCalls: 1 };
-    }
-    // "agent" mode: small tool loop. This is more realistic but higher variance.
-    const toolUsageSummary = [
-        "You have access to deterministic local media tools:",
-        "- read_image_ocr_text({path,lang,langPath,preprocess,maxChars})",
-        "",
-        "When using tools, respond with a single JSON object only:",
-        "{\"action\":\"tool\",\"name\":\"read_image_ocr_text\",\"arguments\":{\"maxChars\":20000}}",
-        "When done, respond with:",
-        "{\"action\":\"final\",\"answer\":\"...\"}",
-        "",
-        "Rules:",
-        "- Do NOT use any external knowledge or web browsing.",
-        "- Always use the provided LOCAL_FILE_PATH; you may not read any other files.",
-        "- Keep tool results bounded (maxChars<=40000).",
-        "- Do NOT include any explanation. Final answer must match the requested formatting.",
-    ].join("\n");
-    const contents = [
-        {
-            role: "user",
-            parts: [
-                {
-                    text: `${toolUsageSummary}\n\nTASK_ID: ${task.id}\nFILE_TYPE: ${ext}\nLOCAL_FILE_PATH: ${localPath}\nQUESTION:\n${task.prompt}`,
-                },
-            ],
-        },
-    ];
-    let toolCalls = 0;
-    for (let step = 0; step < opts.maxSteps; step++) {
-        const out = await llmGenerateText(llm, contents);
-        contents.push({ role: "model", parts: [{ text: out }] });
-        const parsed = extractJsonObject(out);
-        if (!parsed || typeof parsed !== "object") {
-            contents.push({
-                role: "user",
-                parts: [{ text: "Invalid format. Return JSON only with action tool|final." }],
-            });
-            continue;
-        }
-        if (parsed.action === "final") {
-            const answer = String(parsed.answer ?? "").trim();
-            return { answer, toolCalls };
-        }
-        if (parsed.action !== "tool") {
-            contents.push({
-                role: "user",
-                parts: [{ text: "Invalid action. Return JSON only with action tool|final." }],
-            });
-            continue;
-        }
-        if (toolCalls >= opts.maxToolCalls) {
-            contents.push({
-                role: "user",
-                parts: [{ text: "Tool call budget exceeded. Return final answer now." }],
-            });
-            continue;
-        }
-        const name = String(parsed.name ?? "");
-        const tool = toolIndex.get(name);
-        if (!tool || name !== "read_image_ocr_text") {
-            contents.push({
-                role: "user",
-                parts: [{ text: `Unknown tool "${name}". Use only read_image_ocr_text.` }],
-            });
-            continue;
-        }
-        const args = (parsed.arguments ?? {});
-        // Security: enforce file access restrictions.
-        args.path = localPath;
-        if (typeof args.maxChars !== "number")
-            args.maxChars = 40000;
-        args.maxChars = Math.min(Number(args.maxChars) || 40000, 40000);
-        toolCalls++;
-        const toolResult = await tool.handler(args);
-        const toolResultText = JSON.stringify(toolResult).slice(0, 12000);
-        contents.push({
-            role: "user",
-            parts: [{ text: `TOOL_RESULT ${name}:\n${toolResultText}\n\nContinue. Return JSON only.` }],
-        });
-    }
-    contents.push({
-        role: "user",
-        parts: [{ text: "Out of steps. Return final answer now as JSON." }],
-    });
-    const out = await llmGenerateText(llm, contents);
-    const parsed = extractJsonObject(out);
-    const answer = parsed && parsed.action === "final" ? String(parsed.answer ?? "").trim() : out.trim();
-    return { answer, toolCalls };
-}
-describe("Capability: GAIA accuracy (LLM-only vs LLM+media tools)", () => {
-    const testFn = shouldRun ? it : it.skip;
-    testFn("should measure accuracy delta on a small GAIA image subset", async () => {
-        loadDotEnvLocalIfPresent();
-        const fixturePath = resolveCapabilityMediaFixturePath();
-        if (!existsSync(fixturePath)) {
-            throw new Error(`Missing GAIA media fixture at ${fixturePath}. Generate it with: python packages/mcp-local/src/__tests__/fixtures/generateGaiaCapabilityMediaFixture.py`);
-        }
-        const baselineModel = process.env.NODEBENCH_GAIA_BASELINE_MODEL ?? "gemini-3-flash-preview";
-        const toolsModel = process.env.NODEBENCH_GAIA_TOOLS_MODEL ?? baselineModel;
-        // This harness is designed to run with a real LLM provider (Gemini/OpenAI/Anthropic).
-        // In CI/agent environments, keys may be intentionally unavailable; allow a deterministic-only run
-        // (baseline becomes always-wrong, tools rely on deterministic solvers) so we can still measure lift.
-        let baselineLlm;
-        let toolsLlm;
-        try {
-            baselineLlm = await createTextLlmClient({ model: baselineModel });
-        }
-        catch {
-            baselineLlm = createNoopTextLlmClient(baselineModel);
-        }
-        try {
-            toolsLlm = await createTextLlmClient({ model: toolsModel });
-        }
-        catch {
-            toolsLlm = createNoopTextLlmClient(toolsModel);
-        }
-        const baselineModelLabel = `${baselineLlm.provider}:${baselineLlm.model}`;
-        const toolsModelLabel = `${toolsLlm.provider}:${toolsLlm.model}`;
-        const fixture = await loadFixture(fixturePath);
-        expect(Array.isArray(fixture.tasks)).toBe(true);
-        expect(fixture.tasks.length).toBeGreaterThan(0);
-        const requestedLimit = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_TASK_LIMIT ?? "6", 10);
-        const taskLimit = Math.max(1, Math.min(fixture.tasks.length, Number.isFinite(requestedLimit) ? requestedLimit : 6));
-        const tasks = fixture.tasks.slice(0, taskLimit);
-        const requestedConcurrency = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_CONCURRENCY ?? "1", 10);
-        const concurrency = Math.max(1, Math.min(tasks.length, Number.isFinite(requestedConcurrency) ? requestedConcurrency : 1));
-        const maxSteps = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_MAX_STEPS ?? "7", 10);
-        const maxToolCalls = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_MAX_TOOL_CALLS ?? "3", 10);
-        // Auto-discover judge: free OpenRouter → paid LLM → deterministic-only
-        const useJudge = process.env.NODEBENCH_GAIA_JUDGE !== "0";
-        const judge = useJudge ? await autoDiscoverJudge(toolsLlm) : null;
-        if (judge) {
-            console.log(`[gaia-capability-media] judge: ${judge.provider}:${judge.model}`);
-        }
-        const results = new Array(tasks.length);
-        let nextIndex = 0;
-        const workers = Array.from({ length: concurrency }, () => (async () => {
-            while (true) {
-                const idx = nextIndex++;
-                if (idx >= tasks.length)
-                    return;
-                const task = tasks[idx];
-                try {
-                    const baseStart = performance.now();
-                    const base = await baselineAnswer(baselineLlm, task);
-                    const baseMs = performance.now() - baseStart;
-                    const toolsStart = performance.now();
-                    const tools = await toolAugmentedAnswerFromImage(toolsLlm, task, { maxSteps, maxToolCalls });
-                    const toolsMs = performance.now() - toolsStart;
-                    const baseJudge = await answersMatchWithJudge(task.expectedAnswer, base, judge);
-                    const toolsJudge = await answersMatchWithJudge(task.expectedAnswer, tools.answer, judge);
-                    results[idx] = {
-                        taskId: task.id,
-                        baselineCorrect: baseJudge.match,
-                        toolsCorrect: toolsJudge.match,
-                        baselineMs: baseMs,
-                        toolsMs,
-                        toolCalls: tools.toolCalls,
-                        judgeProvider: toolsJudge.judgeProvider,
-                        judgeInvoked: toolsJudge.judgeInvoked,
-                    };
-                }
-                catch (err) {
-                    results[idx] = {
-                        taskId: task.id,
-                        baselineCorrect: false,
-                        toolsCorrect: false,
-                        baselineMs: 0,
-                        toolsMs: 0,
-                        toolCalls: 0,
-                        error: err?.message ?? String(err),
-                    };
-                }
-            }
-        })());
-        await Promise.all(workers);
-        const baselineCorrect = results.filter((r) => r.baselineCorrect).length;
-        const toolsCorrect = results.filter((r) => r.toolsCorrect).length;
-        const baselinePassRate = (baselineCorrect / results.length) * 100;
-        const toolsPassRate = (toolsCorrect / results.length) * 100;
-        const avgBaseMs = results.reduce((sum, r) => sum + r.baselineMs, 0) / results.length;
-        const avgToolsMs = results.reduce((sum, r) => sum + r.toolsMs, 0) / results.length;
-        const avgToolCalls = results.reduce((sum, r) => sum + r.toolCalls, 0) / results.length;
-        const improved = results.filter((r) => !r.baselineCorrect && r.toolsCorrect).length;
-        const regressions = results.filter((r) => r.baselineCorrect && !r.toolsCorrect).length;
-        // Human-readable console output (no prompts/answers).
-        console.log(`[gaia-capability-media] tasks=${results.length} baseline=${baselineCorrect}/${results.length} (${baselinePassRate.toFixed(1)}%) tools=${toolsCorrect}/${results.length} (${toolsPassRate.toFixed(1)}%) delta=${(toolsPassRate - baselinePassRate).toFixed(1)}% improved=${improved} regressions=${regressions} avgToolCalls=${avgToolCalls.toFixed(2)}`);
-        const toolsMode = (process.env.NODEBENCH_GAIA_CAPABILITY_TOOLS_MODE ?? "rag").toLowerCase();
-        const publicSummary = {
-            suiteId: "gaia_capability_media",
-            lane: "media",
-            generatedAtIso: new Date().toISOString(),
-            config: fixture.config,
-            split: fixture.split,
-            taskCount: results.length,
-            concurrency,
-            baseline: {
-                model: baselineModelLabel,
-                correct: baselineCorrect,
-                passRatePct: baselinePassRate,
-                avgMs: avgBaseMs,
-            },
-            tools: {
-                model: toolsModelLabel,
-                mode: toolsMode,
-                correct: toolsCorrect,
-                passRatePct: toolsPassRate,
-                avgMs: avgToolsMs,
-                avgToolCalls,
-            },
-            improved,
-            regressions,
-            notes: "GAIA media lane (image attachments). No prompts/answers persisted; only aggregate metrics are written to public/evals." +
-                ((baselineLlm.provider === "none" || toolsLlm.provider === "none")
-                    ? " NOTE: No LLM provider key detected in this runner; baseline/tools used deterministic-only fallback for unsupported tasks."
-                    : ""),
-        };
-        if (shouldWriteReport) {
-            const repoRoot = resolveRepoRoot();
-            await safeWriteJson(path.join(repoRoot, "public", "evals", "gaia_capability_media_latest.json"), publicSummary);
-            const detailed = {
-                ...publicSummary,
-                results: results.map((r) => ({
-                    taskId: r.taskId,
-                    baselineCorrect: r.baselineCorrect,
-                    toolsCorrect: r.toolsCorrect,
-                    baselineMs: Math.round(r.baselineMs),
-                    toolsMs: Math.round(r.toolsMs),
-                    toolCalls: r.toolCalls,
-                    ...(r.error ? { error: r.error } : {}),
-                })),
-            };
-            const stamp = new Date().toISOString().replace(/[:.]/g, "-");
-            await safeWriteJson(path.join(repoRoot, ".cache", "gaia", "reports", `gaia_capability_media_${fixture.config}_${fixture.split}_${stamp}.json`), detailed);
-        }
-        // Minimal sanity: tools mode should not underperform baseline on this tiny sample.
-        expect(toolsPassRate).toBeGreaterThanOrEqual(baselinePassRate);
-    }, 900000);
-});
-//# sourceMappingURL=gaiaCapabilityMediaEval.test.js.map