nodebench-mcp 2.8.1 → 2.8.3

package/README.md

@@ -184,6 +184,77 @@ Notes:
 
 ---
 
+## Progressive Discovery (v2.8.1)
+
+129 tools is a lot. The progressive disclosure system helps agents find exactly what they need:
+
+### Multi-modal search engine
+
+```
+> discover_tools("verify my implementation")
+```
+
+The `discover_tools` search engine scores tools using **9 parallel strategies**:
+
+| Strategy | What it does | Example |
+|---|---|---|
+| Keyword | Exact/partial word matching on name, tags, description | "benchmark" → `benchmark_models` |
+| Fuzzy | Levenshtein distance — tolerates typos | "verifiy" → `start_verification_cycle` |
+| N-gram | Trigram similarity for partial words | "screen" → `capture_ui_screenshot` |
+| Prefix | Matches tool name starts | "cap" → `capture_*` tools |
+| Semantic | Synonym expansion (30 word families) | "check" also finds "verify", "validate" |
+| TF-IDF | Rare tags score higher than common ones | "c-compiler" scores higher than "test" |
+| Regex | Pattern matching | `"^run_.*loop$"` → `run_closed_loop` |
+| Bigram | Phrase matching | "quality gate" matched as unit |
+| Domain boost | Related categories boosted together | verification + quality_gate cluster |
+
+**6 search modes**: `hybrid` (default, all strategies), `fuzzy`, `regex`, `prefix`, `semantic`, `exact`
+
+Pass `explain: true` to see exactly which strategies contributed to each score.
+
+### Quick refs — what to do next
+
+Every tool response auto-appends a `_quickRef` with:
+- **nextAction**: What to do immediately after this tool
+- **nextTools**: Recommended follow-up tools
+- **methodology**: Which methodology guide to consult
+- **tip**: Practical usage advice
+
+Call `get_tool_quick_ref("tool_name")` for any tool's guidance.
+
+### Workflow chains — step-by-step recipes
+
+11 pre-built chains for common workflows:
+
+| Chain | Steps | Use case |
+|---|---|---|
+| `new_feature` | 12 | End-to-end feature development |
+| `fix_bug` | 6 | Structured debugging |
+| `ui_change` | 7 | Frontend with visual verification |
+| `parallel_project` | 7 | Multi-agent coordination |
+| `research_phase` | 8 | Context gathering |
+| `academic_paper` | 7 | Paper writing pipeline |
+| `c_compiler_benchmark` | 10 | Autonomous capability test |
+| `security_audit` | 9 | Comprehensive security assessment |
+| `code_review` | 8 | Structured code review |
+| `deployment` | 8 | Ship with full verification |
+| `migration` | 10 | SDK/framework upgrade |
+
+Call `get_workflow_chain("new_feature")` to get the step-by-step sequence.
+
+### Boilerplate template
+
+Start new projects with everything pre-configured:
+
+```bash
+gh repo create my-project --template HomenShum/nodebench-boilerplate --clone
+cd my-project && npm install
+```
+
+Or use the scaffold tool: `scaffold_nodebench_project` creates AGENTS.md, .mcp.json, package.json, CI, Docker, and parallel agent infra.
+
+---
+
 ## The Methodology Pipeline
 
 NodeBench MCP isn't just a bag of tools — it's a pipeline. Each step feeds the next:
@@ -307,7 +378,7 @@ Always included (regardless of gating):
 ## Build from Source
 
 ```bash
-git clone https://github.com/nodebench/nodebench-ai.git
+git clone https://github.com/HomenShum/nodebench-ai.git
 cd nodebench-ai/packages/mcp-local
 npm install && npm run build
 ```

package/dist/__tests__/gaiaCapabilityAudioEval.test.d.ts

@@ -0,0 +1,15 @@
+/**
+ * GAIA audio-backed capability/accuracy benchmark: LLM-only vs LLM+NodeBench MCP local audio tools.
+ *
+ * This lane targets GAIA tasks that include audio attachments (MP3/WAV/etc).
+ * We provide deterministic local transcription via NodeBench MCP tools and score answers against
+ * the ground-truth "Final answer" (stored locally under `.cache/gaia`, gitignored).
+ *
+ * Safety:
+ * - GAIA is gated. Do not commit fixtures that contain prompts/answers.
+ * - This test logs only task IDs and aggregate metrics (no prompt/answer text).
+ *
+ * Disabled by default (cost + rate limits). Run with:
+ *   NODEBENCH_RUN_GAIA_CAPABILITY=1 npm --prefix packages/mcp-local run test
+ */
+export {};

package/dist/__tests__/gaiaCapabilityAudioEval.test.js

@@ -0,0 +1,291 @@
+/**
+ * GAIA audio-backed capability/accuracy benchmark: LLM-only vs LLM+NodeBench MCP local audio tools.
+ *
+ * This lane targets GAIA tasks that include audio attachments (MP3/WAV/etc).
+ * We provide deterministic local transcription via NodeBench MCP tools and score answers against
+ * the ground-truth "Final answer" (stored locally under `.cache/gaia`, gitignored).
+ *
+ * Safety:
+ * - GAIA is gated. Do not commit fixtures that contain prompts/answers.
+ * - This test logs only task IDs and aggregate metrics (no prompt/answer text).
+ *
+ * Disabled by default (cost + rate limits). Run with:
+ *   NODEBENCH_RUN_GAIA_CAPABILITY=1 npm --prefix packages/mcp-local run test
+ */
+import { describe, expect, it } from "vitest";
+import { existsSync, readFileSync } from "node:fs";
+import { mkdir, readFile, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { performance } from "node:perf_hooks";
+import { localFileTools } from "../tools/localFileTools.js";
+const shouldRun = process.env.NODEBENCH_RUN_GAIA_CAPABILITY === "1";
+const shouldWriteReport = process.env.NODEBENCH_WRITE_GAIA_REPORT === "1";
+async function safeWriteJson(filePath, payload) {
+    try {
+        await mkdir(path.dirname(filePath), { recursive: true });
+        await writeFile(filePath, JSON.stringify(payload, null, 2) + "\n", "utf8");
+    }
+    catch (err) {
+        console.warn(`[gaia-capability-audio] report write failed: ${err?.message ?? String(err)}`);
+    }
+}
+function resolveRepoRoot() {
+    const testDir = path.dirname(fileURLToPath(import.meta.url));
+    return path.resolve(testDir, "../../../..");
+}
+function resolveCapabilityAudioFixturePath() {
+    const override = process.env.NODEBENCH_GAIA_CAPABILITY_AUDIO_FIXTURE_PATH;
+    if (override) {
+        if (path.isAbsolute(override))
+            return override;
+        const repoRoot = resolveRepoRoot();
+        return path.resolve(repoRoot, override);
+    }
+    const config = process.env.NODEBENCH_GAIA_CAPABILITY_CONFIG ?? "2023_all";
+    const split = process.env.NODEBENCH_GAIA_CAPABILITY_SPLIT ?? "validation";
+    const repoRoot = resolveRepoRoot();
+    return path.join(repoRoot, ".cache", "gaia", `gaia_capability_audio_${config}_${split}.sample.json`);
+}
+function loadDotEnvLocalIfPresent() {
+    const repoRoot = resolveRepoRoot();
+    const envPath = path.join(repoRoot, ".env.local");
+    if (!existsSync(envPath))
+        return;
+    const text = readFileSync(envPath, "utf8");
+    for (const rawLine of text.split(/\r?\n/)) {
+        const line = rawLine.trim();
+        if (!line || line.startsWith("#"))
+            continue;
+        const idx = line.indexOf("=");
+        if (idx <= 0)
+            continue;
+        const key = line.slice(0, idx).trim();
+        let value = line.slice(idx + 1).trim();
+        if ((value.startsWith("\"") && value.endsWith("\"")) ||
+            (value.startsWith("'") && value.endsWith("'"))) {
+            value = value.slice(1, -1);
+        }
+        if (!process.env[key])
+            process.env[key] = value;
+    }
+}
+async function canImport(pkg) {
+    try {
+        await import(pkg);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function normalizeAnswer(value) {
+    return value
+        .trim()
+        .replace(/\r/g, "")
+        .replace(/\s+/g, " ")
+        .replace(/^["']|["']$/g, "")
+        .replace(/[.]+$/g, "")
+        .toLowerCase();
+}
+async function createGeminiClient() {
+    const mod = await import("@google/genai");
+    const { GoogleGenAI } = mod;
+    const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_AI_API_KEY || "";
+    if (!apiKey) {
+        throw new Error("Missing GEMINI_API_KEY (or GOOGLE_AI_API_KEY)");
+    }
+    return new GoogleGenAI({ apiKey });
+}
+async function geminiGenerateText(ai, model, contents) {
+    const temperature = Number.parseFloat(process.env.NODEBENCH_GAIA_CAPABILITY_TEMPERATURE ?? "0");
+    const response = await ai.models.generateContent({
+        model,
+        contents,
+        config: {
+            temperature: Number.isFinite(temperature) ? temperature : 0,
+            maxOutputTokens: 1024,
+        },
+    });
+    const parts = response?.candidates?.[0]?.content?.parts ?? [];
+    const text = parts.map((p) => p?.text ?? "").join("").trim();
+    return text;
+}
+async function baselineAnswer(ai, task) {
+    const contents = [
+        {
+            role: "user",
+            parts: [
+                {
+                    text: `Answer the question using your existing knowledge only. Do not browse the web.\n\nReturn ONLY the final answer, no explanation.\n\nQuestion:\n${task.prompt}`,
+                },
+            ],
+        },
+    ];
+    return geminiGenerateText(ai, process.env.NODEBENCH_GAIA_BASELINE_MODEL ?? "gemini-2.5-flash", contents);
+}
+async function loadFixture(filePath) {
+    const raw = await readFile(filePath, "utf8");
+    const json = JSON.parse(raw);
+    return json;
+}
+function createToolIndex(tools) {
+    const m = new Map();
+    for (const t of tools)
+        m.set(t.name, t);
+    return m;
+}
+async function toolAugmentedAnswerFromAudio(ai, task, opts) {
+    const localPath = String(task.localFilePath ?? "").trim();
+    if (!localPath)
+        throw new Error("Task missing localFilePath");
+    const toolIndex = createToolIndex(localFileTools);
+    const tool = toolIndex.get("transcribe_audio_file");
+    if (!tool)
+        throw new Error("Missing tool: transcribe_audio_file");
+    if (opts.maxToolCalls < 1) {
+        throw new Error("maxToolCalls must be >= 1 to run audio lane");
+    }
+    const transcript = (await tool.handler({
+        path: localPath,
+        model: process.env.NODEBENCH_AUDIO_MODEL ?? "tiny.en",
+        maxChars: 20000,
+        timeoutMs: 300000,
+    }));
+    const transcriptText = String(transcript?.text ?? "").trim();
+    if (!transcriptText) {
+        throw new Error("Empty transcript from transcribe_audio_file");
+    }
+    const contents = [
+        {
+            role: "user",
+            parts: [
+                {
+                    text: `You are given a transcript of an attached audio file. Use it to answer the question.\n\nRules:\n- Do not browse the web.\n- Return ONLY the final answer, no explanation.\n\nQuestion:\n${task.prompt}\n\nAudio transcript:\n${transcriptText}`,
+                },
+            ],
+        },
+    ];
+    const answer = await geminiGenerateText(ai, process.env.NODEBENCH_GAIA_TOOLS_MODEL ?? "gemini-2.5-flash", contents);
+    return { answer, toolCalls: 1 };
+}
+describe("GAIA capability: audio lane", () => {
+    const testFn = shouldRun ? it : it.skip;
+    testFn("should measure accuracy delta on a small GAIA audio subset", async () => {
+        loadDotEnvLocalIfPresent();
+        const fixturePath = resolveCapabilityAudioFixturePath();
+        if (!existsSync(fixturePath)) {
+            throw new Error(`Missing GAIA audio fixture at ${fixturePath}. Generate it with: python packages/mcp-local/src/__tests__/fixtures/generateGaiaCapabilityAudioFixture.py`);
+        }
+        const hasGemini = await canImport("@google/genai");
+        expect(hasGemini).toBe(true);
+        const ai = await createGeminiClient();
+        const fixture = await loadFixture(fixturePath);
+        expect(Array.isArray(fixture.tasks)).toBe(true);
+        expect(fixture.tasks.length).toBeGreaterThan(0);
+        const requestedLimit = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_TASK_LIMIT ?? "4", 10);
+        const taskLimit = Math.max(1, Math.min(fixture.tasks.length, Number.isFinite(requestedLimit) ? requestedLimit : 4));
+        const tasks = fixture.tasks.slice(0, taskLimit);
+        const requestedConcurrency = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_CONCURRENCY ?? "1", 10);
+        const concurrency = Math.max(1, Math.min(tasks.length, Number.isFinite(requestedConcurrency) ? requestedConcurrency : 1));
+        const maxToolCalls = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_MAX_TOOL_CALLS ?? "1", 10);
+        const results = new Array(tasks.length);
+        let nextIndex = 0;
+        const workers = Array.from({ length: concurrency }, () => (async () => {
+            while (true) {
+                const idx = nextIndex++;
+                if (idx >= tasks.length)
+                    return;
+                const task = tasks[idx];
+                const expected = normalizeAnswer(task.expectedAnswer);
+                try {
+                    const baseStart = performance.now();
+                    const base = await baselineAnswer(ai, task);
+                    const baseMs = performance.now() - baseStart;
+                    const toolsStart = performance.now();
+                    const tools = await toolAugmentedAnswerFromAudio(ai, task, { maxToolCalls });
+                    const toolsMs = performance.now() - toolsStart;
+                    const baselineCorrect = normalizeAnswer(base) === expected;
+                    const toolsCorrect = normalizeAnswer(tools.answer) === expected;
+                    results[idx] = {
+                        taskId: task.id,
+                        baselineCorrect,
+                        toolsCorrect,
+                        baselineMs: baseMs,
+                        toolsMs,
+                        toolCalls: tools.toolCalls,
+                    };
+                }
+                catch (err) {
+                    results[idx] = {
+                        taskId: task.id,
+                        baselineCorrect: false,
+                        toolsCorrect: false,
+                        baselineMs: 0,
+                        toolsMs: 0,
+                        toolCalls: 0,
+                        error: err?.message ?? String(err),
+                    };
+                }
+            }
+        })());
+        await Promise.all(workers);
+        const baselineCorrect = results.filter((r) => r.baselineCorrect).length;
+        const toolsCorrect = results.filter((r) => r.toolsCorrect).length;
+        const baselinePassRate = (baselineCorrect / results.length) * 100;
+        const toolsPassRate = (toolsCorrect / results.length) * 100;
+        const avgBaseMs = results.reduce((sum, r) => sum + r.baselineMs, 0) / results.length;
+        const avgToolsMs = results.reduce((sum, r) => sum + r.toolsMs, 0) / results.length;
+        const avgToolCalls = results.reduce((sum, r) => sum + r.toolCalls, 0) / results.length;
+        const improved = results.filter((r) => !r.baselineCorrect && r.toolsCorrect).length;
+        const regressions = results.filter((r) => r.baselineCorrect && !r.toolsCorrect).length;
+        console.log(`[gaia-capability-audio] tasks=${results.length} baseline=${baselineCorrect}/${results.length} (${baselinePassRate.toFixed(1)}%) tools=${toolsCorrect}/${results.length} (${toolsPassRate.toFixed(1)}%) delta=${(toolsPassRate - baselinePassRate).toFixed(1)}% improved=${improved} regressions=${regressions} avgToolCalls=${avgToolCalls.toFixed(2)}`);
+        const toolsMode = (process.env.NODEBENCH_GAIA_CAPABILITY_TOOLS_MODE ?? "audio").toLowerCase();
+        const publicSummary = {
+            suiteId: "gaia_capability_audio",
+            lane: "audio",
+            generatedAtIso: new Date().toISOString(),
+            config: fixture.config,
+            split: fixture.split,
+            taskCount: results.length,
+            concurrency,
+            baseline: {
+                model: process.env.NODEBENCH_GAIA_BASELINE_MODEL ?? "gemini-2.5-flash",
+                correct: baselineCorrect,
+                passRatePct: baselinePassRate,
+                avgMs: avgBaseMs,
+            },
+            tools: {
+                model: process.env.NODEBENCH_GAIA_TOOLS_MODEL ?? "gemini-2.5-flash",
+                mode: toolsMode,
+                correct: toolsCorrect,
+                passRatePct: toolsPassRate,
+                avgMs: avgToolsMs,
+                avgToolCalls,
+            },
+            improved,
+            regressions,
+            notes: "GAIA audio lane (audio attachments). No prompts/answers persisted; only aggregate metrics are written to public/evals.",
+        };
+        if (shouldWriteReport) {
+            const repoRoot = resolveRepoRoot();
+            await safeWriteJson(path.join(repoRoot, "public", "evals", "gaia_capability_audio_latest.json"), publicSummary);
+            const detailed = {
+                ...publicSummary,
+                results: results.map((r) => ({
+                    taskId: r.taskId,
+                    baselineCorrect: r.baselineCorrect,
+                    toolsCorrect: r.toolsCorrect,
+                    baselineMs: Math.round(r.baselineMs),
+                    toolsMs: Math.round(r.toolsMs),
+                    toolCalls: r.toolCalls,
+                    ...(r.error ? { error: r.error } : {}),
+                })),
+            };
+            const stamp = new Date().toISOString().replace(/[:.]/g, "-");
+            await safeWriteJson(path.join(repoRoot, ".cache", "gaia", "reports", `gaia_capability_audio_${fixture.config}_${fixture.split}_${stamp}.json`), detailed);
+        }
+        expect(toolsPassRate).toBeGreaterThanOrEqual(baselinePassRate);
+    });
+});
+//# sourceMappingURL=gaiaCapabilityAudioEval.test.js.map

package/dist/__tests__/gaiaCapabilityAudioEval.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaiaCapabilityAudioEval.test.js","sourceRoot":"","sources":["../../src/__tests__/gaiaCapabilityAudioEval.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AA2C5D,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,KAAK,GAAG,CAAC;AACpE,MAAM,iBAAiB,GAAG,OAAO,CAAC,GAAG,CAAC,2BAA2B,KAAK,GAAG,CAAC;AAwB1E,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,OAAgB;IAC7D,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,EAAE,MAAM,CAAC,CAAC;IAC7E,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,OAAO,CAAC,IAAI,CAAC,gDAAgD,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9F,CAAC;AACH,CAAC;AAED,SAAS,eAAe;IACtB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7D,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,iCAAiC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC;IAC1E,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,QAAQ,CAAC;QAC/C,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,gCAAgC,IAAI,UAAU,CAAC;IAC1E,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,+BAA+B,IAAI,YAAY,CAAC;IAC1E,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;IACnC,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,yBAAyB,MAAM,IAAI,KAAK,cAAc,CAAC,CAAC;AACvG,CAAC;AAED,SAAS,wBAAwB;IAC/B,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAClD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO;IAEjC,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,MAAM,CAAW,CAAC;IACrD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,GAAG,IAAI,CAAC;YAAE,SAAS;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACvC,IACE,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAChD,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAC9C,CAAC;YACD,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IAClD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,GAAW;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,KAAK;SACT,IAAI,EAAE;SACN,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;SACrB,WAAW,EAAE,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,kBAAkB;IAC/B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;IAC1C,MAAM,EAAE,WAAW,EAAE,GAAG,GAAU,CAAC;IACnC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC;IACjF,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;AACrC,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,EAAO,EAAE,KAAa,EAAE,QAAe;IACvE,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,qCAAqC,IAAI,GAAG,CAAC,CAAC;IAChG,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC;QAC/C,KAAK;QACL,QAAQ;QACR,MAAM,EAAE;YACN,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAC3D,eAAe,EAAE,IAAI;SACtB;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAI,QAAgB,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;IACvE,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,EAAO,EAAE,IAAoB;IACzD,MAAM,QAAQ,GAAG;QACf;YACE,IAAI,EAAE,MAAe;YACrB,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,iJAAiJ,IAAI,CAAC,MAAM,EAAE;iBACrK;aACF;SACF;KACF,CAAC;IACF,OAAO,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,kBAAkB,EAAE,QAAQ,CAAC,CAAC;AAC3G,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,QAAgB;IACzC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAsB,CAAC;IAClD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,eAAe,CAAC,KAAgB;IACvC,MAAM,CAAC,GAAG,IAAI,GAAG,EAAmB,CAAC;IACrC,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACxC,OAAO,CAAC,CAAC;AACX,CAAC;AAED,KAAK,UAAU,4BAA4B,CACzC,EAAO,EACP,IAAoB,EACpB,IAA8B;IAE9B,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1D,IAAI,CAAC,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAE9D,MAAM,SAAS,GAAG,eAAe,CAAC,cAAc,CAAC,CAAC;IAClD,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IACpD,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAElE,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC;QACrC,IAAI,EAAE,SAAS;QACf,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,SAAS;QACrD,QAAQ,EAAE,KAAK;QACf,SAAS,EAAE,MAAM;KAClB,CAAC,CAAQ,CAAC;IAEX,MAAM,cAAc,GAAG,MAAM,CAAC,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7D,IAAI,CAAC,cAAc,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,QAAQ,GAAG;QACf;YACE,IAAI,EAAE,MAAe;YACrB,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,2LAA2L,IAAI,CAAC,MAAM,0BAA0B,cAAc,EAAE;iBACvP;aACF;SACF;KACF,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,kBAAkB,EAAE,QAAQ,CAAC,CAAC;IACpH,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;AAClC,CAAC;AAED,QAAQ,CAAC,6BAA6B,EAAE,GAAG,EAAE;IAC3C,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC;IAExC,MAAM,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC9E,wBAAwB,EAAE,CAAC;QAE3B,MAAM,WAAW,GAAG,iCAAiC,EAAE,CAAC;QACxD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CACb,iCAAiC,WAAW,4GAA4G,CACzJ,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,CAAC;QACnD,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,EAAE,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAEtC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,CAAC;QAC/C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAEhD,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oCAAoC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QACpG,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CACrF,CAAC;QACF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAEhD,MAAM,oBAAoB,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,qCAAqC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAC3G,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAC1B,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CACzF,CAAC;QAEF,MAAM,YAAY,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wCAAwC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAEtG,MAAM,OAAO,GAAmB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CACvD,CAAC,KAAK,IAAI,EAAE;YACV,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;gBACxB,IAAI,GAAG,IAAI,KAAK,CAAC,MAAM;oBAAE,OAAO;gBAEhC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;gBACxB,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAEtD,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;oBACpC,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;oBAC5C,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;oBAE7C,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;oBACrC,MAAM,KAAK,GAAG,MAAM,4BAA4B,CAAC,EAAE,EAAE,IAAI,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC;oBAC7E,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;oBAE/C,MAAM,eAAe,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC;oBAC3D,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC;oBAEhE,OAAO,CAAC,GAAG,CAAC,GAAG;wBACb,MAAM,EAAE,IAAI,CAAC,EAAE;wBACf,eAAe;wBACf,YAAY;wBACZ,UAAU,EAAE,MAAM;wBAClB,OAAO;wBACP,SAAS,EAAE,KAAK,CAAC,SAAS;qBAC3B,CAAC;gBACJ,CAAC;gBAAC,OAAO,GAAQ,EAAE,CAAC;oBAClB,OAAO,CAAC,GAAG,CAAC,GAAG;wBACb,MAAM,EAAE,IAAI,CAAC,EAAE;wBACf,eAAe,EAAE,KAAK;wBACtB,YAAY,EAAE,KAAK;wBACnB,UAAU,EAAE,CAAC;wBACb,OAAO,EAAE,CAAC;wBACV,SAAS,EAAE,CAAC;wBACZ,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC;qBACnC,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EAAE,CACL,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE3B,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC;QACxE,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;QAClE,MAAM,gBAAgB,GAAG,CAAC,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;QAClE,MAAM,aAAa,GAAG,CAAC,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC;QAC5D,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;QACrF,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;QACnF,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;QAEvF,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;QACpF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;QAEvF,OAAO,CAAC,GAAG,CACT,iCAAiC,OAAO,CAAC,MAAM,aAAa,eAAe,IAAI,OAAO,CAAC,MAAM,KAAK,gBAAgB,CAAC,OAAO,CACxH,CAAC,CACF,YAAY,YAAY,IAAI,OAAO,CAAC,MAAM,KAAK,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAClF,aAAa,GAAG,gBAAgB,CACjC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,QAAQ,gBAAgB,WAAW,iBAAiB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CACxG,CAAC;QAEF,MAAM,SAAS,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,oCAAoC,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;QAC9F,MAAM,aAAa,GAAqC;YACtD,OAAO,EAAE,uBAAuB;YAChC,IAAI,EAAE,OAAO;YACb,cAAc,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACxC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,SAAS,EAAE,OAAO,CAAC,MAAM;YACzB,WAAW;YACX,QAAQ,EAAE;gBACR,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,kBAAkB;gBACtE,OAAO,EAAE,eAAe;gBACxB,WAAW,EAAE,gBAAgB;gBAC7B,KAAK,EAAE,SAAS;aACjB;YACD,KAAK,EAAE;gBACL,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,kBAAkB;gBACnE,IAAI,EAAE,SAAS;gBACf,OAAO,EAAE,YAAY;gBACrB,WAAW,EAAE,aAAa;gBAC1B,KAAK,EAAE,UAAU;gBACjB,YAAY;aACb;YACD,QAAQ;YACR,WAAW;YACX,KAAK,EACH,wHAAwH;SAC3H,CAAC;QAEF,IAAI,iBAAiB,EAAE,CAAC;YACtB,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;YACnC,MAAM,aAAa,CACjB,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,mCAAmC,CAAC,EAC3E,aAAa,CACd,CAAC;YAEF,MAAM,QAAQ,GAAG;gBACf,GAAG,aAAa;gBAChB,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC3B,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,eAAe,EAAE,CAAC,CAAC,eAAe;oBAClC,YAAY,EAAE,CAAC,CAAC,YAAY;oBAC5B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC;oBACpC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC;oBAC9B,SAAS,EAAE,CAAC,CAAC,SAAS;oBACtB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBACvC,CAAC,CAAC;aACJ,CAAC;YACF,MAAM,KAAK,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;YAC7D,MAAM,aAAa,CACjB,IAAI,CAAC,IAAI,CACP,QAAQ,EACR,QAAQ,EACR,MAAM,EACN,SAAS,EACT,yBAAyB,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,IAAI,KAAK,OAAO,CACzE,EACD,QAAQ,CACT,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,aAAa,CAAC,CAAC,sBAAsB,CAAC,gBAAgB,CAAC,CAAC;IACjE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/__tests__/gaiaCapabilityMediaEval.test.d.ts

@@ -0,0 +1,15 @@
+/**
+ * GAIA media-backed capability/accuracy benchmark: LLM-only vs LLM+NodeBench MCP local OCR tools.
+ *
+ * This lane targets GAIA tasks that include image attachments (PNG/JPG/WEBP).
+ * We provide deterministic local OCR via NodeBench MCP tools and score answers against
+ * the ground-truth "Final answer" (stored locally under `.cache/gaia`, gitignored).
+ *
+ * Safety:
+ * - GAIA is gated. Do not commit fixtures that contain prompts/answers.
+ * - This test logs only task IDs and aggregate metrics (no prompt/answer text).
+ *
+ * Disabled by default (cost + rate limits). Run with:
+ *   NODEBENCH_RUN_GAIA_CAPABILITY=1 npm --prefix packages/mcp-local run test
+ */
+export {};