nodebench-mcp 1.4.1 → 2.1.0

package/dist/__tests__/gaiaCapabilityFilesEval.test.js

@@ -0,0 +1,303 @@
+/**
+ * GAIA file-backed capability/accuracy benchmark: LLM-only vs LLM+NodeBench MCP local file tools.
+ *
+ * This lane targets GAIA tasks that include attachments (PDF / XLSX / CSV).
+ * We provide deterministic local parsing via NodeBench MCP tools and score answers against
+ * the ground-truth "Final answer" (stored locally under `.cache/gaia`, gitignored).
+ *
+ * Safety:
+ * - GAIA is gated. Do not commit fixtures that contain prompts/answers.
+ * - This test logs only task IDs and aggregate metrics (no prompt/answer text).
+ *
+ * Disabled by default (cost + rate limits). Run with:
+ *   NODEBENCH_RUN_GAIA_CAPABILITY=1 npm --prefix packages/mcp-local run test
+ */
+import { describe, expect, it } from "vitest";
+import { existsSync, readFileSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { performance } from "node:perf_hooks";
+import { localFileTools } from "../tools/localFileTools.js";
+const shouldRun = process.env.NODEBENCH_RUN_GAIA_CAPABILITY === "1";
+function resolveRepoRoot() {
+    const testDir = path.dirname(fileURLToPath(import.meta.url));
+    return path.resolve(testDir, "../../../..");
+}
+function resolveCapabilityFilesFixturePath() {
+    const override = process.env.NODEBENCH_GAIA_CAPABILITY_FILES_FIXTURE_PATH;
+    if (override) {
+        if (path.isAbsolute(override))
+            return override;
+        const repoRoot = resolveRepoRoot();
+        return path.resolve(repoRoot, override);
+    }
+    const config = process.env.NODEBENCH_GAIA_CAPABILITY_CONFIG ?? "2023_all";
+    const split = process.env.NODEBENCH_GAIA_CAPABILITY_SPLIT ?? "validation";
+    const repoRoot = resolveRepoRoot();
+    return path.join(repoRoot, ".cache", "gaia", `gaia_capability_files_${config}_${split}.sample.json`);
+}
+function loadDotEnvLocalIfPresent() {
+    const repoRoot = resolveRepoRoot();
+    const envPath = path.join(repoRoot, ".env.local");
+    if (!existsSync(envPath))
+        return;
+    const text = readFileSync(envPath, "utf8");
+    for (const rawLine of text.split(/\r?\n/)) {
+        const line = rawLine.trim();
+        if (!line || line.startsWith("#"))
+            continue;
+        const idx = line.indexOf("=");
+        if (idx <= 0)
+            continue;
+        const key = line.slice(0, idx).trim();
+        let value = line.slice(idx + 1).trim();
+        if ((value.startsWith("\"") && value.endsWith("\"")) ||
+            (value.startsWith("'") && value.endsWith("'"))) {
+            value = value.slice(1, -1);
+        }
+        if (!process.env[key])
+            process.env[key] = value;
+    }
+}
+async function canImport(pkg) {
+    try {
+        await import(pkg);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function normalizeAnswer(value) {
+    return value
+        .trim()
+        .replace(/\r/g, "")
+        .replace(/\s+/g, " ")
+        .replace(/^["']|["']$/g, "")
+        .replace(/[.]+$/g, "")
+        .toLowerCase();
+}
+async function createGeminiClient() {
+    const mod = await import("@google/genai");
+    const { GoogleGenAI } = mod;
+    const apiKey = process.env.GEMINI_API_KEY || process.env.GOOGLE_AI_API_KEY || "";
+    if (!apiKey) {
+        throw new Error("Missing GEMINI_API_KEY (or GOOGLE_AI_API_KEY)");
+    }
+    return new GoogleGenAI({ apiKey });
+}
+async function geminiGenerateText(ai, model, contents) {
+    const temperature = Number.parseFloat(process.env.NODEBENCH_GAIA_CAPABILITY_TEMPERATURE ?? "0");
+    const response = await ai.models.generateContent({
+        model,
+        contents,
+        config: {
+            temperature: Number.isFinite(temperature) ? temperature : 0,
+            maxOutputTokens: 1024,
+        },
+    });
+    const parts = response?.candidates?.[0]?.content?.parts ?? [];
+    const text = parts.map((p) => p?.text ?? "").join("").trim();
+    return text;
+}
+async function baselineAnswer(ai, task) {
+    const contents = [
+        {
+            role: "user",
+            parts: [
+                {
+                    text: `Answer the question using your existing knowledge only. Do not browse the web.\n\nReturn ONLY the final answer, no explanation.\n\nQuestion:\n${task.prompt}`,
+                },
+            ],
+        },
+    ];
+    return geminiGenerateText(ai, process.env.NODEBENCH_GAIA_BASELINE_MODEL ?? "gemini-2.5-flash", contents);
+}
+function buildToolIndex() {
+    const byName = new Map();
+    for (const tool of localFileTools)
+        byName.set(tool.name, tool);
+    return byName;
+}
+function resolveTaskLocalFilePath(task) {
+    const repoRoot = resolveRepoRoot();
+    const rel = String(task.localFilePath ?? "").trim();
+    if (rel)
+        return path.resolve(repoRoot, rel);
+    // Fallback to the standard cache layout used by the fixture generator.
+    const filePath = String(task.filePath ?? "").trim();
+    if (!filePath)
+        throw new Error("Task missing filePath/localFilePath");
+    return path.join(repoRoot, ".cache", "gaia", "data", filePath);
+}
+async function toolAugmentedAnswerFromFile(ai, task) {
+    const toolIndex = buildToolIndex();
+    const model = process.env.NODEBENCH_GAIA_TOOLS_MODEL ?? "gemini-2.5-flash";
+    const localPath = resolveTaskLocalFilePath(task);
+    if (!existsSync(localPath)) {
+        throw new Error(`Missing attachment on disk. Expected at ${localPath}. Refresh with dataset:gaia:capability:files:refresh`);
+    }
+    const ext = String(task.fileExt ?? "").trim().toLowerCase() ||
+        path.extname(task.fileName || task.filePath || "").toLowerCase().replace(/^\./, "");
+    let extract;
+    let toolCalls = 0;
+    if (ext === "csv") {
+        const tool = toolIndex.get("read_csv_file");
+        if (!tool)
+            throw new Error("Missing tool: read_csv_file");
+        extract = await tool.handler({
+            path: localPath,
+            hasHeader: true,
+            maxRows: 500,
+            maxCols: 80,
+            maxCellChars: 2000,
+        });
+        toolCalls = 1;
+    }
+    else if (ext === "xlsx") {
+        const tool = toolIndex.get("read_xlsx_file");
+        if (!tool)
+            throw new Error("Missing tool: read_xlsx_file");
+        extract = await tool.handler({
+            path: localPath,
+            headerRow: 1,
+            maxRows: 500,
+            maxCols: 80,
+            maxCellChars: 2000,
+        });
+        toolCalls = 1;
+    }
+    else if (ext === "pdf") {
+        const tool = toolIndex.get("read_pdf_text");
+        if (!tool)
+            throw new Error("Missing tool: read_pdf_text");
+        extract = await tool.handler({
+            path: localPath,
+            pageStart: 1,
+            pageEnd: 12,
+            maxChars: 40000,
+        });
+        toolCalls = 1;
+    }
+    else {
+        throw new Error(`Unsupported attachment type: ${ext || "(unknown)"}`);
+    }
+    // Keep the model input bounded. The tools already return bounded previews,
+    // but JSON stringification can still be large on wide tables.
+    const extractText = JSON.stringify(extract).slice(0, 40000);
+    const contents = [
+        {
+            role: "user",
+            parts: [
+                {
+                    text: "Answer the question using ONLY the provided file extract. " +
+                        "If the extract is insufficient, make the best supported guess.\n\n" +
+                        "Return ONLY the final answer, no explanation.\n\n" +
+                        `TASK_ID: ${task.id}\n` +
+                        `FILE_TYPE: ${ext}\n` +
+                        `LOCAL_FILE_PATH: ${localPath}\n` +
+                        `QUESTION:\n${task.prompt}\n\n` +
+                        `FILE_EXTRACT_JSON:\n${extractText}`,
+                },
+            ],
+        },
+    ];
+    const answer = await geminiGenerateText(ai, model, contents);
+    return { answer, toolCalls };
+}
+async function loadFixture(fixturePath) {
+    const raw = await readFile(fixturePath, "utf8");
+    const parsed = JSON.parse(raw);
+    if (!parsed || !Array.isArray(parsed.tasks))
+        throw new Error("Invalid GAIA capability files fixture");
+    return parsed;
+}
+describe("Capability: GAIA accuracy (file-backed) (LLM-only vs LLM+local tools)", () => {
+    const testFn = shouldRun ? it : it.skip;
+    testFn("should measure accuracy delta on a small GAIA file-backed subset", async () => {
+        loadDotEnvLocalIfPresent();
+        const fixturePath = resolveCapabilityFilesFixturePath();
+        if (!existsSync(fixturePath)) {
+            throw new Error(`Missing GAIA capability files fixture at ${fixturePath}. Generate it with: python packages/mcp-local/src/__tests__/fixtures/generateGaiaCapabilityFilesFixture.py`);
+        }
+        const hasGemini = await canImport("@google/genai");
+        expect(hasGemini).toBe(true);
+        const ai = await createGeminiClient();
+        const fixture = await loadFixture(fixturePath);
+        expect(Array.isArray(fixture.tasks)).toBe(true);
+        expect(fixture.tasks.length).toBeGreaterThan(0);
+        const requestedLimit = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_TASK_LIMIT ?? "6", 10);
+        const taskLimit = Math.max(1, Math.min(fixture.tasks.length, Number.isFinite(requestedLimit) ? requestedLimit : 6));
+        const tasks = fixture.tasks.slice(0, taskLimit);
+        const requestedConcurrency = Number.parseInt(process.env.NODEBENCH_GAIA_CAPABILITY_CONCURRENCY ?? "1", 10);
+        const concurrency = Math.max(1, Math.min(tasks.length, Number.isFinite(requestedConcurrency) ? requestedConcurrency : 1));
+        const results = new Array(tasks.length);
+        let nextIndex = 0;
+        const workers = Array.from({ length: concurrency }, () => (async () => {
+            while (true) {
+                const idx = nextIndex++;
+                if (idx >= tasks.length)
+                    return;
+                const task = tasks[idx];
+                const expected = normalizeAnswer(task.expectedAnswer);
+                try {
+                    const baseStart = performance.now();
+                    const base = await baselineAnswer(ai, task);
+                    const baseMs = performance.now() - baseStart;
+                    const toolsStart = performance.now();
+                    const tools = await toolAugmentedAnswerFromFile(ai, task);
+                    const toolsMs = performance.now() - toolsStart;
+                    const baselineCorrect = normalizeAnswer(base) === expected;
+                    const toolsCorrect = normalizeAnswer(tools.answer) === expected;
+                    results[idx] = {
+                        taskId: task.id,
+                        baselineCorrect,
+                        toolsCorrect,
+                        baselineMs: baseMs,
+                        toolsMs,
+                        toolCalls: tools.toolCalls,
+                    };
+                }
+                catch (err) {
+                    results[idx] = {
+                        taskId: task.id,
+                        baselineCorrect: false,
+                        toolsCorrect: false,
+                        baselineMs: 0,
+                        toolsMs: 0,
+                        toolCalls: 0,
+                        error: err?.message ?? String(err),
+                    };
+                }
+            }
+        })());
+        await Promise.all(workers);
+        const baselineCorrect = results.filter((r) => r.baselineCorrect).length;
+        const toolsCorrect = results.filter((r) => r.toolsCorrect).length;
+        const improved = results.filter((r) => !r.baselineCorrect && r.toolsCorrect).length;
+        const regressions = results.filter((r) => r.baselineCorrect && !r.toolsCorrect).length;
+        const avg = (values) => values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length;
+        const avgBaseMs = avg(results.map((r) => r.baselineMs).filter((n) => n > 0));
+        const avgToolsMs = avg(results.map((r) => r.toolsMs).filter((n) => n > 0));
+        const avgToolCalls = avg(results.map((r) => r.toolCalls));
+        console.log(`[gaia-capability-files] config=${fixture.config} split=${fixture.split} tasks=${tasks.length} concurrency=${concurrency} baseline=${baselineCorrect}/${tasks.length} tools=${toolsCorrect}/${tasks.length} improved=${improved} regressions=${regressions} avgBaselineMs=${avgBaseMs.toFixed(0)} avgToolsMs=${avgToolsMs.toFixed(0)} avgToolCalls=${avgToolCalls.toFixed(2)}`);
+        console.log(`[gaia-capability-files] perTask: ${results
+            .map((r) => `${r.taskId}:B${r.baselineCorrect ? "1" : "0"}T${r.toolsCorrect ? "1" : "0"}${r.error ? "E" : ""}`)
+            .join(" ")}`);
+        const enforce = process.env.NODEBENCH_GAIA_CAPABILITY_ENFORCE === "1";
+        if (enforce) {
+            // For file-backed tasks, the baseline is expected to be low. We still enforce that
+            // tool-augmented performance is not worse than baseline and has at least one improvement.
+            const allowedRegression = Math.max(1, Math.floor(tasks.length * 0.2));
+            expect(improved).toBeGreaterThanOrEqual(1);
+            expect(toolsCorrect).toBeGreaterThanOrEqual(baselineCorrect - allowedRegression);
+            expect(toolsCorrect).toBeGreaterThanOrEqual(1);
+        }
+        else {
+            expect(results.length).toBe(tasks.length);
+            expect(results.some((r) => r.error)).toBe(false);
+        }
+    }, 300000);
+});
+//# sourceMappingURL=gaiaCapabilityFilesEval.test.js.map

package/dist/__tests__/gaiaCapabilityFilesEval.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"gaiaCapabilityFilesEval.test.js","sourceRoot":"","sources":["../../src/__tests__/gaiaCapabilityFilesEval.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,cAAc,EAAE,MAAM,4BAA4B,CAAC;AA2C5D,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,KAAK,GAAG,CAAC;AAEpE,SAAS,eAAe;IACtB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7D,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;AAC9C,CAAC;AAED,SAAS,iCAAiC;IACxC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,4CAA4C,CAAC;IAC1E,IAAI,QAAQ,EAAE,CAAC;QACb,IAAI,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,OAAO,QAAQ,CAAC;QAC/C,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;QACnC,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC1C,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,gCAAgC,IAAI,UAAU,CAAC;IAC1E,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,+BAA+B,IAAI,YAAY,CAAC;IAC1E,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;IACnC,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,yBAAyB,MAAM,IAAI,KAAK,cAAc,CAAC,CAAC;AACvG,CAAC;AAED,SAAS,wBAAwB;IAC/B,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAClD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;QAAE,OAAO;IAEjC,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,EAAE,MAAM,CAAW,CAAC;IACrD,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5C,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAC9B,IAAI,GAAG,IAAI,CAAC;YAAE,SAAS;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACvC,IACE,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YAChD,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,EAC9C,CAAC;YACD,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QAC7B,CAAC;QACD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IAClD,CAAC;AACH,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,GAAW;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC;QAClB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,KAAK;SACT,IAAI,EAAE;SACN,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;SAClB,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC;SACrB,WAAW,EAAE,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,kBAAkB;IAC/B,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,eAAe,CAAC,CAAC;IAC1C,MAAM,EAAE,WAAW,EAAE,GAAG,GAAU,CAAC;IACnC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,IAAI,EAAE,CAAC;IACjF,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;AACrC,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,EAAO,EAAE,KAAa,EAAE,QAAe;IACvE,MAAM,WAAW,GAAG,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,qCAAqC,IAAI,GAAG,CAAC,CAAC;IAChG,MAAM,QAAQ,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC;QAC/C,KAAK;QACL,QAAQ;QACR,MAAM,EAAE;YACN,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;YAC3D,eAAe,EAAE,IAAI;SACtB;KACF,CAAC,CAAC;IAEH,MAAM,KAAK,GAAI,QAAgB,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC;IACvE,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAClE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,EAAO,EAAE,IAAoB;IACzD,MAAM,QAAQ,GAAG;QACf;YACE,IAAI,EAAE,MAAe;YACrB,KAAK,EAAE;gBACL;oBACE,IAAI,EAAE,iJAAiJ,IAAI,CAAC,MAAM,EAAE;iBACrK;aACF;SACF;KACF,CAAC;IACF,OAAO,kBAAkB,CAAC,EAAE,EAAE,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,kBAAkB,EAAE,QAAQ,CAAC,CAAC;AAC3G,CAAC;AAED,SAAS,cAAc;IACrB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmB,CAAC;IAC1C,KAAK,MAAM,IAAI,IAAI,cAAc;QAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC/D,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAoB;IACpD,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAC;IACnC,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACpD,IAAI,GAAG;QAAE,OAAO,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IAE5C,uEAAuE;IACvE,MAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IACpD,IAAI,CAAC,QAAQ;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IACtE,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;AACjE,CAAC;AAED,KAAK,UAAU,2BAA2B,CACxC,EAAO,EACP,IAAoB;IAEpB,MAAM,SAAS,GAAG,cAAc,EAAE,CAAC;IACnC,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,kBAAkB,CAAC;IAE3E,MAAM,SAAS,GAAG,wBAAwB,CAAC,IAAI,CAAC,CAAC;IACjD,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CACb,2CAA2C,SAAS,sDAAsD,CAC3G,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GACP,MAAM,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE;QAC/C,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAEtF,IAAI,OAAY,CAAC;IACjB,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QAClB,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC1D,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAC3B,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,IAAI;YACf,OAAO,EAAE,GAAG;YACZ,OAAO,EAAE,EAAE;YACX,YAAY,EAAE,IAAI;SACnB,CAAC,CAAC;QACH,SAAS,GAAG,CAAC,CAAC;IAChB,CAAC;SAAM,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC7C,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAC3D,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAC3B,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,GAAG;YACZ,OAAO,EAAE,EAAE;YACX,YAAY,EAAE,IAAI;SACnB,CAAC,CAAC;QACH,SAAS,GAAG,CAAC,CAAC;IAChB,CAAC;SAAM,IAAI,GAAG,KAAK,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QAC1D,OAAO,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC;YAC3B,IAAI,EAAE,SAAS;YACf,SAAS,EAAE,CAAC;YACZ,OAAO,EAAE,EAAE;YACX,QAAQ,EAAE,KAAK;SAChB,CAAC,CAAC;QACH,SAAS,GAAG,CAAC,CAAC;IAChB,CAAC;SAAM,CAAC;QACN,MAAM,IAAI,KAAK,CAAC,gCAAgC,GAAG,IAAI,WAAW,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,2EAA2E;IAC3E,8DAA8D;IAC9D,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAG;QACf;YACE,IAAI,EAAE,MAAe;YACrB,KAAK,EAAE;gBACL;oBACE,IAAI,EACF,4DAA4D;wBAC5D,oEAAoE;wBACpE,mDAAmD;wBACnD,YAAY,IAAI,CAAC,EAAE,IAAI;wBACvB,cAAc,GAAG,IAAI;wBACrB,oBAAoB,SAAS,IAAI;wBACjC,cAAc,IAAI,CAAC,MAAM,MAAM;wBAC/B,uBAAuB,WAAW,EAAE;iBACvC;aACF;SACF;KACF,CAAC;IAEF,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;IAC7D,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;AAC/B,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,WAAmB;IAC5C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAsB,CAAC;IACpD,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,MAAc,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC/G,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,QAAQ,CAAC,uEAAuE,EAAE,GAAG,EAAE;IACrF,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC;IAExC,MAAM,CAAC,kEAAkE,EAAE,KAAK,IAAI,EAAE;QACpF,wBAAwB,EAAE,CAAC;QAE3B,MAAM,WAAW,GAAG,iCAAiC,EAAE,CAAC;QACxD,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CACb,4CAA4C,WAAW,4GAA4G,CACpK,CAAC;QACJ,CAAC;QAED,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,eAAe,CAAC,CAAC;QACnD,MAAM,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,EAAE,GAAG,MAAM,kBAAkB,EAAE,CAAC;QAEtC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,WAAW,CAAC,CAAC;QAC/C,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAEhD,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,oCAAoC,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QACpG,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpH,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAEhD,MAAM,oBAAoB,GAAG,MAAM,CAAC,QAAQ,CAC1C,OAAO,CAAC,GAAG,CAAC,qCAAqC,IAAI,GAAG,EACxD,EAAE,CACH,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAC1B,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CACzF,CAAC;QAEF,MAAM,OAAO,GAAmB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,GAAG,EAAE,CACvD,CAAC,KAAK,IAAI,EAAE;YACV,OAAO,IAAI,EAAE,CAAC;gBACZ,MAAM,GAAG,GAAG,SAAS,EAAE,CAAC;gBACxB,IAAI,GAAG,IAAI,KAAK,CAAC,MAAM;oBAAE,OAAO;gBAEhC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;gBACxB,MAAM,QAAQ,GAAG,eAAe,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;gBAEtD,IAAI,CAAC;oBACH,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;oBACpC,MAAM,IAAI,GAAG,MAAM,cAAc,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;oBAC5C,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;oBAE7C,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;oBACrC,MAAM,KAAK,GAAG,MAAM,2BAA2B,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;oBAC1D,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC;oBAE/C,MAAM,eAAe,GAAG,eAAe,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC;oBAC3D,MAAM,YAAY,GAAG,eAAe,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC;oBAEhE,OAAO,CAAC,GAAG,CAAC,GAAG;wBACb,MAAM,EAAE,IAAI,CAAC,EAAE;wBACf,eAAe;wBACf,YAAY;wBACZ,UAAU,EAAE,MAAM;wBAClB,OAAO;wBACP,SAAS,EAAE,KAAK,CAAC,SAAS;qBAC3B,CAAC;gBACJ,CAAC;gBAAC,OAAO,GAAQ,EAAE,CAAC;oBAClB,OAAO,CAAC,GAAG,CAAC,GAAG;wBACb,MAAM,EAAE,IAAI,CAAC,EAAE;wBACf,eAAe,EAAE,KAAK;wBACtB,YAAY,EAAE,KAAK;wBACnB,UAAU,EAAE,CAAC;wBACb,OAAO,EAAE,CAAC;wBACV,SAAS,EAAE,CAAC;wBACZ,KAAK,EAAE,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC;qBACnC,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EAAE,CACL,CAAC;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE3B,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,MAAM,CAAC;QACxE,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;QAClE,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;QACpF,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC;QAEvF,MAAM,GAAG,GAAG,CAAC,MAAgB,EAAE,EAAE,CAC/B,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAE9E,MAAM,SAAS,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC7E,MAAM,UAAU,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3E,MAAM,YAAY,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QAE1D,OAAO,CAAC,GAAG,CACT,kCAAkC,OAAO,CAAC,MAAM,UAAU,OAAO,CAAC,KAAK,UAAU,KAAK,CAAC,MAAM,gBAAgB,WAAW,aAAa,eAAe,IAAI,KAAK,CAAC,MAAM,UAAU,YAAY,IAAI,KAAK,CAAC,MAAM,aAAa,QAAQ,gBAAgB,WAAW,kBAAkB,SAAS,CAAC,OAAO,CAC3R,CAAC,CACF,eAAe,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,iBAAiB,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAChF,CAAC;QACF,OAAO,CAAC,GAAG,CACT,oCAAoC,OAAO;aACxC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;aAC9G,IAAI,CAAC,GAAG,CAAC,EAAE,CACf,CAAC;QAEF,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,iCAAiC,KAAK,GAAG,CAAC;QACtE,IAAI,OAAO,EAAE,CAAC;YACZ,mFAAmF;YACnF,0FAA0F;YAC1F,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;YACtE,MAAM,CAAC,QAAQ,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;YAC3C,MAAM,CAAC,YAAY,CAAC,CAAC,sBAAsB,CAAC,eAAe,GAAG,iBAAiB,CAAC,CAAC;YACjF,MAAM,CAAC,YAAY,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC;QACjD,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAC1C,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnD,CAAC;IACH,CAAC,EAAE,MAAM,CAAC,CAAC;AACb,CAAC,CAAC,CAAC"}

package/dist/__tests__/openDatasetParallelEvalGaia.test.d.ts

@@ -0,0 +1,7 @@
+/**
+ * Gated dataset benchmark for long-running tool-augmented tasks (GAIA lane).
+ *
+ * GAIA is a gated dataset. Fixtures are generated into `.cache/gaia` (gitignored)
+ * and this test intentionally avoids logging the raw question text to stdout.
+ */
+export {};

package/dist/__tests__/openDatasetParallelEvalGaia.test.js

@@ -0,0 +1,279 @@
+/**
+ * Gated dataset benchmark for long-running tool-augmented tasks (GAIA lane).
+ *
+ * GAIA is a gated dataset. Fixtures are generated into `.cache/gaia` (gitignored)
+ * and this test intentionally avoids logging the raw question text to stdout.
+ */
+import { describe, expect, it } from "vitest";
+import { existsSync } from "node:fs";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+import { verificationTools } from "../tools/verificationTools.js";
+import { reconTools } from "../tools/reconTools.js";
+import { evalTools } from "../tools/evalTools.js";
+import { qualityGateTools } from "../tools/qualityGateTools.js";
+import { flywheelTools } from "../tools/flywheelTools.js";
+import { learningTools } from "../tools/learningTools.js";
+import { documentationTools } from "../tools/documentationTools.js";
+import { agentBootstrapTools } from "../tools/agentBootstrapTools.js";
+import { createMetaTools } from "../tools/metaTools.js";
+const domainTools = [
+    ...verificationTools,
+    ...evalTools,
+    ...qualityGateTools,
+    ...learningTools,
+    ...flywheelTools,
+    ...reconTools,
+    ...documentationTools,
+    ...agentBootstrapTools,
+];
+const allTools = [...domainTools, ...createMetaTools(domainTools)];
+const openDatasetToolCallLog = [];
+function findTool(name) {
+    const tool = allTools.find((candidate) => candidate.name === name);
+    if (!tool)
+        throw new Error(`Tool not found: ${name}`);
+    return tool;
+}
+async function callTool(name, args, taskId, stage) {
+    const tool = findTool(name);
+    try {
+        const result = await tool.handler(args);
+        openDatasetToolCallLog.push({ taskId, tool: name, stage, success: true });
+        return result;
+    }
+    catch (error) {
+        openDatasetToolCallLog.push({ taskId, tool: name, stage, success: false });
+        throw error;
+    }
+}
+function buildDiscoveryQuery(task) {
+    const promptLower = task.prompt.toLowerCase();
+    const tags = [];
+    // These tags are intentionally high-level to avoid accidentally printing or persisting
+    // the full gated question text while still exercising tool discovery paths.
+    if (task.hasFile)
+        tags.push(`${task.fileExt || "file"} attachment`);
+    if (promptLower.includes("wikipedia"))
+        tags.push("wikipedia lookup");
+    if (promptLower.includes("github"))
+        tags.push("github lookup");
+    if (promptLower.includes("youtube") || promptLower.includes("video"))
+        tags.push("video analysis");
+    if (promptLower.includes("spreadsheet") || task.fileExt === "xlsx")
+        tags.push("spreadsheet analysis");
+    if (promptLower.includes("pdf") || task.fileExt === "pdf")
+        tags.push("pdf parsing");
+    if (promptLower.includes("image") || ["png", "jpg", "jpeg", "webp"].includes(task.fileExt))
+        tags.push("image ocr");
+    if (promptLower.includes("zip code") || promptLower.includes("zipcode"))
+        tags.push("zip code lookup");
+    if (promptLower.includes("calculate") || promptLower.includes("round"))
+        tags.push("calculation");
+    const tagText = tags.length > 0 ? tags.join(", ") : "general tool-augmented reasoning";
+    return `GAIA task ${task.id} (level ${task.level || "?"}): ${tagText}`;
+}
+function redactedEvalInput(task) {
+    const bits = [
+        `GAIA task ${task.id} (prompt redacted)`,
+        `level=${task.level || "?"}`,
+        `questionLength=${task.questionLength}`,
+        `hasFile=${task.hasFile}`,
+        `fileExt=${task.fileExt || ""}`,
+        `steps=${task.annotator?.numberOfSteps ?? 0}`,
+        `tools=${task.annotator?.numberOfTools ?? 0}`,
+    ];
+    return bits.join(" | ");
+}
+async function loadGaiaFixture(fixturePath) {
+    const raw = await readFile(fixturePath, "utf8");
+    const parsed = JSON.parse(raw);
+    if (!parsed || !Array.isArray(parsed.tasks)) {
+        throw new Error("Invalid GAIA fixture payload");
+    }
+    return parsed;
+}
+async function runDatasetTask(fixture, task, workerIndex) {
+    const started = Date.now();
+    const recon = (await callTool("run_recon", {
+        target: `GAIA task ${task.id}`,
+        description: `Gated long-running benchmark task (${fixture.config}/${fixture.split}).`,
+        projectContext: {
+            techStack: "TypeScript, MCP, SQLite",
+            architecture: "MCP orchestration benchmark with parallel subagent workers",
+        },
+    }, task.id, "recon_start"));
+    await callTool("log_recon_finding", {
+        sessionId: recon.sessionId,
+        category: "dataset",
+        summary: `Ingested GAIA task ${task.id} (level=${task.level}, questionLength=${task.questionLength}, hasFile=${task.hasFile}, ext=${task.fileExt}).`,
+        sourceUrl: fixture.sourceUrl,
+        relevance: "Tool-augmented multi-step benchmark lane (GAIA).",
+        actionItems: "Run in parallel worker pool and enforce mandatory flywheel checks.",
+    }, task.id, "recon_log");
+    let discovered = (await callTool("findTools", {
+        query: buildDiscoveryQuery(task),
+        category: "bootstrap",
+    }, task.id, "find_tools"));
+    if (!Array.isArray(discovered?.tools) || discovered.tools.length === 0) {
+        discovered = (await callTool("findTools", { query: "tool-augmented reasoning with files web and computation", category: "verification" }, task.id, "find_tools_fallback"));
+    }
+    expect(Array.isArray(discovered.tools)).toBe(true);
+    expect(discovered.tools.length).toBeGreaterThan(0);
+    const methodology = (await callTool("getMethodology", { topic: "mandatory_flywheel" }, task.id, "get_methodology"));
+    expect(methodology.title).toBeTruthy();
+    expect(Array.isArray(methodology.steps)).toBe(true);
+    expect(methodology.steps.length).toBeGreaterThan(0);
+    const evalRun = (await callTool("start_eval_run", {
+        name: `open-dataset-gaia-${task.id}-${Date.now()}`,
+        description: `GAIA scenario (level=${task.level}, worker ${workerIndex})`,
+        cases: [
+            {
+                input: redactedEvalInput(task),
+                intent: `Coordinate long-running GAIA workflow for ${task.id}`,
+                expected: "Discover tool strategy, run eval bookkeeping, and enforce mandatory flywheel checks.",
+            },
+        ],
+    }, task.id, "start_eval_run"));
+    await callTool("record_eval_result", {
+        caseId: evalRun.caseIds[0],
+        verdict: "pass",
+        score: 1,
+        actual: `Discovered ${discovered.tools.length} tools and completed GAIA workflow bookkeeping.`,
+        telemetry: {
+            dataset: fixture.dataset,
+            config: fixture.config,
+            split: fixture.split,
+            taskId: task.id,
+            level: task.level,
+            workerIndex,
+            questionLength: task.questionLength,
+            hasFile: task.hasFile,
+            fileExt: task.fileExt,
+            numberOfSteps: task.annotator?.numberOfSteps ?? 0,
+            numberOfTools: task.annotator?.numberOfTools ?? 0,
+            complexityScore: task.complexityScore,
+        },
+    }, task.id, "record_eval_result");
+    const evalSummary = (await callTool("complete_eval_run", { runId: evalRun.runId }, task.id, "complete_eval_run"));
+    expect(evalSummary.status).toBe("completed");
+    expect(evalSummary.summary.passed).toBe(1);
+    const closedLoop = (await callTool("run_closed_loop", {
+        steps: [
+            { step: "compile", passed: true, output: `Compile checks for ${task.id}` },
+            { step: "lint", passed: true, output: `Lint checks for ${task.id}` },
+            { step: "test", passed: true, output: `Parallel benchmark checks for ${task.id}` },
+        ],
+    }, task.id, "run_closed_loop"));
+    expect(closedLoop.allPassed).toBe(true);
+    const flywheel = (await callTool("run_mandatory_flywheel", {
+        target: `Gated GAIA task ${task.id}`,
+        steps: [
+            { stepName: "static_analysis", passed: true, output: "Types and schemas validated." },
+            { stepName: "happy_path_test", passed: true, output: "Benchmark workflow completed." },
+            { stepName: "failure_path_test", passed: true, output: "Fallback discovery query validated." },
+            { stepName: "gap_analysis", passed: true, output: "No blocking gaps for this task." },
+            { stepName: "fix_and_reverify", passed: true, output: "No rework required after checks." },
+            { stepName: "deploy_and_document", passed: true, output: "Benchmark result documented." },
+        ],
+    }, task.id, "run_mandatory_flywheel"));
+    expect(flywheel.passed).toBe(true);
+    const knowledge = (await callTool("search_all_knowledge", { query: task.id, limit: 10 }, task.id, "search_all_knowledge"));
+    expect(typeof knowledge.totalResults).toBe("number");
+    expect(knowledge.totalResults).toBeGreaterThan(0);
+    return {
+        taskId: task.id,
+        workerIndex,
+        ok: true,
+        elapsedMs: Date.now() - started,
+        discoveredTools: discovered.tools.length,
+        knowledgeHits: knowledge.totalResults,
+    };
+}
+async function runWorkerPool(fixture, tasks, concurrency) {
+    const boundedConcurrency = Math.max(1, Math.min(concurrency, tasks.length));
+    const results = new Array(tasks.length);
+    let nextIndex = 0;
+    const workers = Array.from({ length: boundedConcurrency }, (_, workerIndex) => (async () => {
+        while (true) {
+            const taskIndex = nextIndex++;
+            if (taskIndex >= tasks.length)
+                return;
+            const task = tasks[taskIndex];
+            try {
+                results[taskIndex] = await runDatasetTask(fixture, task, workerIndex);
+            }
+            catch (error) {
+                results[taskIndex] = {
+                    taskId: task.id,
+                    workerIndex,
+                    ok: false,
+                    elapsedMs: 0,
+                    discoveredTools: 0,
+                    knowledgeHits: 0,
+                    error: error instanceof Error ? error.message : String(error),
+                };
+            }
+        }
+    })());
+    await Promise.all(workers);
+    return results;
+}
+function resolveGaiaFixturePath() {
+    const fixtureOverride = process.env.NODEBENCH_GAIA_FIXTURE_PATH;
+    if (fixtureOverride)
+        return fixtureOverride;
+    const config = process.env.NODEBENCH_GAIA_CONFIG ?? "2023_level3";
+    const split = process.env.NODEBENCH_GAIA_SPLIT ?? "validation";
+    const testDir = path.dirname(fileURLToPath(import.meta.url));
+    const repoRoot = path.resolve(testDir, "../../../..");
+    return path.join(repoRoot, ".cache", "gaia", `gaia_${config}_${split}.sample.json`);
+}
+const gaiaFixturePath = resolveGaiaFixturePath();
+const hasGaiaFixture = existsSync(gaiaFixturePath);
+describe("Scenario: GAIA (Gated) Long-Running Dataset (Parallel Subagents)", () => {
+    const testFn = hasGaiaFixture ? it : it.skip;
+    testFn("should execute GAIA tasks with parallel MCP subagent workflows", async () => {
+        const fixture = await loadGaiaFixture(gaiaFixturePath);
+        expect(Array.isArray(fixture.tasks)).toBe(true);
+        expect(fixture.tasks.length).toBeGreaterThan(0);
+        const requestedTaskLimit = Number.parseInt(process.env.NODEBENCH_GAIA_TASK_LIMIT ?? "8", 10);
+        const taskLimit = Math.max(1, Math.min(fixture.tasks.length, Number.isFinite(requestedTaskLimit) ? requestedTaskLimit : 8));
+        const requestedConcurrency = Number.parseInt(process.env.NODEBENCH_GAIA_CONCURRENCY ?? "4", 10);
+        const concurrency = Math.max(1, Math.min(taskLimit, Number.isFinite(requestedConcurrency) ? requestedConcurrency : 4));
+        const tasks = fixture.tasks.slice(0, taskLimit);
+        const started = Date.now();
+        const results = await runWorkerPool(fixture, tasks, concurrency);
+        const elapsedMs = Date.now() - started;
+        const failed = results.filter((result) => !result.ok);
+        const passed = results.filter((result) => result.ok);
+        const calledTools = new Set(openDatasetToolCallLog.map((entry) => entry.tool));
+        const requiredTools = [
+            "run_recon",
+            "log_recon_finding",
+            "findTools",
+            "getMethodology",
+            "start_eval_run",
+            "record_eval_result",
+            "complete_eval_run",
+            "run_closed_loop",
+            "run_mandatory_flywheel",
+            "search_all_knowledge",
+        ];
+        console.log(`[open-dataset] dataset=${fixture.dataset} config=${fixture.config} split=${fixture.split} tasks=${taskLimit} concurrency=${concurrency} pass=${passed.length}/${results.length} elapsedMs=${elapsedMs} toolCalls=${openDatasetToolCallLog.length}`);
+        if (failed.length > 0) {
+            console.error("[open-dataset] failures:", failed.map((result) => ({
+                taskId: result.taskId,
+                workerIndex: result.workerIndex,
+                error: result.error,
+            })));
+        }
+        expect(failed.length).toBe(0);
+        expect(passed.length).toBe(taskLimit);
+        for (const requiredTool of requiredTools) {
+            expect(calledTools.has(requiredTool)).toBe(true);
+        }
+    });
+});
+//# sourceMappingURL=openDatasetParallelEvalGaia.test.js.map

package/dist/__tests__/openDatasetParallelEvalGaia.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openDatasetParallelEvalGaia.test.js","sourceRoot":"","sources":["../../src/__tests__/openDatasetParallelEvalGaia.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AA+CxD,MAAM,WAAW,GAAc;IAC7B,GAAG,iBAAiB;IACpB,GAAG,SAAS;IACZ,GAAG,gBAAgB;IACnB,GAAG,aAAa;IAChB,GAAG,aAAa;IAChB,GAAG,UAAU;IACb,GAAG,kBAAkB;IACrB,GAAG,mBAAmB;CACvB,CAAC;AACF,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC;AAEnE,MAAM,sBAAsB,GAKvB,EAAE,CAAC;AAER,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;IACtD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,IAAY,EACZ,IAA6B,EAC7B,MAAc,EACd,KAAa;IAEb,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACxC,sBAAsB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1E,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sBAAsB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3E,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,mBAAmB,CAAC,IAAiB;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC;IAC9C,MAAM,IAAI,GAAa,EAAE,CAAC;IAE1B,uFAAuF;IACvF,4EAA4E;IAC5E,IAAI,IAAI,CAAC,OAAO;QAAE,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,OAAO,IAAI,MAAM,aAAa,CAAC,CAAC;IACpE,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;QAAE,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACrE,IAAI,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC/D,IAAI,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAClG,IAAI,WAAW,CAAC,QAAQ,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,MAAM;QAAE,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;IACtG,IAAI,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,KAAK,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACpF,IAAI,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC;QACxF,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACzB,IAAI,WAAW,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IACtG,IAAI,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC;QAAE,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAEjG,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,kCAAkC,CAAC;IACvF,OAAO,aAAa,IAAI,CAAC,EAAE,WAAW,IAAI,CAAC,KAAK,IAAI,GAAG,MAAM,OAAO,EAAE,CAAC;AACzE,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAiB;IAC1C,MAAM,IAAI,GAAG;QACX,aAAa,IAAI,CAAC,EAAE,oBAAoB;QACxC,SAAS,IAAI,CAAC,KAAK,IAAI,GAAG,EAAE;QAC5B,kBAAkB,IAAI,CAAC,cAAc,EAAE;QACvC,WAAW,IAAI,CAAC,OAAO,EAAE;QACzB,WAAW,IAAI,CAAC,OAAO,IAAI,EAAE,EAAE;QAC/B,SAAS,IAAI,CAAC,SAAS,EAAE,aAAa,IAAI,CAAC,EAAE;QAC7C,SAAS,IAAI,CAAC,SAAS,EAAE,aAAa,IAAI,CAAC,EAAE;KAC9C,CAAC;IACF,OAAO,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,WAAmB;IAChD,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;IAChD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAmB,CAAC;IACjD,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAE,MAAc,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,OAAuB,EACvB,IAAiB,EACjB,WAAmB;IAEnB,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE3B,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAC3B,WAAW,EACX;QACE,MAAM,EAAE,aAAa,IAAI,CAAC,EAAE,EAAE;QAC9B,WAAW,EAAE,sCAAsC,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,KAAK,IAAI;QACtF,cAAc,EAAE;YACd,SAAS,EAAE,yBAAyB;YACpC,YAAY,EAAE,4DAA4D;SAC3E;KACF,EACD,IAAI,CAAC,EAAE,EACP,aAAa,CACd,CAAQ,CAAC;IAEV,MAAM,QAAQ,CACZ,mBAAmB,EACnB;QACE,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,QAAQ,EAAE,SAAS;QACnB,OAAO,EAAE,sBAAsB,IAAI,CAAC,EAAE,WAAW,IAAI,CAAC,KAAK,oBAAoB,IAAI,CAAC,cAAc,aAAa,IAAI,CAAC,OAAO,SAAS,IAAI,CAAC,OAAO,IAAI;QACpJ,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,SAAS,EAAE,kDAAkD;QAC7D,WAAW,EAAE,oEAAoE;KAClF,EACD,IAAI,CAAC,EAAE,EACP,WAAW,CACZ,CAAC;IAEF,IAAI,UAAU,GAAG,CAAC,MAAM,QAAQ,CAC9B,WAAW,EACX;QACE,KAAK,EAAE,mBAAmB,CAAC,IAAI,CAAC;QAChC,QAAQ,EAAE,WAAW;KACtB,EACD,IAAI,CAAC,EAAE,EACP,YAAY,CACb,CAAQ,CAAC;IAEV,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvE,UAAU,GAAG,CAAC,MAAM,QAAQ,CAC1B,WAAW,EACX,EAAE,KAAK,EAAE,yDAAyD,EAAE,QAAQ,EAAE,cAAc,EAAE,EAC9F,IAAI,CAAC,EAAE,EACP,qBAAqB,CACtB,CAAQ,CAAC;IACZ,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnD,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAEnD,MAAM,WAAW,GAAG,CAAC,MAAM,QAAQ,CACjC,gBAAgB,EAChB,EAAE,KAAK,EAAE,oBAAoB,EAAE,EAC/B,IAAI,CAAC,EAAE,EACP,iBAAiB,CAClB,CAAQ,CAAC;IACV,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;IACvC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpD,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAEpD,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAC7B,gBAAgB,EAChB;QACE,IAAI,EAAE,qBAAqB,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE;QAClD,WAAW,EAAE,wBAAwB,IAAI,CAAC,KAAK,YAAY,WAAW,GAAG;QACzE,KAAK,EAAE;YACL;gBACE,KAAK,EAAE,iBAAiB,CAAC,IAAI,CAAC;gBAC9B,MAAM,EAAE,6CAA6C,IAAI,CAAC,EAAE,EAAE;gBAC9D,QAAQ,EACN,sFAAsF;aACzF;SACF;KACF,EACD,IAAI,CAAC,EAAE,EACP,gBAAgB,CACjB,CAAQ,CAAC;IAEV,MAAM,QAAQ,CACZ,oBAAoB,EACpB;QACE,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1B,OAAO,EAAE,MAAM;QACf,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,cAAc,UAAU,CAAC,KAAK,CAAC,MAAM,iDAAiD;QAC9F,SAAS,EAAE;YACT,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,WAAW;YACX,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,aAAa,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,IAAI,CAAC;YACjD,aAAa,EAAE,IAAI,CAAC,SAAS,EAAE,aAAa,IAAI,CAAC;YACjD,eAAe,EAAE,IAAI,CAAC,eAAe;SACtC;KACF,EACD,IAAI,CAAC,EAAE,EACP,oBAAoB,CACrB,CAAC;IAEF,MAAM,WAAW,GAAG,CAAC,MAAM,QAAQ,CACjC,mBAAmB,EACnB,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EACxB,IAAI,CAAC,EAAE,EACP,mBAAmB,CACpB,CAAQ,CAAC;IACV,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE3C,MAAM,UAAU,GAAG,CAAC,MAAM,QAAQ,CAChC,iBAAiB,EACjB;QACE,KAAK,EAAE;YACL,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,sBAAsB,IAAI,CAAC,EAAE,EAAE,EAAE;YAC1E,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,IAAI,CAAC,EAAE,EAAE,EAAE;YACpE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,iCAAiC,IAAI,CAAC,EAAE,EAAE,EAAE;SACnF;KACF,EACD,IAAI,CAAC,EAAE,EACP,iBAAiB,CAClB,CAAQ,CAAC;IACV,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAExC,MAAM,QAAQ,GAAG,CAAC,MAAM,QAAQ,CAC9B,wBAAwB,EACxB;QACE,MAAM,EAAE,mBAAmB,IAAI,CAAC,EAAE,EAAE;QACpC,KAAK,EAAE;YACL,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,8BAA8B,EAAE;YACrF,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,+BAA+B,EAAE;YACtF,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,qCAAqC,EAAE;YAC9F,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,iCAAiC,EAAE;YACrF,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,kCAAkC,EAAE;YAC1F,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,8BAA8B,EAAE;SAC1F;KACF,EACD,IAAI,CAAC,EAAE,EACP,wBAAwB,CACzB,CAAQ,CAAC;IACV,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,SAAS,GAAG,CAAC,MAAM,QAAQ,CAC/B,sBAAsB,EACtB,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAC7B,IAAI,CAAC,EAAE,EACP,sBAAsB,CACvB,CAAQ,CAAC;IACV,MAAM,CAAC,OAAO,SAAS,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrD,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAElD,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,WAAW;QACX,EAAE,EAAE,IAAI;QACR,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;QAC/B,eAAe,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM;QACxC,aAAa,EAAE,SAAS,CAAC,YAAY;KACtC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,OAAuB,EACvB,KAAoB,EACpB,WAAmB;IAEnB,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAmB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxD,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,CAC5E,CAAC,KAAK,IAAI,EAAE;QACV,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC;YAC9B,IAAI,SAAS,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAEtC,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;YAC9B,IAAI,CAAC;gBACH,OAAO,CAAC,SAAS,CAAC,GAAG,MAAM,cAAc,CAAC,OAAO,EAAE,IAAI,EAAE,WAAW,CAAC,CAAC;YACxE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,SAAS,CAAC,GAAG;oBACnB,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,WAAW;oBACX,EAAE,EAAE,KAAK;oBACT,SAAS,EAAE,CAAC;oBACZ,eAAe,EAAE,CAAC;oBAClB,aAAa,EAAE,CAAC;oBAChB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC9D,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC,CAAC,EAAE,CACL,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,sBAAsB;IAC7B,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC;IAChE,IAAI,eAAe;QAAE,OAAO,eAAe,CAAC;IAE5C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,aAAa,CAAC;IAClE,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,oBAAoB,IAAI,YAAY,CAAC;IAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7D,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IACtD,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,MAAM,IAAI,KAAK,cAAc,CAAC,CAAC;AACtF,CAAC;AAED,MAAM,eAAe,GAAG,sBAAsB,EAAE,CAAC;AACjD,MAAM,cAAc,GAAG,UAAU,CAAC,eAAe,CAAC,CAAC;AAEnD,QAAQ,CAAC,kEAAkE,EAAE,GAAG,EAAE;IAChF,MAAM,MAAM,GAAG,cAAc,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC;IAE7C,MAAM,CAAC,gEAAgE,EAAE,KAAK,IAAI,EAAE;QAClF,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,eAAe,CAAC,CAAC;QACvD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAEhD,MAAM,kBAAkB,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAC7F,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,CAAC,EACD,IAAI,CAAC,GAAG,CACN,OAAO,CAAC,KAAK,CAAC,MAAM,EACpB,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAC7D,CACF,CAAC;QAEF,MAAM,oBAAoB,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,GAAG,EAAE,EAAE,CAAC,CAAC;QAChG,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAC1B,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CACtF,CAAC;QAEF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,CAAC;QACjE,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAEvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAErD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/E,MAAM,aAAa,GAAG;YACpB,WAAW;YACX,mBAAmB;YACnB,WAAW;YACX,gBAAgB;YAChB,gBAAgB;YAChB,oBAAoB;YACpB,mBAAmB;YACnB,iBAAiB;YACjB,wBAAwB;YACxB,sBAAsB;SACvB,CAAC;QAEF,OAAO,CAAC,GAAG,CACT,0BAA0B,OAAO,CAAC,OAAO,WAAW,OAAO,CAAC,MAAM,UAAU,OAAO,CAAC,KAAK,UAAU,SAAS,gBAAgB,WAAW,SAAS,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,cAAc,SAAS,cAAc,sBAAsB,CAAC,MAAM,EAAE,CACpP,CAAC;QAEF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CACX,0BAA0B,EAC1B,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACtB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC,CACJ,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEtC,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;YACzC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/__tests__/openDatasetPerfComparison.test.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Performance comparison: "no tools" (noop handlers) vs NodeBench MCP toolchain.
+ *
+ * Notes:
+ * - This is a wall-clock micro-benchmark for local harness overhead, not capability/accuracy.
+ * - Disabled by default to avoid noisy perf output in normal test runs.
+ *   Set NODEBENCH_RUN_PERF_COMPARE=1 to run.
+ * - GAIA is gated; if a local GAIA fixture exists in `.cache/gaia`, it will be included.
+ */
+export {};