nodebench-mcp 3.0.0 → 3.0.2

package/dist/__tests__/openDatasetParallelEvalSwebench.test.js

@@ -1,220 +0,0 @@
-/**
- * Open-source dataset benchmark for long-running tasks (SWE-bench lane).
- *
- * This test uses SWE-bench Verified issue tasks and runs task workflows
- * through NodeBench MCP tools in parallel "subagent" workers.
- */
-import { describe, expect, it } from "vitest";
-import datasetFixture from "./fixtures/swebench_verified.sample.json";
-import { verificationTools } from "../tools/verificationTools.js";
-import { reconTools } from "../tools/reconTools.js";
-import { evalTools } from "../tools/evalTools.js";
-import { qualityGateTools } from "../tools/qualityGateTools.js";
-import { flywheelTools } from "../tools/flywheelTools.js";
-import { learningTools } from "../tools/learningTools.js";
-import { documentationTools } from "../tools/documentationTools.js";
-import { agentBootstrapTools } from "../tools/agentBootstrapTools.js";
-import { createMetaTools } from "../tools/metaTools.js";
-const fixture = datasetFixture;
-const domainTools = [
-    ...verificationTools,
-    ...evalTools,
-    ...qualityGateTools,
-    ...learningTools,
-    ...flywheelTools,
-    ...reconTools,
-    ...documentationTools,
-    ...agentBootstrapTools,
-];
-const allTools = [...domainTools, ...createMetaTools(domainTools)];
-const openDatasetToolCallLog = [];
-function findTool(name) {
-    const tool = allTools.find((candidate) => candidate.name === name);
-    if (!tool)
-        throw new Error(`Tool not found: ${name}`);
-    return tool;
-}
-async function callTool(name, args, taskId, stage) {
-    const tool = findTool(name);
-    try {
-        const result = await tool.handler(args);
-        openDatasetToolCallLog.push({ taskId, tool: name, stage, success: true });
-        return result;
-    }
-    catch (error) {
-        openDatasetToolCallLog.push({ taskId, tool: name, stage, success: false });
-        throw error;
-    }
-}
-async function runDatasetTask(task, workerIndex) {
-    const started = Date.now();
-    const recon = (await callTool("run_recon", {
-        target: `SWE-bench Verified task ${task.id}`,
-        description: `Open-source long-running engineering benchmark (${task.repo}, ${task.difficulty}).`,
-        projectContext: {
-            techStack: "TypeScript, MCP, SQLite",
-            architecture: "Parallel subagent MCP workflow benchmark",
-        },
-    }, task.id, "recon_start"));
-    await callTool("log_recon_finding", {
-        sessionId: recon.sessionId,
-        category: "dataset",
-        summary: `Ingested SWE-bench task ${task.id} (${task.repo}) with failToPass=${task.failToPassCount}.`,
-        sourceUrl: `${fixture.sourceUrl}`,
-        relevance: "Real-world software issue benchmark for MCP orchestration quality.",
-        actionItems: "Run in parallel worker pool and enforce mandatory flywheel checks.",
-    }, task.id, "recon_log");
-    let discovered = (await callTool("findTools", {
-        query: task.prompt.slice(0, 600),
-        category: "verification",
-    }, task.id, "find_tools"));
-    if (!Array.isArray(discovered?.tools) || discovered.tools.length === 0) {
-        discovered = (await callTool("findTools", { query: "software bugfix verification eval workflow", category: "eval" }, task.id, "find_tools_fallback"));
-    }
-    expect(Array.isArray(discovered.tools)).toBe(true);
-    expect(discovered.tools.length).toBeGreaterThan(0);
-    const difficultyLower = task.difficulty.toLowerCase();
-    const methodologyTopic = difficultyLower.includes("1-4 hours") || task.complexityScore >= 140
-        ? "mandatory_flywheel"
-        : "verification";
-    const methodology = (await callTool("getMethodology", { topic: methodologyTopic }, task.id, "get_methodology"));
-    expect(methodology.title).toBeTruthy();
-    expect(Array.isArray(methodology.steps)).toBe(true);
-    expect(methodology.steps.length).toBeGreaterThan(0);
-    const evalRun = (await callTool("start_eval_run", {
-        name: `open-dataset-swebench-${task.id}-${Date.now()}`,
-        description: `SWE-bench scenario (${task.repo}, worker ${workerIndex})`,
-        cases: [
-            {
-                input: task.prompt,
-                intent: `Coordinate long-running SWE-bench workflow for ${task.id}`,
-                expected: "Discover strategy, run eval bookkeeping, and complete closed-loop plus mandatory flywheel checks.",
-            },
-        ],
-    }, task.id, "start_eval_run"));
-    await callTool("record_eval_result", {
-        caseId: evalRun.caseIds[0],
-        verdict: "pass",
-        score: 1,
-        actual: `Discovered ${discovered.tools.length} tools and completed SWE-bench workflow.`,
-        telemetry: {
-            dataset: fixture.dataset,
-            split: fixture.split,
-            taskId: task.id,
-            repo: task.repo,
-            difficulty: task.difficulty,
-            workerIndex,
-            statementLength: task.statementLength,
-            hintLength: task.hintLength,
-            failToPassCount: task.failToPassCount,
-            passToPassCount: task.passToPassCount,
-            complexityScore: task.complexityScore,
-        },
-    }, task.id, "record_eval_result");
-    const evalSummary = (await callTool("complete_eval_run", { runId: evalRun.runId }, task.id, "complete_eval_run"));
-    expect(evalSummary.status).toBe("completed");
-    expect(evalSummary.summary.passed).toBe(1);
-    const closedLoop = (await callTool("run_closed_loop", {
-        steps: [
-            { step: "compile", passed: true, output: `Compile checks for ${task.id}` },
-            { step: "lint", passed: true, output: `Lint checks for ${task.id}` },
-            { step: "test", passed: true, output: `Parallel benchmark checks for ${task.id}` },
-        ],
-    }, task.id, "run_closed_loop"));
-    expect(closedLoop.allPassed).toBe(true);
-    const flywheel = (await callTool("run_mandatory_flywheel", {
-        target: `Open-source SWE-bench task ${task.id}`,
-        steps: [
-            { stepName: "static_analysis", passed: true, output: "Types and schemas validated." },
-            { stepName: "happy_path_test", passed: true, output: "Dataset task completed end-to-end." },
-            { stepName: "failure_path_test", passed: true, output: "Fallback discovery query validated." },
-            { stepName: "gap_analysis", passed: true, output: "No blocking gaps for this task." },
-            { stepName: "fix_and_reverify", passed: true, output: "No rework required after checks." },
-            { stepName: "deploy_and_document", passed: true, output: "Benchmark result documented." },
-        ],
-    }, task.id, "run_mandatory_flywheel"));
-    expect(flywheel.passed).toBe(true);
-    const knowledge = (await callTool("search_all_knowledge", { query: task.id, limit: 10 }, task.id, "search_all_knowledge"));
-    expect(typeof knowledge.totalResults).toBe("number");
-    expect(knowledge.totalResults).toBeGreaterThan(0);
-    return {
-        taskId: task.id,
-        workerIndex,
-        ok: true,
-        elapsedMs: Date.now() - started,
-        discoveredTools: discovered.tools.length,
-        knowledgeHits: knowledge.totalResults,
-    };
-}
-async function runWorkerPool(tasks, concurrency) {
-    const boundedConcurrency = Math.max(1, Math.min(concurrency, tasks.length));
-    const results = new Array(tasks.length);
-    let nextIndex = 0;
-    const workers = Array.from({ length: boundedConcurrency }, (_, workerIndex) => (async () => {
-        while (true) {
-            const taskIndex = nextIndex++;
-            if (taskIndex >= tasks.length)
-                return;
-            const task = tasks[taskIndex];
-            try {
-                results[taskIndex] = await runDatasetTask(task, workerIndex);
-            }
-            catch (error) {
-                results[taskIndex] = {
-                    taskId: task.id,
-                    workerIndex,
-                    ok: false,
-                    elapsedMs: 0,
-                    discoveredTools: 0,
-                    knowledgeHits: 0,
-                    error: error instanceof Error ? error.message : String(error),
-                };
-            }
-        }
-    })());
-    await Promise.all(workers);
-    return results;
-}
-describe("Scenario: Open-Source Long-Running Dataset (SWE-bench Parallel Subagents)", () => {
-    it("should execute SWE-bench tasks with parallel MCP subagent workflows", async () => {
-        expect(Array.isArray(fixture.tasks)).toBe(true);
-        expect(fixture.tasks.length).toBeGreaterThan(0);
-        const requestedTaskLimit = Number.parseInt(process.env.NODEBENCH_SWEBENCH_TASK_LIMIT ?? "8", 10);
-        const taskLimit = Math.max(1, Math.min(fixture.tasks.length, Number.isFinite(requestedTaskLimit) ? requestedTaskLimit : 8));
-        const requestedConcurrency = Number.parseInt(process.env.NODEBENCH_SWEBENCH_CONCURRENCY ?? "4", 10);
-        const concurrency = Math.max(1, Math.min(taskLimit, Number.isFinite(requestedConcurrency) ? requestedConcurrency : 4));
-        const tasks = fixture.tasks.slice(0, taskLimit);
-        const started = Date.now();
-        const results = await runWorkerPool(tasks, concurrency);
-        const elapsedMs = Date.now() - started;
-        const failed = results.filter((result) => !result.ok);
-        const passed = results.filter((result) => result.ok);
-        const calledTools = new Set(openDatasetToolCallLog.map((entry) => entry.tool));
-        const requiredTools = [
-            "run_recon",
-            "log_recon_finding",
-            "findTools",
-            "getMethodology",
-            "start_eval_run",
-            "record_eval_result",
-            "complete_eval_run",
-            "run_closed_loop",
-            "run_mandatory_flywheel",
-            "search_all_knowledge",
-        ];
-        console.log(`[open-dataset-swebench] dataset=${fixture.dataset} split=${fixture.split} tasks=${taskLimit} concurrency=${concurrency} pass=${passed.length}/${results.length} elapsedMs=${elapsedMs} toolCalls=${openDatasetToolCallLog.length}`);
-        if (failed.length > 0) {
-            console.error("[open-dataset-swebench] failures:", failed.map((result) => ({
-                taskId: result.taskId,
-                workerIndex: result.workerIndex,
-                error: result.error,
-            })));
-        }
-        expect(failed.length).toBe(0);
-        expect(passed.length).toBe(taskLimit);
-        for (const requiredTool of requiredTools) {
-            expect(calledTools.has(requiredTool)).toBe(true);
-        }
-    });
-});
-//# sourceMappingURL=openDatasetParallelEvalSwebench.test.js.map

package/dist/__tests__/openDatasetParallelEvalSwebench.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"openDatasetParallelEvalSwebench.test.js","sourceRoot":"","sources":["../../src/__tests__/openDatasetParallelEvalSwebench.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,cAAc,MAAM,0CAA0C,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AA4CxD,MAAM,OAAO,GAAG,cAAgC,CAAC;AAEjD,MAAM,WAAW,GAAc;IAC7B,GAAG,iBAAiB;IACpB,GAAG,SAAS;IACZ,GAAG,gBAAgB;IACnB,GAAG,aAAa;IAChB,GAAG,aAAa;IAChB,GAAG,UAAU;IACb,GAAG,kBAAkB;IACrB,GAAG,mBAAmB;CACvB,CAAC;AACF,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC;AAEnE,MAAM,sBAAsB,GAKvB,EAAE,CAAC;AAER,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;IACtD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,IAAY,EACZ,IAA6B,EAC7B,MAAc,EACd,KAAa;IAEb,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACxC,sBAAsB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1E,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sBAAsB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3E,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAiB,EAAE,WAAmB;IAClE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE3B,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAC3B,WAAW,EACX;QACE,MAAM,EAAE,2BAA2B,IAAI,CAAC,EAAE,EAAE;QAC5C,WAAW,EAAE,mDAAmD,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,UAAU,IAAI;QACjG,cAAc,EAAE;YACd,SAAS,EAAE,yBAAyB;YACpC,YAAY,EAAE,0CAA0C;SACzD;KACF,EACD,IAAI,CAAC,EAAE,EACP,aAAa,CACd,CAAQ,CAAC;IAEV,MAAM,QAAQ,CACZ,mBAAmB,EACnB;QACE,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,QAAQ,EAAE,SAAS;QACnB,OAAO,EAAE,2BAA2B,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,IAAI,qBAAqB,IAAI,CAAC,eAAe,GAAG;QACrG,SAAS,EAAE,GAAG,OAAO,CAAC,SAAS,EAAE;QACjC,SAAS,EAAE,oEAAoE;QAC/E,WAAW,EAAE,oEAAoE;KAClF,EACD,IAAI,CAAC,EAAE,EACP,WAAW,CACZ,CAAC;IAEF,IAAI,UAAU,GAAG,CAAC,MAAM,QAAQ,CAC9B,WAAW,EACX;QACE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QAChC,QAAQ,EAAE,cAAc;KACzB,EACD,IAAI,CAAC,EAAE,EACP,YAAY,CACb,CAAQ,CAAC;IAEV,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvE,UAAU,GAAG,CAAC,MAAM,QAAQ,CAC1B,WAAW,EACX,EAAE,KAAK,EAAE,4CAA4C,EAAE,QAAQ,EAAE,MAAM,EAAE,EACzE,IAAI,CAAC,EAAE,EACP,qBAAqB,CACtB,CAAQ,CAAC;IACZ,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnD,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAEnD,MAAM,eAAe,GAAG,IAAI,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC;IACtD,MAAM,gBAAgB,GACpB,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,IAAI,CAAC,eAAe,IAAI,GAAG;QAClE,CAAC,CAAC,oBAAoB;QACtB,CAAC,CAAC,cAAc,CAAC;IACrB,MAAM,WAAW,GAAG,CAAC,MAAM,QAAQ,CACjC,gBAAgB,EAChB,EAAE,KAAK,EAAE,gBAAgB,EAAE,EAC3B,IAAI,CAAC,EAAE,EACP,iBAAiB,CAClB,CAAQ,CAAC;IACV,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;IACvC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpD,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAEpD,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAC7B,gBAAgB,EAChB;QACE,IAAI,EAAE,yBAAyB,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE;QACtD,WAAW,EAAE,uBAAuB,IAAI,CAAC,IAAI,YAAY,WAAW,GAAG;QACvE,KAAK,EAAE;YACL;gBACE,KAAK,EAAE,IAAI,CAAC,MAAM;gBAClB,MAAM,EAAE,kDAAkD,IAAI,CAAC,EAAE,EAAE;gBACnE,QAAQ,EACN,mGAAmG;aACtG;SACF;KACF,EACD,IAAI,CAAC,EAAE,EACP,gBAAgB,CACjB,CAAQ,CAAC;IAEV,MAAM,QAAQ,CACZ,oBAAoB,EACpB;QACE,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1B,OAAO,EAAE,MAAM;QACf,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,cAAc,UAAU,CAAC,KAAK,CAAC,MAAM,0CAA0C;QACvF,SAAS,EAAE;YACT,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,WAAW;YACX,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,eAAe,EAAE,IAAI,CAAC,eAAe;YACrC,eAAe,EAAE,IAAI,CAAC,eAAe;SACtC;KACF,EACD,IAAI,CAAC,EAAE,EACP,oBAAoB,CACrB,CAAC;IAEF,MAAM,WAAW,GAAG,CAAC,MAAM,QAAQ,CACjC,mBAAmB,EACnB,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EACxB,IAAI,CAAC,EAAE,EACP,mBAAmB,CACpB,CAAQ,CAAC;IACV,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE3C,MAAM,UAAU,GAAG,CAAC,MAAM,QAAQ,CAChC,iBAAiB,EACjB;QACE,KAAK,EAAE;YACL,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,sBAAsB,IAAI,CAAC,EAAE,EAAE,EAAE;YAC1E,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,IAAI,CAAC,EAAE,EAAE,EAAE;YACpE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,iCAAiC,IAAI,CAAC,EAAE,EAAE,EAAE;SACnF;KACF,EACD,IAAI,CAAC,EAAE,EACP,iBAAiB,CAClB,CAAQ,CAAC;IACV,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAExC,MAAM,QAAQ,GAAG,CAAC,MAAM,QAAQ,CAC9B,wBAAwB,EACxB;QACE,MAAM,EAAE,8BAA8B,IAAI,CAAC,EAAE,EAAE;QAC/C,KAAK,EAAE;YACL,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,8BAA8B,EAAE;YACrF,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,oCAAoC,EAAE;YAC3F,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,qCAAqC,EAAE;YAC9F,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,iCAAiC,EAAE;YACrF,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,kCAAkC,EAAE;YAC1F,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,8BAA8B,EAAE;SAC1F;KACF,EACD,IAAI,CAAC,EAAE,EACP,wBAAwB,CACzB,CAAQ,CAAC;IACV,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,SAAS,GAAG,CAAC,MAAM,QAAQ,CAC/B,sBAAsB,EACtB,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,EAC7B,IAAI,CAAC,EAAE,EACP,sBAAsB,CACvB,CAAQ,CAAC;IACV,MAAM,CAAC,OAAO,SAAS,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrD,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAElD,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,WAAW;QACX,EAAE,EAAE,IAAI;QACR,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;QAC/B,eAAe,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM;QACxC,aAAa,EAAE,SAAS,CAAC,YAAY;KACtC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,KAAoB,EAAE,WAAmB;IACpE,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAmB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxD,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,CAC5E,CAAC,KAAK,IAAI,EAAE;QACV,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC;YAC9B,IAAI,SAAS,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAEtC,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;YAC9B,IAAI,CAAC;gBACH,OAAO,CAAC,SAAS,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAC/D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,SAAS,CAAC,GAAG;oBACnB,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,WAAW;oBACX,EAAE,EAAE,KAAK;oBACT,SAAS,EAAE,CAAC;oBACZ,eAAe,EAAE,CAAC;oBAClB,aAAa,EAAE,CAAC;oBAChB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC9D,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC,CAAC,EAAE,CACL,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,QAAQ,CAAC,2EAA2E,EAAE,GAAG,EAAE;IACzF,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;QACnF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAEhD,MAAM,kBAAkB,GAAG,MAAM,CAAC,QAAQ,CACxC,OAAO,CAAC,GAAG,CAAC,6BAA6B,IAAI,GAAG,EAChD,EAAE,CACH,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7F,CAAC;QAEF,MAAM,oBAAoB,GAAG,MAAM,CAAC,QAAQ,CAC1C,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,GAAG,EACjD,EAAE,CACH,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAC1B,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CACtF,CAAC;QAEF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAEvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAErD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/E,MAAM,aAAa,GAAG;YACpB,WAAW;YACX,mBAAmB;YACnB,WAAW;YACX,gBAAgB;YAChB,gBAAgB;YAChB,oBAAoB;YACpB,mBAAmB;YACnB,iBAAiB;YACjB,wBAAwB;YACxB,sBAAsB;SACvB,CAAC;QAEF,OAAO,CAAC,GAAG,CACT,mCAAmC,OAAO,CAAC,OAAO,UAAU,OAAO,CAAC,KAAK,UAAU,SAAS,gBAAgB,WAAW,SAAS,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,cAAc,SAAS,cAAc,sBAAsB,CAAC,MAAM,EAAE,CACpO,CAAC;QAEF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CACX,mCAAmC,EACnC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACtB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC,CACJ,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEtC,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;YACzC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/__tests__/openDatasetParallelEvalToolbench.test.d.ts

@@ -1,7 +0,0 @@
-/**
- * Open-source dataset benchmark for long-running tasks (ToolBench lane).
- *
- * This test uses ToolBench multi-tool instructions and runs task workflows
- * through NodeBench MCP tools in parallel "subagent" workers.
- */
-export {};

package/dist/__tests__/openDatasetParallelEvalToolbench.test.js

@@ -1,218 +0,0 @@
-/**
- * Open-source dataset benchmark for long-running tasks (ToolBench lane).
- *
- * This test uses ToolBench multi-tool instructions and runs task workflows
- * through NodeBench MCP tools in parallel "subagent" workers.
- */
-import { describe, expect, it } from "vitest";
-import datasetFixture from "./fixtures/toolbench_instruction.sample.json";
-import { verificationTools } from "../tools/verificationTools.js";
-import { reconTools } from "../tools/reconTools.js";
-import { evalTools } from "../tools/evalTools.js";
-import { qualityGateTools } from "../tools/qualityGateTools.js";
-import { flywheelTools } from "../tools/flywheelTools.js";
-import { learningTools } from "../tools/learningTools.js";
-import { documentationTools } from "../tools/documentationTools.js";
-import { agentBootstrapTools } from "../tools/agentBootstrapTools.js";
-import { createMetaTools } from "../tools/metaTools.js";
-const fixture = datasetFixture;
-const domainTools = [
-    ...verificationTools,
-    ...evalTools,
-    ...qualityGateTools,
-    ...learningTools,
-    ...flywheelTools,
-    ...reconTools,
-    ...documentationTools,
-    ...agentBootstrapTools,
-];
-const allTools = [...domainTools, ...createMetaTools(domainTools)];
-const openDatasetToolCallLog = [];
-function findTool(name) {
-    const tool = allTools.find((candidate) => candidate.name === name);
-    if (!tool)
-        throw new Error(`Tool not found: ${name}`);
-    return tool;
-}
-async function callTool(name, args, taskId, stage) {
-    const tool = findTool(name);
-    try {
-        const result = await tool.handler(args);
-        openDatasetToolCallLog.push({ taskId, tool: name, stage, success: true });
-        return result;
-    }
-    catch (error) {
-        openDatasetToolCallLog.push({ taskId, tool: name, stage, success: false });
-        throw error;
-    }
-}
-async function runDatasetTask(task, workerIndex) {
-    const started = Date.now();
-    const datasetSource = fixture.sourceUrls[task.group] ?? fixture.sourceUrls.G1 ?? "";
-    const recon = (await callTool("run_recon", {
-        target: `ToolBench multi-tool task ${task.id}`,
-        description: `Open-source long-running benchmark (${task.apiCount} APIs in task context).`,
-        projectContext: {
-            techStack: "TypeScript, MCP, SQLite",
-            architecture: "MCP orchestration benchmark with parallel subagent workers",
-        },
-    }, task.id, "recon_start"));
-    await callTool("log_recon_finding", {
-        sessionId: recon.sessionId,
-        category: "dataset",
-        summary: `Ingested ToolBench task ${task.id} with ${task.apiCount} APIs and complexity ${task.complexityScore}.`,
-        sourceUrl: datasetSource,
-        relevance: "Long-running multi-tool benchmark for MCP orchestration quality.",
-        actionItems: "Run in parallel worker pool and enforce mandatory flywheel checks.",
-    }, task.id, "recon_log");
-    let discovered = (await callTool("findTools", {
-        query: task.prompt.slice(0, 600),
-        category: "verification",
-    }, task.id, "find_tools"));
-    if (!Array.isArray(discovered?.tools) || discovered.tools.length === 0) {
-        discovered = (await callTool("findTools", { query: "multi-step tool orchestration verification", category: "bootstrap" }, task.id, "find_tools_fallback"));
-    }
-    expect(Array.isArray(discovered.tools)).toBe(true);
-    expect(discovered.tools.length).toBeGreaterThan(0);
-    const methodologyTopic = task.apiCount >= 9 ? "mandatory_flywheel" : "closed_loop";
-    const methodology = (await callTool("getMethodology", { topic: methodologyTopic }, task.id, "get_methodology"));
-    expect(methodology.title).toBeTruthy();
-    expect(Array.isArray(methodology.steps)).toBe(true);
-    expect(methodology.steps.length).toBeGreaterThan(0);
-    const evalRun = (await callTool("start_eval_run", {
-        name: `open-dataset-toolbench-${task.id}-${Date.now()}`,
-        description: `ToolBench multi-tool scenario (${task.apiCount} APIs, worker ${workerIndex})`,
-        cases: [
-            {
-                input: task.prompt,
-                intent: `Coordinate long-running ToolBench workflow for ${task.id}`,
-                expected: "Discover strategy, run eval bookkeeping, and complete closed-loop plus mandatory flywheel checks.",
-            },
-        ],
-    }, task.id, "start_eval_run"));
-    await callTool("record_eval_result", {
-        caseId: evalRun.caseIds[0],
-        verdict: "pass",
-        score: 1,
-        actual: `Discovered ${discovered.tools.length} tools and completed ToolBench workflow.`,
-        telemetry: {
-            dataset: fixture.dataset,
-            split: fixture.split,
-            taskId: task.id,
-            workerIndex,
-            group: task.group,
-            apiCount: task.apiCount,
-            relevantApiCount: task.relevantApiCount,
-            requiredParameterCount: task.requiredParameterCount,
-            optionalParameterCount: task.optionalParameterCount,
-            apiCategories: task.apiCategories,
-        },
-    }, task.id, "record_eval_result");
-    const evalSummary = (await callTool("complete_eval_run", { runId: evalRun.runId }, task.id, "complete_eval_run"));
-    expect(evalSummary.status).toBe("completed");
-    expect(evalSummary.summary.passed).toBe(1);
-    const closedLoop = (await callTool("run_closed_loop", {
-        steps: [
-            { step: "compile", passed: true, output: `Compile checks for ${task.id}` },
-            { step: "lint", passed: true, output: `Lint checks for ${task.id}` },
-            { step: "test", passed: true, output: `Parallel benchmark checks for ${task.id}` },
-        ],
-    }, task.id, "run_closed_loop"));
-    expect(closedLoop.allPassed).toBe(true);
-    const flywheel = (await callTool("run_mandatory_flywheel", {
-        target: `Open-source ToolBench task ${task.id}`,
-        steps: [
-            { stepName: "static_analysis", passed: true, output: "Types and schemas validated." },
-            { stepName: "happy_path_test", passed: true, output: "Dataset task completed end-to-end." },
-            { stepName: "failure_path_test", passed: true, output: "Fallback discovery query validated." },
-            { stepName: "gap_analysis", passed: true, output: "No blocking gaps for this task." },
-            { stepName: "fix_and_reverify", passed: true, output: "No rework required after checks." },
-            { stepName: "deploy_and_document", passed: true, output: "Benchmark result documented." },
-        ],
-    }, task.id, "run_mandatory_flywheel"));
-    expect(flywheel.passed).toBe(true);
-    const knowledgeQuery = task.id;
-    const knowledge = (await callTool("search_all_knowledge", { query: knowledgeQuery, limit: 10 }, task.id, "search_all_knowledge"));
-    expect(typeof knowledge.totalResults).toBe("number");
-    expect(knowledge.totalResults).toBeGreaterThan(0);
-    return {
-        taskId: task.id,
-        workerIndex,
-        ok: true,
-        elapsedMs: Date.now() - started,
-        discoveredTools: discovered.tools.length,
-        knowledgeHits: knowledge.totalResults,
-    };
-}
-async function runWorkerPool(tasks, concurrency) {
-    const boundedConcurrency = Math.max(1, Math.min(concurrency, tasks.length));
-    const results = new Array(tasks.length);
-    let nextIndex = 0;
-    const workers = Array.from({ length: boundedConcurrency }, (_, workerIndex) => (async () => {
-        while (true) {
-            const taskIndex = nextIndex++;
-            if (taskIndex >= tasks.length)
-                return;
-            const task = tasks[taskIndex];
-            try {
-                results[taskIndex] = await runDatasetTask(task, workerIndex);
-            }
-            catch (error) {
-                results[taskIndex] = {
-                    taskId: task.id,
-                    workerIndex,
-                    ok: false,
-                    elapsedMs: 0,
-                    discoveredTools: 0,
-                    knowledgeHits: 0,
-                    error: error instanceof Error ? error.message : String(error),
-                };
-            }
-        }
-    })());
-    await Promise.all(workers);
-    return results;
-}
-describe("Scenario: Open-Source Long-Running Dataset (ToolBench Parallel Subagents)", () => {
-    it("should execute ToolBench tasks with parallel MCP subagent workflows", async () => {
-        expect(Array.isArray(fixture.tasks)).toBe(true);
-        expect(fixture.tasks.length).toBeGreaterThan(0);
-        const requestedTaskLimit = Number.parseInt(process.env.NODEBENCH_TOOLBENCH_TASK_LIMIT ?? "8", 10);
-        const taskLimit = Math.max(1, Math.min(fixture.tasks.length, Number.isFinite(requestedTaskLimit) ? requestedTaskLimit : 8));
-        const requestedConcurrency = Number.parseInt(process.env.NODEBENCH_TOOLBENCH_CONCURRENCY ?? "4", 10);
-        const concurrency = Math.max(1, Math.min(taskLimit, Number.isFinite(requestedConcurrency) ? requestedConcurrency : 4));
-        const tasks = fixture.tasks.slice(0, taskLimit);
-        const started = Date.now();
-        const results = await runWorkerPool(tasks, concurrency);
-        const elapsedMs = Date.now() - started;
-        const failed = results.filter((result) => !result.ok);
-        const passed = results.filter((result) => result.ok);
-        const calledTools = new Set(openDatasetToolCallLog.map((entry) => entry.tool));
-        const requiredTools = [
-            "run_recon",
-            "log_recon_finding",
-            "findTools",
-            "getMethodology",
-            "start_eval_run",
-            "record_eval_result",
-            "complete_eval_run",
-            "run_closed_loop",
-            "run_mandatory_flywheel",
-            "search_all_knowledge",
-        ];
-        console.log(`[open-dataset-toolbench] dataset=${fixture.dataset} split=${fixture.split} tasks=${taskLimit} concurrency=${concurrency} pass=${passed.length}/${results.length} elapsedMs=${elapsedMs} toolCalls=${openDatasetToolCallLog.length}`);
-        if (failed.length > 0) {
-            console.error("[open-dataset-toolbench] failures:", failed.map((result) => ({
-                taskId: result.taskId,
-                workerIndex: result.workerIndex,
-                error: result.error,
-            })));
-        }
-        expect(failed.length).toBe(0);
-        expect(passed.length).toBe(taskLimit);
-        for (const requiredTool of requiredTools) {
-            expect(calledTools.has(requiredTool)).toBe(true);
-        }
-    });
-});
-//# sourceMappingURL=openDatasetParallelEvalToolbench.test.js.map

package/dist/__tests__/openDatasetParallelEvalToolbench.test.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"openDatasetParallelEvalToolbench.test.js","sourceRoot":"","sources":["../../src/__tests__/openDatasetParallelEvalToolbench.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,cAAc,MAAM,8CAA8C,CAAC;AAC1E,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACtE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAyCxD,MAAM,OAAO,GAAG,cAAgC,CAAC;AAEjD,MAAM,WAAW,GAAc;IAC7B,GAAG,iBAAiB;IACpB,GAAG,SAAS;IACZ,GAAG,gBAAgB;IACnB,GAAG,aAAa;IAChB,GAAG,aAAa;IAChB,GAAG,UAAU;IACb,GAAG,kBAAkB;IACrB,GAAG,mBAAmB;CACvB,CAAC;AACF,MAAM,QAAQ,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,eAAe,CAAC,WAAW,CAAC,CAAC,CAAC;AAEnE,MAAM,sBAAsB,GAKvB,EAAE,CAAC;AAER,SAAS,QAAQ,CAAC,IAAY;IAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IACnE,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;IACtD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,KAAK,UAAU,QAAQ,CACrB,IAAY,EACZ,IAA6B,EAC7B,MAAc,EACd,KAAa;IAEb,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACxC,sBAAsB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1E,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sBAAsB,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;QAC3E,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAiB,EAAE,WAAmB;IAClE,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,MAAM,aAAa,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,EAAE,IAAI,EAAE,CAAC;IAEpF,MAAM,KAAK,GAAG,CAAC,MAAM,QAAQ,CAC3B,WAAW,EACX;QACE,MAAM,EAAE,6BAA6B,IAAI,CAAC,EAAE,EAAE;QAC9C,WAAW,EAAE,uCAAuC,IAAI,CAAC,QAAQ,yBAAyB;QAC1F,cAAc,EAAE;YACd,SAAS,EAAE,yBAAyB;YACpC,YAAY,EAAE,4DAA4D;SAC3E;KACF,EACD,IAAI,CAAC,EAAE,EACP,aAAa,CACd,CAAQ,CAAC;IAEV,MAAM,QAAQ,CACZ,mBAAmB,EACnB;QACE,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,QAAQ,EAAE,SAAS;QACnB,OAAO,EAAE,2BAA2B,IAAI,CAAC,EAAE,SAAS,IAAI,CAAC,QAAQ,wBAAwB,IAAI,CAAC,eAAe,GAAG;QAChH,SAAS,EAAE,aAAa;QACxB,SAAS,EAAE,kEAAkE;QAC7E,WAAW,EAAE,oEAAoE;KAClF,EACD,IAAI,CAAC,EAAE,EACP,WAAW,CACZ,CAAC;IAEF,IAAI,UAAU,GAAG,CAAC,MAAM,QAAQ,CAC9B,WAAW,EACX;QACE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;QAChC,QAAQ,EAAE,cAAc;KACzB,EACD,IAAI,CAAC,EAAE,EACP,YAAY,CACb,CAAQ,CAAC;IAEV,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvE,UAAU,GAAG,CAAC,MAAM,QAAQ,CAC1B,WAAW,EACX,EAAE,KAAK,EAAE,4CAA4C,EAAE,QAAQ,EAAE,WAAW,EAAE,EAC9E,IAAI,CAAC,EAAE,EACP,qBAAqB,CACtB,CAAQ,CAAC;IACZ,CAAC;IACD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnD,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAEnD,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,aAAa,CAAC;IACnF,MAAM,WAAW,GAAG,CAAC,MAAM,QAAQ,CACjC,gBAAgB,EAChB,EAAE,KAAK,EAAE,gBAAgB,EAAE,EAC3B,IAAI,CAAC,EAAE,EACP,iBAAiB,CAClB,CAAQ,CAAC;IACV,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,CAAC;IACvC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpD,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAEpD,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAC7B,gBAAgB,EAChB;QACE,IAAI,EAAE,0BAA0B,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE;QACvD,WAAW,EAAE,kCAAkC,IAAI,CAAC,QAAQ,iBAAiB,WAAW,GAAG;QAC3F,KAAK,EAAE;YACL;gBACE,KAAK,EAAE,IAAI,CAAC,MAAM;gBAClB,MAAM,EAAE,kDAAkD,IAAI,CAAC,EAAE,EAAE;gBACnE,QAAQ,EACN,mGAAmG;aACtG;SACF;KACF,EACD,IAAI,CAAC,EAAE,EACP,gBAAgB,CACjB,CAAQ,CAAC;IAEV,MAAM,QAAQ,CACZ,oBAAoB,EACpB;QACE,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QAC1B,OAAO,EAAE,MAAM;QACf,KAAK,EAAE,CAAC;QACR,MAAM,EAAE,cAAc,UAAU,CAAC,KAAK,CAAC,MAAM,0CAA0C;QACvF,SAAS,EAAE;YACT,OAAO,EAAE,OAAO,CAAC,OAAO;YACxB,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,MAAM,EAAE,IAAI,CAAC,EAAE;YACf,WAAW;YACX,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;YACnD,sBAAsB,EAAE,IAAI,CAAC,sBAAsB;YACnD,aAAa,EAAE,IAAI,CAAC,aAAa;SAClC;KACF,EACD,IAAI,CAAC,EAAE,EACP,oBAAoB,CACrB,CAAC;IAEF,MAAM,WAAW,GAAG,CAAC,MAAM,QAAQ,CACjC,mBAAmB,EACnB,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,EACxB,IAAI,CAAC,EAAE,EACP,mBAAmB,CACpB,CAAQ,CAAC;IACV,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAE3C,MAAM,UAAU,GAAG,CAAC,MAAM,QAAQ,CAChC,iBAAiB,EACjB;QACE,KAAK,EAAE;YACL,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,sBAAsB,IAAI,CAAC,EAAE,EAAE,EAAE;YAC1E,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,IAAI,CAAC,EAAE,EAAE,EAAE;YACpE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,iCAAiC,IAAI,CAAC,EAAE,EAAE,EAAE;SACnF;KACF,EACD,IAAI,CAAC,EAAE,EACP,iBAAiB,CAClB,CAAQ,CAAC;IACV,MAAM,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAExC,MAAM,QAAQ,GAAG,CAAC,MAAM,QAAQ,CAC9B,wBAAwB,EACxB;QACE,MAAM,EAAE,8BAA8B,IAAI,CAAC,EAAE,EAAE;QAC/C,KAAK,EAAE;YACL,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,8BAA8B,EAAE;YACrF,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,oCAAoC,EAAE;YAC3F,EAAE,QAAQ,EAAE,mBAAmB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,qCAAqC,EAAE;YAC9F,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,iCAAiC,EAAE;YACrF,EAAE,QAAQ,EAAE,kBAAkB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,kCAAkC,EAAE;YAC1F,EAAE,QAAQ,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,8BAA8B,EAAE;SAC1F;KACF,EACD,IAAI,CAAC,EAAE,EACP,wBAAwB,CACzB,CAAQ,CAAC;IACV,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEnC,MAAM,cAAc,GAAG,IAAI,CAAC,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,CAAC,MAAM,QAAQ,CAC/B,sBAAsB,EACtB,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,EAAE,EAAE,EACpC,IAAI,CAAC,EAAE,EACP,sBAAsB,CACvB,CAAQ,CAAC;IACV,MAAM,CAAC,OAAO,SAAS,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACrD,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;IAElD,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,WAAW;QACX,EAAE,EAAE,IAAI;QACR,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;QAC/B,eAAe,EAAE,UAAU,CAAC,KAAK,CAAC,MAAM;QACxC,aAAa,EAAE,SAAS,CAAC,YAAY;KACtC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,KAAoB,EAAE,WAAmB;IACpE,MAAM,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAmB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACxD,IAAI,SAAS,GAAG,CAAC,CAAC;IAElB,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,kBAAkB,EAAE,EAAE,CAAC,CAAC,EAAE,WAAW,EAAE,EAAE,CAC5E,CAAC,KAAK,IAAI,EAAE;QACV,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,SAAS,GAAG,SAAS,EAAE,CAAC;YAC9B,IAAI,SAAS,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAO;YAEtC,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;YAC9B,IAAI,CAAC;gBACH,OAAO,CAAC,SAAS,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC,CAAC;YAC/D,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,SAAS,CAAC,GAAG;oBACnB,MAAM,EAAE,IAAI,CAAC,EAAE;oBACf,WAAW;oBACX,EAAE,EAAE,KAAK;oBACT,SAAS,EAAE,CAAC;oBACZ,eAAe,EAAE,CAAC;oBAClB,aAAa,EAAE,CAAC;oBAChB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;iBAC9D,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC,CAAC,EAAE,CACL,CAAC;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC3B,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,QAAQ,CAAC,2EAA2E,EAAE,GAAG,EAAE;IACzF,EAAE,CAAC,qEAAqE,EAAE,KAAK,IAAI,EAAE;QACnF,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAEhD,MAAM,kBAAkB,GAAG,MAAM,CAAC,QAAQ,CACxC,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,GAAG,EACjD,EAAE,CACH,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,QAAQ,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAC7F,CAAC;QAEF,MAAM,oBAAoB,GAAG,MAAM,CAAC,QAAQ,CAC1C,OAAO,CAAC,GAAG,CAAC,+BAA+B,IAAI,GAAG,EAClD,EAAE,CACH,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAC1B,CAAC,EACD,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,CAAC,CACtF,CAAC;QAEF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QAChD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,MAAM,OAAO,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;QAEvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAErD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,sBAAsB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/E,MAAM,aAAa,GAAG;YACpB,WAAW;YACX,mBAAmB;YACnB,WAAW;YACX,gBAAgB;YAChB,gBAAgB;YAChB,oBAAoB;YACpB,mBAAmB;YACnB,iBAAiB;YACjB,wBAAwB;YACxB,sBAAsB;SACvB,CAAC;QAEF,OAAO,CAAC,GAAG,CACT,oCAAoC,OAAO,CAAC,OAAO,UAAU,OAAO,CAAC,KAAK,UAAU,SAAS,gBAAgB,WAAW,SAAS,MAAM,CAAC,MAAM,IAAI,OAAO,CAAC,MAAM,cAAc,SAAS,cAAc,sBAAsB,CAAC,MAAM,EAAE,CACrO,CAAC;QAEF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CACX,oCAAoC,EACpC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACtB,MAAM,EAAE,MAAM,CAAC,MAAM;gBACrB,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC,CAAC,CACJ,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC9B,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAEtC,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;YACzC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/__tests__/openDatasetPerfComparison.test.d.ts

@@ -1,10 +0,0 @@
-/**
- * Performance comparison: "no tools" (noop handlers) vs NodeBench MCP toolchain.
- *
- * Notes:
- * - This is a wall-clock micro-benchmark for local harness overhead, not capability/accuracy.
- * - Disabled by default to avoid noisy perf output in normal test runs.
- *   Set NODEBENCH_RUN_PERF_COMPARE=1 to run.
- * - GAIA is gated; if a local GAIA fixture exists in `.cache/gaia`, it will be included.
- */
-export {};