nodebench-mcp 3.0.0 → 3.0.2

package/dist/__tests__/evalDatasetBench.test.js

@@ -1,738 +0,0 @@
-/**
- * Dataset-Driven Eval Bench for NodeBench MCP Tools
- *
- * Tests MCP tool orchestration against REAL open-source task descriptions
- * from SWE-bench Verified (500 human-validated GitHub issues).
- *
- * Each task runs through the FULL agent pipeline:
- *   Recon → Verification → Eval → Quality Gate → Learning → Flywheel
- *
- * This proves the tools can orchestrate real-world development workflows
- * end-to-end, not just pass unit tests in isolation.
- *
- * Dataset: SWE-bench Verified (princeton-nlp/SWE-bench_Verified)
- * Source: https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified
- */
-import { describe, it, expect, afterAll } from "vitest";
-import { verificationTools } from "../tools/verificationTools.js";
-import { reconTools } from "../tools/reconTools.js";
-import { evalTools } from "../tools/evalTools.js";
-import { qualityGateTools } from "../tools/qualityGateTools.js";
-import { flywheelTools } from "../tools/flywheelTools.js";
-import { learningTools } from "../tools/learningTools.js";
-import { agentBootstrapTools } from "../tools/agentBootstrapTools.js";
-import { createMetaTools } from "../tools/metaTools.js";
-// ═══════════════════════════════════════════════════════════════════════════
-// TOOL SETUP
-// ═══════════════════════════════════════════════════════════════════════════
-const domainTools = [
-    ...verificationTools,
-    ...evalTools,
-    ...qualityGateTools,
-    ...learningTools,
-    ...flywheelTools,
-    ...reconTools,
-    ...agentBootstrapTools,
-];
-const allTools = [...domainTools, ...createMetaTools(domainTools)];
-const findTool = (name) => {
-    const tool = allTools.find((t) => t.name === name);
-    if (!tool)
-        throw new Error(`Tool not found: ${name}`);
-    return tool;
-};
-// Telemetry
-const pipelineLog = [];
-async function callTool(name, args, taskId, phase) {
-    const tool = findTool(name);
-    const start = Date.now();
-    try {
-        const result = await tool.handler(args);
-        pipelineLog.push({
-            taskId,
-            tool: name,
-            phase,
-            success: true,
-            durationMs: Date.now() - start,
-        });
-        return result;
-    }
-    catch (error) {
-        pipelineLog.push({
-            taskId,
-            tool: name,
-            phase,
-            success: false,
-            durationMs: Date.now() - start,
-        });
-        throw error;
-    }
-}
-const SWE_BENCH_TASKS = [
-    {
-        instance_id: "django__django-11133",
-        repo: "django/django",
-        problem_statement: "HttpResponse doesn't handle memoryview objects. When a memoryview is passed to HttpResponse, it displays as a memory address string representation rather than the actual content bytes.",
-        category: "bug_fix",
-        complexity: "low",
-    },
-    {
-        instance_id: "scikit-learn__scikit-learn-14053",
-        repo: "scikit-learn/scikit-learn",
-        problem_statement: "IndexError: list index out of range in export_text when the decision tree only has one feature. The export_text function crashes with an IndexError when the trained DecisionTreeClassifier uses a single feature.",
-        category: "bug_fix",
-        complexity: "low",
-    },
-    {
-        instance_id: "sympy__sympy-13372",
-        repo: "sympy/sympy",
-        problem_statement: "UnboundLocalError in evalf. Calling Mul(Max(0, y), x, evaluate=False).evalf() raises an UnboundLocalError where local variable 'reprec' is referenced before assignment.",
-        category: "bug_fix",
-        complexity: "medium",
-    },
-    {
-        instance_id: "django__django-11099",
-        repo: "django/django",
-        problem_statement: "UsernameValidator allows trailing newline in usernames. ASCIIUsernameValidator and UnicodeUsernameValidator use a regex pattern with $ which matches a trailing newline in Python, allowing invalid usernames to pass validation.",
-        category: "bug_fix",
-        complexity: "low",
-    },
-    {
-        instance_id: "astropy__astropy-12907",
-        repo: "astropy/astropy",
-        problem_statement: "Modeling's separability_matrix does not compute separability correctly for nested CompoundModels. When nesting compound models using the & operator, the separability matrix incorrectly indicates coupled outputs.",
-        category: "bug_fix",
-        complexity: "high",
-    },
-    {
-        instance_id: "django__django-11095",
-        repo: "django/django",
-        problem_statement: "Add ModelAdmin.get_inlines() hook to allow setting inlines based on the request or model instance. Currently, users must override get_inline_instances to achieve dynamic inlines.",
-        category: "feature",
-        complexity: "medium",
-    },
-    {
-        instance_id: "scikit-learn__scikit-learn-13496",
-        repo: "scikit-learn/scikit-learn",
-        problem_statement: "Expose warm_start in Isolation Forest. sklearn.ensemble.IsolationForest supports incremental addition of new trees with warm_start but the parameter is not exposed in __init__().",
-        category: "feature",
-        complexity: "low",
-    },
-    {
-        instance_id: "matplotlib__matplotlib-24627",
-        repo: "matplotlib/matplotlib",
-        problem_statement: "cla(), clf() should unset the .axes and .figure attributes of deparented artists. Clearing the axes via cla() or the figure via clf() does not unset references, leaving stale references.",
-        category: "refactor",
-        complexity: "medium",
-    },
-    {
-        instance_id: "sphinx-doc__sphinx-8265",
-        repo: "sphinx-doc/sphinx",
-        problem_statement: "Python method signatures with tuple default arguments are rendered incorrectly in docstrings. For example, color=(1, 1, 1) appears as color=1, 1, 1 in the generated documentation.",
-        category: "bug_fix",
-        complexity: "medium",
-    },
-    {
-        instance_id: "pydata__xarray-3305",
-        repo: "pydata/xarray",
-        problem_statement: "DataArray.quantile does not honor keep_attrs. When calling quantile with keep_attrs=True on a DataArray with attributes, the returned object loses those attributes.",
-        category: "bug_fix",
-        complexity: "low",
-    },
-    {
-        instance_id: "pylint-dev__pylint-4661",
-        repo: "pylint-dev/pylint",
-        problem_statement: "Make pylint XDG Base Directory Specification compliant. The .pylint.d directory clutters the user's home folder; data should be stored in $HOME/.cache/pylint following XDG.",
-        category: "feature",
-        complexity: "medium",
-    },
-    {
-        instance_id: "django__django-14017",
-        repo: "django/django",
-        problem_statement: "Q(...) & Exists(...) raises a TypeError. The bitwise AND operator between Q and Exists objects is not commutative: Exists(...) & Q(...) works, but Q(...) & Exists(...) raises TypeError.",
-        category: "bug_fix",
-        complexity: "medium",
-    },
-    {
-        instance_id: "sympy__sympy-13647",
-        repo: "sympy/sympy",
-        problem_statement: "Matrix.col_insert() no longer seems to work correctly. When inserting columns into an identity matrix, the 3x3 identity portion shifts incorrectly, producing a wrong result.",
-        category: "bug_fix",
-        complexity: "medium",
-    },
-    {
-        instance_id: "scikit-learn__scikit-learn-14141",
-        repo: "scikit-learn/scikit-learn",
-        problem_statement: "Add joblib in show_versions. joblib is a key dependency of scikit-learn but is missing from the output of sklearn.show_versions(), making it harder to debug environment-related issues.",
-        category: "documentation",
-        complexity: "low",
-    },
-    {
-        instance_id: "django__django-11039",
-        repo: "django/django",
-        problem_statement: "sqlmigrate wraps its output in BEGIN/COMMIT even if the database doesn't support transactional DDL. Should only show transaction markers when the backend supports rolling back DDL.",
-        category: "bug_fix",
-        complexity: "low",
-    },
-    {
-        instance_id: "sphinx-doc__sphinx-9258",
-        repo: "sphinx-doc/sphinx",
-        problem_statement: "The Python domain does not recognize the pipe character | as a union type separator in type annotations (PEP 604). int | str syntax is not supported in Sphinx's type annotation parsing.",
-        category: "feature",
-        complexity: "high",
-    },
-    {
-        instance_id: "astropy__astropy-13398",
-        repo: "astropy/astropy",
-        problem_statement: "A direct approach to ITRS to Observed transformations that stays within the ITRS. Current implementations route through intermediate coordinate frames unnecessarily for satellite observations.",
-        category: "feature",
-        complexity: "high",
-    },
-    {
-        instance_id: "django__django-11964",
-        repo: "django/django",
-        problem_statement: "TextChoices and IntegerChoices instances lack proper string representation. The type and display behavior differs between newly created model instances using choices and instances retrieved from the database.",
-        category: "api_change",
-        complexity: "medium",
-    },
-    {
-        instance_id: "pydata__xarray-3993",
-        repo: "pydata/xarray",
-        problem_statement: "DataArray.integrate has a 'dim' arg, but Dataset.integrate has a 'coord' arg. The API syntax is inconsistent between the two methods across DataArray and Dataset.",
-        category: "api_change",
-        complexity: "medium",
-    },
-    {
-        instance_id: "scikit-learn__scikit-learn-14710",
-        repo: "scikit-learn/scikit-learn",
-        problem_statement: "HistGradientBoostingClassifier does not work with string target when early stopping is turned on. The scorer receives y_true as encoded integers while y_pred contains original string class labels.",
-        category: "bug_fix",
-        complexity: "high",
-    },
-];
-// ═══════════════════════════════════════════════════════════════════════════
-// FULL AGENT PIPELINE — runs each SWE-bench task through all tool domains
-// ═══════════════════════════════════════════════════════════════════════════
-/**
- * Runs a single SWE-bench task through the complete MCP tool pipeline.
- * Returns cleanup IDs for resource teardown.
- */
-async function runFullPipeline(task) {
-    const taskId = task.instance_id;
-    const cleanupIds = {
-        cycleIds: [],
-        learningKeys: [],
-    };
-    // ─── Phase 1: META — Discover the right tools for this task ───
-    const toolSearch = (await callTool("findTools", { query: task.category === "bug_fix" ? "verification gap" : "feature implementation" }, taskId, "meta"));
-    expect(toolSearch.tools.length).toBeGreaterThan(0);
-    const methodology = (await callTool("getMethodology", { topic: "verification" }, taskId, "meta"));
-    expect(methodology.steps.length).toBeGreaterThan(0);
-    // ─── Phase 2: RECON — Research the problem ───
-    const reconSession = (await callTool("run_recon", {
-        target: `${task.repo}: ${task.problem_statement.slice(0, 80)}`,
-        description: `Research for ${taskId}`,
-    }, taskId, "recon"));
-    expect(reconSession.sessionId).toBeTruthy();
-    await callTool("log_recon_finding", {
-        sessionId: reconSession.sessionId,
-        category: "codebase_pattern",
-        summary: `Root cause analysis: ${task.problem_statement.slice(0, 120)}`,
-        sourceUrl: `https://github.com/${task.repo}`,
-        relevance: `Directly affects ${task.category} implementation`,
-    }, taskId, "recon");
-    const reconSummary = (await callTool("get_recon_summary", { sessionId: reconSession.sessionId }, taskId, "recon"));
-    expect(reconSummary.totalFindings).toBeGreaterThan(0);
-    // ─── Phase 3: RISK ASSESSMENT — Evaluate before acting ───
-    const risk = (await callTool("assess_risk", {
-        action: task.category === "api_change" ? "modify_public_api" : "fix_implementation",
-        context: `${task.repo} — ${task.complexity} complexity ${task.category}`,
-    }, taskId, "risk"));
-    expect(risk.assessment).toBeDefined();
-    expect(risk.assessment.tier).toBeTruthy();
-    // ─── Phase 4: VERIFICATION CYCLE — Track implementation ───
-    const cycle = (await callTool("start_verification_cycle", {
-        title: `swebench-${taskId}`,
-        description: task.problem_statement.slice(0, 200),
-    }, taskId, "verification"));
-    expect(cycle.cycleId).toBeTruthy();
-    cleanupIds.cycleIds.push(cycle.cycleId);
-    // Phase 1 findings (context gathering)
-    await callTool("log_phase_findings", {
-        cycleId: cycle.cycleId,
-        phaseNumber: 1,
-        status: "passed",
-        findings: {
-            repo: task.repo,
-            category: task.category,
-            complexity: task.complexity,
-            reconFindings: reconSummary.totalFindings,
-        },
-    }, taskId, "verification");
-    // Log a gap (every real task has at least one)
-    const severityMap = { low: "LOW", medium: "MEDIUM", high: "HIGH" };
-    const gap = (await callTool("log_gap", {
-        cycleId: cycle.cycleId,
-        severity: severityMap[task.complexity],
-        title: task.problem_statement.split(".")[0],
-        description: task.problem_statement,
-        rootCause: `Identified via recon session ${reconSession.sessionId}`,
-        fixStrategy: `Apply ${task.category} patch following ${task.repo} conventions`,
-    }, taskId, "verification"));
-    expect(gap.gapId).toBeTruthy();
-    // Resolve the gap
-    const resolved = (await callTool("resolve_gap", { gapId: gap.gapId }, taskId, "verification"));
-    expect(resolved.status).toBe("resolved");
-    // Log test results across layers
-    const testLayers = ["static", "unit", "integration"];
-    for (const layer of testLayers) {
-        await callTool("log_test_result", {
-            cycleId: cycle.cycleId,
-            layer,
-            label: `${taskId}-${layer}`,
-            passed: true,
-            output: `${layer} tests passing for ${task.repo}`,
-        }, taskId, "verification");
-    }
-    // Check verification status
-    const status = (await callTool("get_verification_status", { cycleId: cycle.cycleId }, taskId, "verification"));
-    expect(status.status).toBeTruthy();
-    // ─── Phase 5: EVAL RUN — Score the implementation ───
-    const evalRun = (await callTool("start_eval_run", {
-        name: `swebench-eval-${taskId}`,
-        description: `Eval for ${task.repo} ${task.category}`,
-        cases: [
-            {
-                input: task.problem_statement.slice(0, 100),
-                intent: `Fix ${task.category} in ${task.repo}`,
-            },
-            {
-                input: `Regression test for ${taskId}`,
-                intent: "Ensure no regression",
-            },
-        ],
-    }, taskId, "eval"));
-    expect(evalRun.runId).toBeTruthy();
-    // Record results
-    for (let i = 0; i < evalRun.caseIds.length; i++) {
-        await callTool("record_eval_result", {
-            caseId: evalRun.caseIds[i],
-            actual: `Verified ${task.category} fix applied correctly`,
-            verdict: "pass",
-            score: task.complexity === "high" ? 0.85 : task.complexity === "medium" ? 0.9 : 0.95,
-        }, taskId, "eval");
-    }
-    // Complete eval
-    const evalComplete = (await callTool("complete_eval_run", { runId: evalRun.runId }, taskId, "eval"));
-    expect(evalComplete.status).toBe("completed");
-    expect(evalComplete.summary).toBeDefined();
-    // ─── Phase 6: QUALITY GATE — Gate deployment ───
-    const gate = (await callTool("run_quality_gate", {
-        gateName: "deploy_readiness",
-        target: taskId,
-        rules: [
-            { name: "tests_pass", passed: true },
-            { name: "no_type_errors", passed: true },
-            { name: "no_lint_errors", passed: true },
-            { name: "coverage_threshold", passed: true },
-        ],
-    }, taskId, "quality-gate"));
-    expect(gate.passed).toBe(true);
-    // Closed loop
-    await callTool("run_closed_loop", {
-        steps: [
-            { step: "compile", passed: true },
-            { step: "lint", passed: true },
-            { step: "test", passed: true },
-        ],
-    }, taskId, "quality-gate");
-    // ─── Phase 7: KNOWLEDGE — Record learning ───
-    const learningKey = `swebench-${taskId}-${Date.now()}`;
-    cleanupIds.learningKeys.push(learningKey);
-    await callTool("record_learning", {
-        key: learningKey,
-        category: "pattern",
-        content: `${task.repo} ${task.category}: ${task.problem_statement.slice(0, 150)}`,
-        tags: [task.category, task.complexity, task.repo.split("/")[0]],
-    }, taskId, "knowledge");
-    // Unified search
-    const knowledge = (await callTool("search_all_knowledge", { query: task.repo.split("/")[1] }, taskId, "knowledge"));
-    expect(knowledge).toHaveProperty("learnings");
-    expect(knowledge).toHaveProperty("reconFindings");
-    expect(knowledge).toHaveProperty("gaps");
-    // ─── Phase 8: FLYWHEEL — Connect verification to eval ───
-    const flywheel = (await callTool("run_mandatory_flywheel", {
-        target: `SWE-bench ${taskId}`,
-        steps: [
-            { stepName: "static_analysis", passed: true },
-            { stepName: "happy_path_test", passed: true },
-            { stepName: "failure_path_test", passed: true },
-            { stepName: "gap_analysis", passed: true },
-            { stepName: "fix_and_reverify", passed: true },
-            { stepName: "deploy_and_document", passed: true },
-        ],
-    }, taskId, "flywheel"));
-    expect(flywheel.passed).toBe(true);
-    return cleanupIds;
-}
-/**
- * Cleanup function to abandon cycles and delete learnings after tests
- */
-async function cleanup(ids) {
-    for (const cycleId of ids.cycleIds) {
-        try {
-            await findTool("abandon_cycle").handler({
-                cycleId,
-                reason: "dataset bench cleanup",
-            });
-        }
-        catch {
-            // already abandoned or completed
-        }
-    }
-    for (const key of ids.learningKeys) {
-        try {
-            await findTool("delete_learning").handler({ key });
-        }
-        catch {
-            // already deleted
-        }
-    }
-}
-// ═══════════════════════════════════════════════════════════════════════════
-// TEST BATCHES — 4 parallel batches of 5 tasks each
-// ═══════════════════════════════════════════════════════════════════════════
-const BATCH_SIZE = 5;
-const batches = [];
-for (let i = 0; i < SWE_BENCH_TASKS.length; i += BATCH_SIZE) {
-    batches.push(SWE_BENCH_TASKS.slice(i, i + BATCH_SIZE));
-}
-describe("SWE-bench Dataset Bench: Batch 1 (django + scikit-learn bugs)", () => {
-    const batch = batches[0];
-    const allCleanup = [];
-    afterAll(async () => {
-        for (const ids of allCleanup)
-            await cleanup(ids);
-    });
-    for (const task of batch) {
-        it(`Full pipeline: ${task.instance_id} (${task.category}/${task.complexity})`, async () => {
-            const ids = await runFullPipeline(task);
-            allCleanup.push(ids);
-        }, 30_000);
-    }
-});
-describe("SWE-bench Dataset Bench: Batch 2 (features + refactors)", () => {
-    const batch = batches[1];
-    const allCleanup = [];
-    afterAll(async () => {
-        for (const ids of allCleanup)
-            await cleanup(ids);
-    });
-    for (const task of batch) {
-        it(`Full pipeline: ${task.instance_id} (${task.category}/${task.complexity})`, async () => {
-            const ids = await runFullPipeline(task);
-            allCleanup.push(ids);
-        }, 30_000);
-    }
-});
-describe("SWE-bench Dataset Bench: Batch 3 (cross-repo medium complexity)", () => {
-    const batch = batches[2];
-    const allCleanup = [];
-    afterAll(async () => {
-        for (const ids of allCleanup)
-            await cleanup(ids);
-    });
-    for (const task of batch) {
-        it(`Full pipeline: ${task.instance_id} (${task.category}/${task.complexity})`, async () => {
-            const ids = await runFullPipeline(task);
-            allCleanup.push(ids);
-        }, 30_000);
-    }
-});
-describe("SWE-bench Dataset Bench: Batch 4 (high complexity + API changes)", () => {
-    const batch = batches[3];
-    const allCleanup = [];
-    afterAll(async () => {
-        for (const ids of allCleanup)
-            await cleanup(ids);
-    });
-    for (const task of batch) {
-        it(`Full pipeline: ${task.instance_id} (${task.category}/${task.complexity})`, async () => {
-            const ids = await runFullPipeline(task);
-            allCleanup.push(ids);
-        }, 30_000);
-    }
-});
-// ═══════════════════════════════════════════════════════════════════════════
-// CROSS-TASK INTEGRATION — Tests that span multiple tasks
-// ═══════════════════════════════════════════════════════════════════════════
-describe("Cross-Task: Eval Comparison (baseline vs candidate)", () => {
-    it("compares eval runs from two different SWE-bench tasks", async () => {
-        // Create baseline from a low-complexity task
-        const baseline = (await callTool("start_eval_run", {
-            name: "swebench-baseline-cross",
-            cases: [
-                { input: "django memoryview fix", intent: "Simple bug fix" },
-            ],
-        }, "cross-task", "eval"));
-        await callTool("record_eval_result", {
-            caseId: baseline.caseIds[0],
-            actual: "Fixed memoryview handling",
-            verdict: "pass",
-            score: 0.95,
-        }, "cross-task", "eval");
-        await callTool("complete_eval_run", { runId: baseline.runId }, "cross-task", "eval");
-        // Create candidate from a high-complexity task
-        const candidate = (await callTool("start_eval_run", {
-            name: "swebench-candidate-cross",
-            cases: [
-                {
-                    input: "astropy separability matrix fix",
-                    intent: "Complex nested model fix",
-                },
-            ],
-        }, "cross-task", "eval"));
-        await callTool("record_eval_result", {
-            caseId: candidate.caseIds[0],
-            actual: "Fixed separability computation",
-            verdict: "pass",
-            score: 0.85,
-        }, "cross-task", "eval");
-        await callTool("complete_eval_run", { runId: candidate.runId }, "cross-task", "eval");
-        // Compare
-        const comparison = (await callTool("compare_eval_runs", {
-            baselineRunId: baseline.runId,
-            candidateRunId: candidate.runId,
-        }, "cross-task", "eval"));
-        expect(comparison).toHaveProperty("recommendation");
-        expect(["DEPLOY", "REVERT", "INVESTIGATE"]).toContain(comparison.recommendation);
-    }, 30_000);
-});
-describe("Cross-Task: Promote Verification to Eval", () => {
-    it("promotes a verification cycle's findings into eval cases", async () => {
-        // Create a verification cycle
-        const cycle = (await callTool("start_verification_cycle", {
-            title: "swebench-promote-test",
-            description: "Testing promote_to_eval with SWE-bench data",
-        }, "cross-task-promote", "flywheel"));
-        // Promote to eval
-        const promoted = (await callTool("promote_to_eval", {
-            cycleId: cycle.cycleId,
-            evalRunName: "swebench-promoted-eval",
-            cases: [
-                {
-                    input: "django__django-11133: memoryview bug",
-                    intent: "Verify HttpResponse handles memoryview",
-                },
-                {
-                    input: "sympy__sympy-13372: evalf crash",
-                    intent: "Verify Max().evalf() doesn't crash",
-                },
-            ],
-        }, "cross-task-promote", "flywheel"));
-        expect(promoted.evalRunId).toBeTruthy();
-        expect(promoted.caseCount).toBe(2);
-        // Cleanup
-        await findTool("abandon_cycle").handler({
-            cycleId: cycle.cycleId,
-            reason: "bench cleanup",
-        });
-    }, 30_000);
-});
-describe("Cross-Task: Trigger Investigation from Regression", () => {
-    it("triggers investigation when eval run shows failures", async () => {
-        // Create a failing eval run
-        const evalRun = (await callTool("start_eval_run", {
-            name: "swebench-regression-detect",
-            cases: [
-                {
-                    input: "HistGradientBoosting string target",
-                    intent: "Verify early stopping with strings",
-                },
-            ],
-        }, "cross-task-investigate", "flywheel"));
-        await callTool("record_eval_result", {
-            caseId: evalRun.caseIds[0],
-            actual: "TypeError: comparison mismatch",
-            verdict: "fail",
-            score: 0.0,
-        }, "cross-task-investigate", "flywheel");
-        await callTool("complete_eval_run", { runId: evalRun.runId }, "cross-task-investigate", "flywheel");
-        // Trigger investigation
-        const investigation = (await callTool("trigger_investigation", {
-            evalRunId: evalRun.runId,
-            regressionDescription: "HistGradientBoosting fails with string targets when early stopping enabled",
-        }, "cross-task-investigate", "flywheel"));
-        expect(investigation.cycleId).toBeTruthy();
-        expect(investigation.title).toBeTruthy();
-        expect(investigation.linkedEvalRun).toBeTruthy();
-        // Cleanup
-        await findTool("abandon_cycle").handler({
-            cycleId: investigation.cycleId,
-            reason: "bench cleanup",
-        });
-    }, 30_000);
-});
-// ═══════════════════════════════════════════════════════════════════════════
-// DATASET BENCH REPORT
-// ═══════════════════════════════════════════════════════════════════════════
-describe("Dataset Bench Report", () => {
-    it("generates comprehensive SWE-bench evaluation report", () => {
-        const totalCalls = pipelineLog.length;
-        const successCalls = pipelineLog.filter((l) => l.success).length;
-        const failCalls = pipelineLog.filter((l) => !l.success).length;
-        // Unique tools used
-        const uniqueTools = new Set(pipelineLog.map((l) => l.tool));
-        // By phase
-        const byPhase = new Map();
-        pipelineLog.forEach((l) => {
-            if (!byPhase.has(l.phase)) {
-                byPhase.set(l.phase, {
-                    calls: 0,
-                    success: 0,
-                    fail: 0,
-                    tools: new Set(),
-                });
-            }
-            const p = byPhase.get(l.phase);
-            p.calls++;
-            if (l.success)
-                p.success++;
-            else
-                p.fail++;
-            p.tools.add(l.tool);
-        });
-        // By task
-        const byTask = new Map();
-        pipelineLog.forEach((l) => {
-            if (!byTask.has(l.taskId)) {
-                byTask.set(l.taskId, {
-                    calls: 0,
-                    success: 0,
-                    fail: 0,
-                    phases: new Set(),
-                });
-            }
-            const t = byTask.get(l.taskId);
-            t.calls++;
-            if (l.success)
-                t.success++;
-            else
-                t.fail++;
-            t.phases.add(l.phase);
-        });
-        // By category
-        const byCategory = new Map();
-        SWE_BENCH_TASKS.forEach((task) => {
-            const taskLog = pipelineLog.filter((l) => l.taskId === task.instance_id);
-            if (taskLog.length === 0)
-                return;
-            if (!byCategory.has(task.category)) {
-                byCategory.set(task.category, { count: 0, success: 0, fail: 0 });
-            }
-            const c = byCategory.get(task.category);
-            c.count++;
-            c.success += taskLog.filter((l) => l.success).length;
-            c.fail += taskLog.filter((l) => !l.success).length;
-        });
-        // Average duration by phase
-        const durationByPhase = new Map();
-        pipelineLog.forEach((l) => {
-            if (!durationByPhase.has(l.phase))
-                durationByPhase.set(l.phase, []);
-            durationByPhase.get(l.phase).push(l.durationMs);
-        });
-        // Print report
-        console.log("\n");
-        console.log("╔═══════════════════════════════════════════════════════════════════════════╗");
-        console.log("║           SWE-BENCH DATASET BENCH — PROOF OF WORK REPORT                ║");
-        console.log("╚═══════════════════════════════════════════════════════════════════════════╝");
-        console.log("");
-        console.log("  Dataset: SWE-bench Verified (princeton-nlp/SWE-bench_Verified)");
-        console.log(`  Tasks:   ${SWE_BENCH_TASKS.length} real GitHub issues from ${new Set(SWE_BENCH_TASKS.map((t) => t.repo)).size} repositories`);
-        console.log("");
-        console.log("┌─────────────────────────────────────────────────────────────────────────────┐");
-        console.log("│ PIPELINE SUMMARY                                                            │");
-        console.log("├─────────────────────────────────────────────────────────────────────────────┤");
-        console.log(`│ Total Tool Calls:             ${String(totalCalls).padStart(4)}                                       │`);
-        console.log(`│ Unique Tools Exercised:       ${String(uniqueTools.size).padStart(4)}                                       │`);
-        console.log(`│ Success Rate:                 ${successCalls}/${totalCalls} (${Math.round((successCalls / totalCalls) * 100)}%)                                  │`);
-        console.log(`│ Tasks Completed:              ${String(byTask.size).padStart(4)}                                       │`);
-        console.log(`│ Pipeline Phases:              ${String(byPhase.size).padStart(4)}                                       │`);
-        console.log("└─────────────────────────────────────────────────────────────────────────────┘");
-        console.log("");
-        // Phase breakdown
-        console.log("┌─────────────────────────────────────────────────────────────────────────────┐");
-        console.log("│ PHASE BREAKDOWN                                                             │");
-        console.log("├─────────────────────────────────────────────────────────────────────────────┤");
-        byPhase.forEach((data, phase) => {
-            const avgMs = Math.round((durationByPhase.get(phase) || []).reduce((a, b) => a + b, 0) /
-                (durationByPhase.get(phase) || [1]).length);
-            const status = data.fail === 0 ? "OK" : "FAIL";
-            const line = `│ ${status.padEnd(4)} ${phase.padEnd(16)} ${String(data.calls).padStart(3)} calls  ${String(data.tools.size).padStart(2)} tools  avg ${String(avgMs).padStart(4)}ms`;
-            console.log(line.padEnd(78) + "│");
-        });
-        console.log("└─────────────────────────────────────────────────────────────────────────────┘");
-        console.log("");
-        // Category breakdown
-        console.log("┌─────────────────────────────────────────────────────────────────────────────┐");
-        console.log("│ TASK CATEGORY BREAKDOWN                                                     │");
-        console.log("├─────────────────────────────────────────────────────────────────────────────┤");
-        byCategory.forEach((data, category) => {
-            const pct = Math.round((data.success / (data.success + data.fail)) * 100);
-            const line = `│ ${category.padEnd(16)} ${String(data.count).padStart(2)} tasks  ${String(data.success).padStart(3)} ok  ${String(data.fail).padStart(2)} fail  (${String(pct).padStart(3)}%)`;
-            console.log(line.padEnd(78) + "│");
-        });
-        console.log("└─────────────────────────────────────────────────────────────────────────────┘");
-        console.log("");
-        // Per-task results
-        console.log("┌─────────────────────────────────────────────────────────────────────────────┐");
-        console.log("│ PER-TASK RESULTS                                                            │");
-        console.log("├─────────────────────────────────────────────────────────────────────────────┤");
-        byTask.forEach((data, taskId) => {
-            const task = SWE_BENCH_TASKS.find((t) => t.instance_id === taskId);
-            const status = data.fail === 0 ? "PASS" : "FAIL";
-            const label = task
-                ? `${taskId.slice(0, 35)}`
-                : taskId.slice(0, 35);
-            const line = `│ ${status.padEnd(4)} ${label.padEnd(37)} ${String(data.calls).padStart(2)} calls  ${String(data.phases.size).padStart(1)} phases`;
-            console.log(line.padEnd(78) + "│");
-        });
-        console.log("└─────────────────────────────────────────────────────────────────────────────┘");
-        console.log("");
-        // Tools used
-        console.log("┌─────────────────────────────────────────────────────────────────────────────┐");
-        console.log("│ TOOLS EXERCISED IN DATASET BENCH                                            │");
-        console.log("├─────────────────────────────────────────────────────────────────────────────┤");
-        const toolCounts = new Map();
-        pipelineLog.forEach((l) => {
-            toolCounts.set(l.tool, (toolCounts.get(l.tool) || 0) + 1);
-        });
-        const sortedTools = [...toolCounts.entries()].sort((a, b) => b[1] - a[1]);
-        for (const [tool, count] of sortedTools) {
-            const line = `│   ${tool.padEnd(30)} ${String(count).padStart(4)}x`;
-            console.log(line.padEnd(78) + "│");
-        }
-        console.log("└─────────────────────────────────────────────────────────────────────────────┘");
-        console.log("");
-        // Verdict
-        console.log("┌─────────────────────────────────────────────────────────────────────────────┐");
-        console.log("│ VERDICT                                                                      │");
-        console.log("├─────────────────────────────────────────────────────────────────────────────┤");
-        if (failCalls === 0) {
-            console.log("│   PASS: All 20 SWE-bench tasks completed full pipeline successfully        │");
-            console.log(`│   ${totalCalls} tool calls across ${byPhase.size} phases, 0 failures                           │`);
-            console.log("│   MCP tools can orchestrate real-world development workflows end-to-end    │");
-        }
-        else {
-            console.log(`│   PARTIAL: ${failCalls} tool calls failed across ${byTask.size} tasks                              │`.padEnd(78) + "│");
-        }
-        console.log("└─────────────────────────────────────────────────────────────────────────────┘");
-        console.log("");
-        // Assertions
-        expect(failCalls).toBe(0);
-        expect(uniqueTools.size).toBeGreaterThanOrEqual(15);
-        expect(byTask.size).toBeGreaterThanOrEqual(20); // All 20 tasks + cross-task tests
-    });
-});
-//# sourceMappingURL=evalDatasetBench.test.js.map