nodebench-mcp 1.2.0 → 1.4.0

package/dist/__tests__/evalHarness.test.js

@@ -0,0 +1,830 @@
+/**
+ * Eval Harness for NodeBench MCP Tools
+ *
+ * Tests REAL agent scenarios to prove tools work in practice.
+ * Each scenario exercises multiple tools in realistic workflows.
+ *
+ * Coverage Goals:
+ * - Every tool called at least once
+ * - Every methodology workflow tested
+ * - Cross-tool integration verified
+ */
+import { describe, it, expect } from "vitest";
+import { verificationTools } from "../tools/verificationTools.js";
+import { reconTools } from "../tools/reconTools.js";
+import { evalTools } from "../tools/evalTools.js";
+import { qualityGateTools } from "../tools/qualityGateTools.js";
+import { flywheelTools } from "../tools/flywheelTools.js";
+import { learningTools } from "../tools/learningTools.js";
+import { documentationTools } from "../tools/documentationTools.js";
+import { agentBootstrapTools } from "../tools/agentBootstrapTools.js";
+import { selfEvalTools } from "../tools/selfEvalTools.js";
+import { createMetaTools } from "../tools/metaTools.js";
+// Assemble all tools
+const domainTools = [
+    ...verificationTools,
+    ...evalTools,
+    ...qualityGateTools,
+    ...learningTools,
+    ...flywheelTools,
+    ...reconTools,
+    ...documentationTools,
+    ...agentBootstrapTools,
+    ...selfEvalTools,
+];
+const allTools = [...domainTools, ...createMetaTools(domainTools)];
+const findTool = (name) => {
+    const tool = allTools.find((t) => t.name === name);
+    if (!tool)
+        throw new Error(`Tool not found: ${name}`);
+    return tool;
+};
+// Track which tools are called
+const toolCallLog = [];
+async function callTool(name, args, scenario) {
+    const tool = findTool(name);
+    try {
+        const result = await tool.handler(args);
+        toolCallLog.push({ tool: name, scenario, success: true });
+        return result;
+    }
+    catch (error) {
+        toolCallLog.push({ tool: name, scenario, success: false });
+        throw error;
+    }
+}
+// ═══════════════════════════════════════════════════════════════════════════
+// SCENARIO 1: New Feature Development (verification methodology)
+// ═══════════════════════════════════════════════════════════════════════════
+describe("Scenario: New Feature Development", () => {
+    let cycleId;
+    let gapId;
+    it("Step 1: Start verification cycle", async () => {
+        const result = await callTool("start_verification_cycle", {
+            title: "eval-feature-development",
+            description: "Implementing user authentication",
+        }, "feature-dev");
+        expect(result.cycleId).toBeTruthy();
+        cycleId = result.cycleId;
+    });
+    it("Step 2: Log context gathering (Phase 1)", async () => {
+        const result = await callTool("log_phase_findings", {
+            cycleId,
+            phaseNumber: 1,
+            status: "passed",
+            findings: { patterns: ["JWT", "session-based"], recommendation: "use JWT" },
+        }, "feature-dev");
+        expect(result.phaseRecorded).toBe(1);
+    });
+    it("Step 3: Log a gap found during implementation", async () => {
+        const result = await callTool("log_gap", {
+            cycleId,
+            severity: "MEDIUM",
+            title: "Missing rate limiting",
+            description: "Auth endpoint needs rate limiting",
+            rootCause: "Security oversight",
+            fixStrategy: "Add express-rate-limit middleware",
+        }, "feature-dev");
+        expect(result.gapId).toBeTruthy();
+        gapId = result.gapId;
+    });
+    it("Step 4: Get verification status", async () => {
+        const result = await callTool("get_verification_status", {
+            cycleId,
+        }, "feature-dev");
+        // Tool returns status (active/completed/abandoned), currentPhase, etc.
+        expect(result.status).toBeTruthy();
+    });
+    it("Step 5: Resolve the gap", async () => {
+        const result = await callTool("resolve_gap", {
+            gapId,
+        }, "feature-dev");
+        expect(result.status).toBe("resolved");
+    });
+    it("Step 6: Log test result", async () => {
+        const result = await callTool("log_test_result", {
+            cycleId,
+            layer: "integration", // Required field
+            label: "auth-integration-test", // Required field (not testName)
+            passed: true,
+            output: "All auth flows passing",
+        }, "feature-dev");
+        expect(result.testId).toBeTruthy();
+    });
+    it("Step 7: Cleanup - abandon cycle", async () => {
+        const result = await callTool("abandon_cycle", {
+            cycleId,
+            reason: "eval harness cleanup",
+        }, "feature-dev");
+        expect(result.abandoned).toBe(true);
+    });
+});
+// ═══════════════════════════════════════════════════════════════════════════
+// SCENARIO 2: Eval-Driven Development (eval methodology)
+// ═══════════════════════════════════════════════════════════════════════════
+describe("Scenario: Eval-Driven Development", () => {
+    let evalRunId;
+    let caseIds;
+    it("Step 1: Start eval run with test cases", async () => {
+        // Actual schema: name, cases (with input, intent, expected)
+        const result = await callTool("start_eval_run", {
+            name: "eval-harness-run",
+            description: "Testing prompt quality",
+            cases: [
+                { input: "Hello", intent: "greeting" },
+                { input: "Help me code", intent: "assistance" },
+            ],
+        }, "eval-driven");
+        expect(result.runId).toBeTruthy();
+        evalRunId = result.runId;
+        caseIds = result.caseIds;
+    });
+    it("Step 2: Record eval results", async () => {
+        // Actual schema: caseId, verdict (pass/fail/partial), actual, score
+        const result1 = await callTool("record_eval_result", {
+            caseId: caseIds[0],
+            actual: "greeting response",
+            verdict: "pass",
+            score: 0.9,
+        }, "eval-driven");
+        expect(result1.caseId).toBe(caseIds[0]);
+        expect(result1.verdict).toBe("pass");
+        const result2 = await callTool("record_eval_result", {
+            caseId: caseIds[1],
+            actual: "help response",
+            verdict: "pass",
+            score: 0.85,
+        }, "eval-driven");
+        expect(result2.caseId).toBe(caseIds[1]);
+    });
+    it("Step 3: Complete eval run", async () => {
+        const result = await callTool("complete_eval_run", {
+            runId: evalRunId,
+        }, "eval-driven");
+        expect(result.runId).toBe(evalRunId);
+        expect(result.status).toBe("completed");
+        expect(result.summary).toBeDefined();
+    });
+    it("Step 4: List eval runs", async () => {
+        const result = await callTool("list_eval_runs", {
+            limit: 10,
+        }, "eval-driven");
+        expect(result.runs).toBeDefined();
+        expect(result.runs.length).toBeGreaterThan(0);
+    });
+});
+// ═══════════════════════════════════════════════════════════════════════════
+// SCENARIO 3: Knowledge Management (learning methodology)
+// ═══════════════════════════════════════════════════════════════════════════
+describe("Scenario: Knowledge Management", () => {
+    const uniqueKey = `eval-learning-${Date.now()}`;
+    it("Step 1: Record a learning", async () => {
+        const result = await callTool("record_learning", {
+            key: uniqueKey,
+            category: "pattern",
+            content: "Use scenario-based testing to verify tool chains work together",
+            tags: ["testing", "eval", "integration"],
+        }, "knowledge");
+        expect(result.key).toBe(uniqueKey);
+        expect(result.success).toBe(true);
+    });
+    it("Step 2: Search for the learning", async () => {
+        // Returns { query, count, learnings: [...] }
+        const result = await callTool("search_learnings", {
+            query: "scenario testing",
+        }, "knowledge");
+        expect(result.learnings).toBeDefined();
+    });
+    it("Step 3: List all learnings", async () => {
+        const result = await callTool("list_learnings", {
+            limit: 20,
+        }, "knowledge");
+        expect(result.learnings).toBeDefined();
+    });
+    it("Step 4: Delete the learning", async () => {
+        // Returns { success: true, message }
+        const result = await callTool("delete_learning", {
+            key: uniqueKey,
+        }, "knowledge");
+        expect(result.success).toBe(true);
+    });
+});
+// ═══════════════════════════════════════════════════════════════════════════
+// SCENARIO 4: Quality Gates (quality_gates methodology)
+// ═══════════════════════════════════════════════════════════════════════════
+describe("Scenario: Quality Gates", () => {
+    it("Step 1: Get deploy_readiness preset", async () => {
+        const result = await callTool("get_gate_preset", {
+            preset: "deploy_readiness",
+        }, "quality-gates");
+        expect(result.preset).toBe("deploy_readiness");
+        expect(result.rules.length).toBeGreaterThan(0);
+    });
+    it("Step 2: Run quality gate", async () => {
+        const result = await callTool("run_quality_gate", {
+            gateName: "deploy_readiness",
+            target: "eval-harness-test",
+            rules: [
+                { name: "tests_pass", passed: true },
+                { name: "no_type_errors", passed: true },
+                { name: "no_lint_errors", passed: true },
+                { name: "coverage_threshold", passed: false },
+            ],
+        }, "quality-gates");
+        expect(result.passed).toBe(false);
+        expect(result.failures).toContain("coverage_threshold");
+    });
+    it("Step 3: Get gate history", async () => {
+        // Returns { gateName, runs: [...], trend }
+        const result = await callTool("get_gate_history", {
+            gateName: "deploy_readiness",
+            limit: 10,
+        }, "quality-gates");
+        expect(result.gateName).toBe("deploy_readiness");
+        expect(result.runs).toBeDefined();
+    });
+    it("Step 4: Run closed loop verification", async () => {
+        // Actual schema: steps with { step: enum, passed: boolean }
+        const result = await callTool("run_closed_loop", {
+            steps: [
+                { step: "compile", passed: true },
+                { step: "lint", passed: true },
+                { step: "test", passed: true },
+            ],
+        }, "quality-gates");
+        expect(result.allPassed).toBe(true);
+    });
+});
+// ═══════════════════════════════════════════════════════════════════════════
+// SCENARIO 5: Flywheel Orchestration (flywheel methodology)
+// ═══════════════════════════════════════════════════════════════════════════
+describe("Scenario: Flywheel Orchestration", () => {
+    it("Step 1: Get flywheel status", async () => {
+        // Returns { innerLoop, outerLoop, connections }
+        const result = await callTool("get_flywheel_status", {}, "flywheel");
+        expect(result).toHaveProperty("innerLoop");
+        expect(result).toHaveProperty("outerLoop");
+        expect(result).toHaveProperty("connections");
+    });
+    it("Step 2: Run mandatory flywheel check", async () => {
+        // Actual schema: target, steps array with stepName enum
+        const result = await callTool("run_mandatory_flywheel", {
+            target: "Added new auth feature",
+            steps: [
+                { stepName: "static_analysis", passed: true },
+                { stepName: "happy_path_test", passed: true },
+                { stepName: "failure_path_test", passed: true },
+                { stepName: "gap_analysis", passed: true },
+                { stepName: "fix_and_reverify", passed: true },
+                { stepName: "deploy_and_document", passed: true },
+            ],
+        }, "flywheel");
+        expect(result).toHaveProperty("passed");
+        expect(result.passed).toBe(true);
+    });
+});
+// ═══════════════════════════════════════════════════════════════════════════
+// SCENARIO 5.5: Flywheel Integration (promote, investigate, compare)
+// Tests the 4 previously untested flywheel integration tools
+// ═══════════════════════════════════════════════════════════════════════════
+describe("Scenario: Flywheel Integration", () => {
+    // Test the 4 previously untested flywheel tools in isolated tests
+    it("Step 1: list_verification_cycles - lists cycles", async () => {
+        // First create a cycle so we have something to list
+        const createResult = await callTool("start_verification_cycle", {
+            title: "List test cycle", // Note: title, not goal
+            description: "Testing list_verification_cycles",
+        }, "flywheel-integration");
+        const testCycleId = createResult.cycleId;
+        // Now list cycles
+        const result = await callTool("list_verification_cycles", {
+            limit: 10,
+        }, "flywheel-integration");
+        expect(result).toHaveProperty("count");
+        expect(result).toHaveProperty("cycles");
+        expect(Array.isArray(result.cycles)).toBe(true);
+        // Each cycle has cycleId property (not id)
+        expect(result.cycles.some((c) => c.cycleId === testCycleId)).toBe(true);
+        // Cleanup
+        await callTool("abandon_cycle", {
+            cycleId: testCycleId,
+            reason: "Test cleanup",
+        }, "flywheel-integration");
+    });
+    it("Step 2: promote_to_eval - promotes verification to eval suite", async () => {
+        // Create a cycle to promote from
+        const cycleResult = await callTool("start_verification_cycle", {
+            title: "Promote test cycle",
+            description: "Testing promote_to_eval",
+        }, "flywheel-integration");
+        // Promote with explicit cases (required)
+        const result = await callTool("promote_to_eval", {
+            cycleId: cycleResult.cycleId,
+            evalRunName: "promoted-eval-test",
+            cases: [
+                { input: "test input", intent: "Test intent" },
+            ],
+        }, "flywheel-integration");
+        expect(result).toHaveProperty("evalRunId");
+        expect(result).toHaveProperty("caseIds");
+        expect(result.caseCount).toBe(1);
+        // Cleanup
+        await callTool("abandon_cycle", {
+            cycleId: cycleResult.cycleId,
+            reason: "Test cleanup",
+        }, "flywheel-integration");
+    });
+    it("Step 3: compare_eval_runs - compares two completed evals", async () => {
+        // Create and complete baseline eval
+        const baseline = await callTool("start_eval_run", {
+            name: "baseline-for-compare",
+            cases: [{ input: "test", intent: "baseline" }],
+        }, "flywheel-integration");
+        await callTool("record_eval_result", {
+            caseId: baseline.caseIds[0],
+            actual: "result",
+            verdict: "pass",
+        }, "flywheel-integration");
+        await callTool("complete_eval_run", {
+            runId: baseline.runId,
+        }, "flywheel-integration");
+        // Create and complete candidate eval
+        const candidate = await callTool("start_eval_run", {
+            name: "candidate-for-compare",
+            cases: [{ input: "test", intent: "candidate" }],
+        }, "flywheel-integration");
+        await callTool("record_eval_result", {
+            caseId: candidate.caseIds[0],
+            actual: "result",
+            verdict: "pass",
+        }, "flywheel-integration");
+        await callTool("complete_eval_run", {
+            runId: candidate.runId,
+        }, "flywheel-integration");
+        // Compare them
+        const result = await callTool("compare_eval_runs", {
+            baselineRunId: baseline.runId,
+            candidateRunId: candidate.runId,
+        }, "flywheel-integration");
+        expect(result).toHaveProperty("recommendation");
+        expect(["DEPLOY", "REVERT", "INVESTIGATE"]).toContain(result.recommendation);
+    });
+    it("Step 4: trigger_investigation - creates investigation cycle", async () => {
+        // Create and complete an eval run to investigate
+        const eval1 = await callTool("start_eval_run", {
+            name: "eval-to-investigate",
+            cases: [{ input: "test", intent: "investigate" }],
+        }, "flywheel-integration");
+        await callTool("record_eval_result", {
+            caseId: eval1.caseIds[0],
+            actual: "failed",
+            verdict: "fail",
+        }, "flywheel-integration");
+        await callTool("complete_eval_run", {
+            runId: eval1.runId,
+        }, "flywheel-integration");
+        // Trigger investigation
+        const result = await callTool("trigger_investigation", {
+            evalRunId: eval1.runId,
+            regressionDescription: "Test failure detected",
+        }, "flywheel-integration");
+        // Returns cycleId, title, linkedEvalRun, phase1Instructions
+        expect(result).toHaveProperty("cycleId");
+        expect(result).toHaveProperty("title");
+        expect(result).toHaveProperty("linkedEvalRun");
+        // Cleanup
+        await callTool("abandon_cycle", {
+            cycleId: result.cycleId,
+            reason: "Test cleanup",
+        }, "flywheel-integration");
+    });
+});
+// ═══════════════════════════════════════════════════════════════════════════
+// SCENARIO 6: Research & Discovery (recon methodology)
+// ═══════════════════════════════════════════════════════════════════════════
+describe("Scenario: Research & Discovery", () => {
+    let reconSessionId;
+    it("Step 1: Start recon session", async () => {
+        // Actual schema: target (required), description, projectContext
+        const result = await callTool("run_recon", {
+            target: "MCP server best practices",
+            description: "Research for eval harness",
+        }, "research");
+        expect(result.sessionId).toBeTruthy();
+        reconSessionId = result.sessionId;
+    });
+    it("Step 2: Log recon finding", async () => {
+        // Actual schema: sessionId, category (enum), summary, sourceUrl, relevance
+        const result = await callTool("log_recon_finding", {
+            sessionId: reconSessionId,
+            category: "best_practice",
+            summary: "Organize tools by domain for better discoverability",
+            sourceUrl: "https://docs.anthropic.com",
+            relevance: "Applies to MCP tool organization",
+        }, "research");
+        expect(result.findingId).toBeTruthy();
+        expect(result.findingCount).toBeGreaterThan(0);
+    });
+    it("Step 3: Get recon summary", async () => {
+        // Returns { sessionId, target, status, totalFindings, findingsByCategory, ... }
+        const result = await callTool("get_recon_summary", {
+            sessionId: reconSessionId,
+        }, "research");
+        expect(result.sessionId).toBe(reconSessionId);
+        expect(result.totalFindings).toBeGreaterThan(0);
+    });
+    it("Step 4: Check framework updates", async () => {
+        // Actual schema: ecosystem (enum)
+        const result = await callTool("check_framework_updates", {
+            ecosystem: "mcp",
+        }, "research");
+        expect(result.ecosystem).toBe("mcp");
+        expect(result.sources).toBeDefined();
+    });
+    it("Step 5: Bootstrap project context", async () => {
+        // Actual schema: projectName (required), techStack, architecture, etc.
+        const result = await callTool("bootstrap_project", {
+            projectName: "eval-harness-project",
+            techStack: "TypeScript, Vitest, MCP",
+            architecture: "Modular tool system",
+        }, "research");
+        expect(result.projectName).toBe("eval-harness-project");
+        expect(result.storedFields).toBeDefined();
+    });
+    it("Step 6: Get project context", async () => {
+        // Returns { context: {}, knowledgeBase: {} }
+        const result = await callTool("get_project_context", {}, "research");
+        expect(result).toHaveProperty("context");
+        expect(result).toHaveProperty("knowledgeBase");
+    });
+    it("Step 7: Search all knowledge", async () => {
+        const result = await callTool("search_all_knowledge", {
+            query: "MCP tools",
+        }, "research");
+        expect(result).toHaveProperty("learnings");
+        expect(result).toHaveProperty("reconFindings");
+        expect(result).toHaveProperty("gaps");
+    });
+});
+// ═══════════════════════════════════════════════════════════════════════════
+// SCENARIO 7: Agent Self-Bootstrap (agent_bootstrap methodology)
+// ═══════════════════════════════════════════════════════════════════════════
+describe("Scenario: Agent Self-Bootstrap", () => {
+    it("Step 1: Discover infrastructure", async () => {
+        const result = await callTool("discover_infrastructure", {
+            categories: ["agent_loop", "telemetry"],
+            depth: "shallow",
+        }, "bootstrap");
+        expect(result).toHaveProperty("discovered");
+        expect(result).toHaveProperty("missing");
+    });
+    it("Step 2: Triple verify a component", async () => {
+        const result = await callTool("triple_verify", {
+            target: "verification-tools",
+            scope: "implementation", // Valid: implementation|integration|deployment|full
+            includeWebSearch: false,
+        }, "bootstrap");
+        // Returns verification1_internal, verification2_external, verification3_synthesis
+        expect(result).toHaveProperty("verification1_internal");
+        expect(result).toHaveProperty("verification2_external");
+        expect(result).toHaveProperty("verification3_synthesis");
+    });
+    it("Step 3: Self-implement missing component", async () => {
+        const result = await callTool("self_implement", {
+            component: "telemetry",
+            dryRun: true,
+        }, "bootstrap");
+        expect(result).toHaveProperty("component");
+        // Returns plan, files, nextSteps
+        expect(result).toHaveProperty("plan");
+        expect(result).toHaveProperty("files");
+    });
+    it("Step 4: Generate self-instructions", async () => {
+        const result = await callTool("generate_self_instructions", {
+            format: "claude_md",
+            includeExternalSources: false,
+        }, "bootstrap");
+        expect(result).toHaveProperty("format");
+        // Returns content (not instructions)
+        expect(result).toHaveProperty("content");
+    });
+    it("Step 5: Connect channels", async () => {
+        const result = await callTool("connect_channels", {
+            channels: ["web", "github"],
+            query: "mcp tools",
+            aggressive: false,
+        }, "bootstrap");
+        // Returns query, results (array of {channel, findings, sources})
+        expect(result).toHaveProperty("query");
+        expect(result).toHaveProperty("results");
+        expect(Array.isArray(result.results)).toBe(true);
+    });
+});
+// ═══════════════════════════════════════════════════════════════════════════
+// SCENARIO 8: Autonomous Maintenance (autonomous_maintenance methodology)
+// ═══════════════════════════════════════════════════════════════════════════
+describe("Scenario: Autonomous Maintenance", () => {
+    it("Step 1: Assess risk before action", async () => {
+        const result = await callTool("assess_risk", {
+            action: "update_agents_md",
+            context: "Adding new documentation",
+        }, "autonomous");
+        expect(result.assessment.tier).toBe("medium");
+        expect(result.assessment.recommendation).toBe("log_and_proceed");
+    });
+    it("Step 2: Decide re-update vs create", async () => {
+        const result = await callTool("decide_re_update", {
+            targetContent: "New methodology documentation",
+            contentType: "documentation",
+            existingFiles: ["README.md", "AGENTS.md"],
+        }, "autonomous");
+        expect(["update_existing", "create_new", "merge"]).toContain(result.action);
+    });
+    it("Step 3: Run self-maintenance", async () => {
+        const result = await callTool("run_self_maintenance", {
+            scope: "quick",
+            autoFix: false,
+            dryRun: true,
+        }, "autonomous");
+        expect(result).toHaveProperty("checksPerformed");
+        expect(result).toHaveProperty("issuesFound");
+    });
+    it("Step 4: Scaffold directory structure", async () => {
+        const result = await callTool("scaffold_directory", {
+            component: "agent_loop",
+            includeTests: true,
+            dryRun: true,
+        }, "autonomous");
+        expect(result.component).toBe("agent_loop");
+        expect(result.structure.files.length).toBeGreaterThan(0);
+    });
+    it("Step 5: Run autonomous loop with guardrails", async () => {
+        const result = await callTool("run_autonomous_loop", {
+            goal: "Verify all documentation is in sync",
+            maxIterations: 3,
+            maxDurationMs: 5000,
+            stopOnFirstFailure: true,
+        }, "autonomous");
+        expect(result.goal).toBeTruthy();
+        expect(result.iterations).toBeLessThanOrEqual(3);
+        expect(["completed", "stopped", "timeout", "failed"]).toContain(result.status);
+    });
+});
+// ═══════════════════════════════════════════════════════════════════════════
+// SCENARIO 9: Meta Tools (tool discovery)
+// ═══════════════════════════════════════════════════════════════════════════
+describe("Scenario: Meta Tool Discovery", () => {
+    it("Step 1: Find tools by keyword", async () => {
+        const result = await callTool("findTools", {
+            query: "verification",
+        }, "meta");
+        expect(result.tools.length).toBeGreaterThan(0);
+    });
+    it("Step 2: Find tools by category", async () => {
+        const result = await callTool("findTools", {
+            category: "bootstrap",
+        }, "meta");
+        expect(result.tools.length).toBeGreaterThan(0);
+    });
+    it("Step 3: Get methodology overview", async () => {
+        const result = await callTool("getMethodology", {
+            topic: "overview",
+        }, "meta");
+        expect(result.title).toContain("Overview");
+        const topics = Object.keys(result.steps[0].topics);
+        expect(topics.length).toBe(17);
+    });
+    it("Step 4: Get specific methodology", async () => {
+        const methodologies = [
+            "verification", "eval", "flywheel", "mandatory_flywheel",
+            "reconnaissance", "quality_gates", "ui_ux_qa", "agentic_vision",
+            "closed_loop", "learnings", "project_ideation", "tech_stack_2026",
+            "telemetry_setup", "agents_md_maintenance", "agent_bootstrap",
+            "autonomous_maintenance",
+            "self_reinforced_learning",
+        ];
+        for (const topic of methodologies) {
+            const result = await callTool("getMethodology", { topic }, "meta");
+            expect(result.title).toBeTruthy();
+            expect(result.steps.length).toBeGreaterThan(0);
+        }
+    });
+});
+// ═══════════════════════════════════════════════════════════════════════════
+// SCENARIO 10: Self-Reinforced Learning (trajectory analysis)
+// ═══════════════════════════════════════════════════════════════════════════
+describe("Scenario: Self-Reinforced Learning", () => {
+    it("Step 1: Log tool calls to build trajectory data", async () => {
+        const result = await callTool("log_tool_call", {
+            sessionId: "eval-harness-self-eval",
+            toolName: "start_verification_cycle",
+            durationMs: 25,
+            resultStatus: "success",
+            phase: "verification",
+        }, "self-eval");
+        expect(result.logged).toBe(true);
+        await callTool("log_tool_call", {
+            sessionId: "eval-harness-self-eval",
+            toolName: "log_phase_findings",
+            durationMs: 12,
+            resultStatus: "success",
+            phase: "verification",
+        }, "self-eval");
+        await callTool("log_tool_call", {
+            sessionId: "eval-harness-self-eval",
+            toolName: "run_mandatory_flywheel",
+            durationMs: 35,
+            resultStatus: "success",
+            phase: "flywheel",
+        }, "self-eval");
+    });
+    it("Step 2: Analyze trajectory patterns", async () => {
+        const result = await callTool("get_trajectory_analysis", {
+            sessionId: "eval-harness-self-eval",
+        }, "self-eval");
+        expect(result.totalCalls).toBeGreaterThanOrEqual(3);
+        expect(result.uniqueTools).toBeGreaterThanOrEqual(3);
+        expect(result.topTools.length).toBeGreaterThan(0);
+    });
+    it("Step 3: Generate self-eval health report", async () => {
+        const result = await callTool("get_self_eval_report", {
+            sinceDaysAgo: 30,
+        }, "self-eval");
+        expect(typeof result.healthScore).toBe("number");
+        expect(result).toHaveProperty("verification");
+        expect(result).toHaveProperty("gaps");
+        expect(result).toHaveProperty("evalRuns");
+        expect(result).toHaveProperty("toolTrajectory");
+    });
+    it("Step 4: Get improvement recommendations", async () => {
+        const result = await callTool("get_improvement_recommendations", {
+            sinceDaysAgo: 30,
+            focus: "all",
+        }, "self-eval");
+        expect(typeof result.totalRecommendations).toBe("number");
+        expect(Array.isArray(result.recommendations)).toBe(true);
+        expect(result._selfReinforcement.nextSteps.length).toBe(4);
+    });
+});
+// ═══════════════════════════════════════════════════════════════════════════
+// COVERAGE REPORT
+// ═══════════════════════════════════════════════════════════════════════════
+describe("Coverage Report", () => {
+    it("should generate comprehensive Proof of Work report", () => {
+        const testedTools = new Set(toolCallLog.map(l => l.tool));
+        const allToolNames = allTools.map(t => t.name);
+        // Tools that require external dependencies (skip in automated tests)
+        const externalDependencyTools = [
+            "capture_ui_screenshot", // Requires Playwright
+            "capture_responsive_suite", // Requires Playwright
+            "discover_vision_env", // Dynamic SDK imports
+            "analyze_screenshot", // Requires AI API key
+            "manipulate_screenshot", // Requires Sharp
+            "web_search", // Requires AI API key
+            "fetch_url", // External network calls
+            "search_github", // Requires GitHub API
+            "analyze_repo", // Requires GitHub API
+            "update_agents_md", // File system - tested separately
+            "research_job_market", // Covered in tools.test.ts
+            "setup_local_env", // Covered in tools.test.ts
+        ];
+        // Deprecated tools (kept for backwards compatibility, but flagged)
+        const deprecatedTools = [
+            { tool: "search_learnings", reason: "DEPRECATED: Use search_all_knowledge instead" },
+            { tool: "list_learnings", reason: "DEPRECATED: Use search_all_knowledge instead" },
+        ];
+        const untestedTools = allToolNames.filter(name => !testedTools.has(name) && !externalDependencyTools.includes(name));
+        // Build tool-by-scenario matrix
+        const toolScenarioMap = new Map();
+        toolCallLog.forEach(l => {
+            if (!toolScenarioMap.has(l.tool))
+                toolScenarioMap.set(l.tool, []);
+            if (!toolScenarioMap.get(l.tool).includes(l.scenario)) {
+                toolScenarioMap.get(l.tool).push(l.scenario);
+            }
+        });
+        // Count successes and failures
+        const successCount = toolCallLog.filter(l => l.success).length;
+        const failureCount = toolCallLog.filter(l => !l.success).length;
+        // Build scenario summary
+        const byScenario = new Map();
+        toolCallLog.forEach(l => {
+            if (!byScenario.has(l.scenario)) {
+                byScenario.set(l.scenario, { tools: [], success: 0, fail: 0 });
+            }
+            const s = byScenario.get(l.scenario);
+            if (!s.tools.includes(l.tool))
+                s.tools.push(l.tool);
+            if (l.success)
+                s.success++;
+            else
+                s.fail++;
+        });
+        console.log("\n");
+        console.log("╔═══════════════════════════════════════════════════════════════════════════╗");
+        console.log("║                    NODEBENCH MCP - PROOF OF WORK REPORT                   ║");
+        console.log("╚═══════════════════════════════════════════════════════════════════════════╝");
+        console.log("");
+        console.log("┌─────────────────────────────────────────────────────────────────────────────┐");
+        console.log("│ SUMMARY                                                                     │");
+        console.log("├─────────────────────────────────────────────────────────────────────────────┤");
+        console.log(`│ Total Tools in MCP:           ${String(allToolNames.length).padStart(3)}                                         │`);
+        console.log(`│ Tools Tested in Scenarios:    ${String(testedTools.size).padStart(3)}  (${Math.round(testedTools.size / allToolNames.length * 100)}%)                                   │`);
+        console.log(`│ External Dependency (skip):   ${String(externalDependencyTools.length).padStart(3)}  (require API keys/network)               │`);
+        console.log(`│ Untested (GAPS):              ${String(untestedTools.length).padStart(3)}                                         │`);
+        console.log(`│ Total Tool Calls:             ${String(toolCallLog.length).padStart(3)}                                         │`);
+        console.log(`│ Success Rate:                 ${successCount}/${toolCallLog.length} (${Math.round(successCount / toolCallLog.length * 100)}%)                                  │`);
+        console.log("└─────────────────────────────────────────────────────────────────────────────┘");
+        console.log("");
+        // Scenario breakdown
+        console.log("┌─────────────────────────────────────────────────────────────────────────────┐");
+        console.log("│ SCENARIOS TESTED                                                            │");
+        console.log("├─────────────────────────────────────────────────────────────────────────────┤");
+        byScenario.forEach((data, scenario) => {
+            const status = data.fail === 0 ? "✓" : "✗";
+            const line = `│ ${status} ${scenario.padEnd(25)} ${String(data.tools.length).padStart(2)} tools, ${String(data.success).padStart(2)} calls`;
+            console.log(line.padEnd(78) + "│");
+        });
+        console.log("└─────────────────────────────────────────────────────────────────────────────┘");
+        console.log("");
+        // Tool coverage matrix (grouped by domain)
+        const domainMap = {
+            "Verification": ["start_verification_cycle", "log_phase_findings", "log_gap", "resolve_gap", "log_test_result", "get_verification_status", "list_verification_cycles", "abandon_cycle"],
+            "Eval": ["start_eval_run", "record_eval_result", "complete_eval_run", "compare_eval_runs", "list_eval_runs"],
+            "Quality Gates": ["run_quality_gate", "get_gate_preset", "get_gate_history", "run_closed_loop"],
+            "Learning": ["record_learning", "search_learnings", "list_learnings", "delete_learning"],
+            "Flywheel": ["get_flywheel_status", "promote_to_eval", "trigger_investigation", "run_mandatory_flywheel"],
+            "Recon": ["run_recon", "log_recon_finding", "get_recon_summary", "check_framework_updates", "search_all_knowledge", "bootstrap_project", "get_project_context"],
+            "Bootstrap": ["discover_infrastructure", "triple_verify", "self_implement", "generate_self_instructions", "connect_channels"],
+            "Autonomous": ["assess_risk", "decide_re_update", "run_self_maintenance", "scaffold_directory", "run_autonomous_loop"],
+            "Self-Eval": ["log_tool_call", "get_trajectory_analysis", "get_self_eval_report", "get_improvement_recommendations"],
+            "Meta": ["findTools", "getMethodology"],
+            "External (skip)": externalDependencyTools,
+        };
+        console.log("┌─────────────────────────────────────────────────────────────────────────────┐");
+        console.log("│ TOOL COVERAGE BY DOMAIN                                                     │");
+        console.log("├─────────────────────────────────────────────────────────────────────────────┤");
+        for (const [domain, tools] of Object.entries(domainMap)) {
+            const tested = tools.filter(t => testedTools.has(t)).length;
+            const total = tools.length;
+            const pct = Math.round(tested / total * 100);
+            const bar = "█".repeat(Math.round(pct / 10)) + "░".repeat(10 - Math.round(pct / 10));
+            const line = `│ ${domain.padEnd(18)} ${bar} ${String(tested).padStart(2)}/${String(total).padStart(2)} (${String(pct).padStart(3)}%)`;
+            console.log(line.padEnd(78) + "│");
+        }
+        console.log("└─────────────────────────────────────────────────────────────────────────────┘");
+        console.log("");
+        // Gaps
+        if (untestedTools.length > 0) {
+            console.log("┌─────────────────────────────────────────────────────────────────────────────┐");
+            console.log("│ ⚠ GAPS (Untested Tools)                                                    │");
+            console.log("├─────────────────────────────────────────────────────────────────────────────┤");
+            untestedTools.forEach(t => {
+                console.log(`│   - ${t}`.padEnd(78) + "│");
+            });
+            console.log("└─────────────────────────────────────────────────────────────────────────────┘");
+            console.log("");
+        }
+        // Deprecated tools analysis
+        console.log("┌─────────────────────────────────────────────────────────────────────────────┐");
+        console.log("│ ⚠️  DEPRECATED TOOLS                                                         │");
+        console.log("├─────────────────────────────────────────────────────────────────────────────┤");
+        if (deprecatedTools.length === 0) {
+            console.log("│   No deprecated tools.".padEnd(78) + "│");
+        }
+        else {
+            deprecatedTools.forEach(d => {
+                console.log(`│   - ${d.tool}: ${d.reason}`.slice(0, 77).padEnd(78) + "│");
+            });
+        }
+        console.log("│                                                                              │");
+        console.log("│   These tools are kept for backwards compatibility but return a             │");
+        console.log("│   deprecation notice. Use search_all_knowledge for unified search.          │");
+        console.log("└─────────────────────────────────────────────────────────────────────────────┘");
+        console.log("");
+        // Final verdict
+        const allCovered = untestedTools.length === 0;
+        const allPassed = failureCount === 0;
+        console.log("┌─────────────────────────────────────────────────────────────────────────────┐");
+        console.log("│ VERDICT                                                                      │");
+        console.log("├─────────────────────────────────────────────────────────────────────────────┤");
+        if (allCovered && allPassed) {
+            console.log("│   ✅ ALL TOOLS TESTED AND WORKING                                           │");
+            console.log("│   \"Yah it definitely works!\"                                                │");
+        }
+        else if (allPassed) {
+            console.log("│   ✅ ALL TESTED TOOLS WORKING                                               │");
+            console.log(`│   ⚠ ${untestedTools.length} tools not covered in scenario tests (see gaps above)`.padEnd(78) + "│");
+        }
+        else {
+            console.log(`│   ❌ ${failureCount} tool calls failed - investigate before shipping`.padEnd(78) + "│");
+        }
+        console.log("└─────────────────────────────────────────────────────────────────────────────┘");
+        console.log("");
+        // Assert minimum coverage
+        expect(testedTools.size).toBeGreaterThan(35); // Should test at least 35 tools
+        expect(untestedTools.length).toBe(0); // All non-external tools should be tested
+        expect(failureCount).toBe(0); // No failures allowed
+    });
+});
+//# sourceMappingURL=evalHarness.test.js.map