nodebench-mcp 2.11.0 → 2.13.0

package/dist/__tests__/presetRealWorldBench.test.js

@@ -0,0 +1,839 @@
+/**
+ * Preset Real-World Benchmark — Impact-Driven Evaluation
+ *
+ * Inspired by 8 open-source Claude Code ecosystem repos:
+ *   - obra/superpowers: Mandatory skill-check gate, 4-phase debugging
+ *   - wshobson/agents: Conductor (Context → Spec → Plan → Implement), agent-teams
+ *   - ruvnet/claude-flow: Queen-led swarm, 5-layer memory, 3-tier model routing
+ *   - Yeachan-Heo/oh-my-claudecode: Compaction-resilient notepad, learner skills
+ *   - thedotmack/claude-mem: Session observations, token economics, context config
+ *   - anthropic/planning-with-files: Manus-style markdown planning with checkpoints
+ *   - K-Dense-AI/claude-scientific-skills: 140 domain skills, category-based discovery
+ *   - zebbern/claude-code-guide: Best practices, workflow patterns, agent setup
+ *
+ * Fills gaps identified in existing eval suite:
+ *   GAP 1: Cross-domain workflows (domain silos → end-to-end)
+ *   GAP 2: Error recovery & failure paths
+ *   GAP 3: Preset transitions (meta → lite → core escalation)
+ *   GAP 4: Knowledge lifecycle (record → search → synthesize → reuse)
+ *   GAP 5: Research writing workflows (0% coverage → full pipeline)
+ *   GAP 6: Bootstrap cold-start (agent onboarding end-to-end)
+ *   GAP 7: Multi-agent coordination at scale
+ *   GAP 8: Progressive discovery search quality
+ *
+ * Architecture:
+ *   8 real-world scenarios × 4 presets (meta, lite, core, full) = 32 trajectories
+ *   Each preset runs as a parallel "subagent" within each scenario
+ *   Measures: tool calls, phases, knowledge reuse, token overhead, gaps found
+ *
+ * Run: npx vitest run src/__tests__/presetRealWorldBench.test.ts
+ */
+import { describe, it, expect, afterAll } from "vitest";
+import { verificationTools } from "../tools/verificationTools.js";
+import { reconTools } from "../tools/reconTools.js";
+import { evalTools } from "../tools/evalTools.js";
+import { qualityGateTools } from "../tools/qualityGateTools.js";
+import { flywheelTools } from "../tools/flywheelTools.js";
+import { learningTools } from "../tools/learningTools.js";
+import { agentBootstrapTools } from "../tools/agentBootstrapTools.js";
+import { selfEvalTools } from "../tools/selfEvalTools.js";
+import { parallelAgentTools } from "../tools/parallelAgentTools.js";
+import { uiCaptureTools } from "../tools/uiCaptureTools.js";
+import { visionTools } from "../tools/visionTools.js";
+import { webTools } from "../tools/webTools.js";
+import { githubTools } from "../tools/githubTools.js";
+import { documentationTools } from "../tools/documentationTools.js";
+import { localFileTools, gaiaMediaSolvers } from "../tools/localFileTools.js";
+import { llmTools } from "../tools/llmTools.js";
+import { securityTools } from "../tools/securityTools.js";
+import { platformTools } from "../tools/platformTools.js";
+import { researchWritingTools } from "../tools/researchWritingTools.js";
+import { flickerDetectionTools } from "../tools/flickerDetectionTools.js";
+import { figmaFlowTools } from "../tools/figmaFlowTools.js";
+import { boilerplateTools } from "../tools/boilerplateTools.js";
+import { cCompilerBenchmarkTools } from "../tools/cCompilerBenchmarkTools.js";
+import { sessionMemoryTools } from "../tools/sessionMemoryTools.js";
+import { createMetaTools } from "../tools/metaTools.js";
+import { createProgressiveDiscoveryTools } from "../tools/progressiveDiscoveryTools.js";
+// ═══════════════════════════════════════════════════════════════════════════
+// PRESET & TOOLSET DEFINITIONS (mirrors index.ts exactly)
+// ═══════════════════════════════════════════════════════════════════════════
+const TOOLSET_MAP = {
+    verification: verificationTools,
+    eval: evalTools,
+    quality_gate: qualityGateTools,
+    learning: learningTools,
+    flywheel: flywheelTools,
+    recon: reconTools,
+    ui_capture: uiCaptureTools,
+    vision: visionTools,
+    local_file: localFileTools,
+    web: webTools,
+    github: githubTools,
+    docs: documentationTools,
+    bootstrap: agentBootstrapTools,
+    self_eval: selfEvalTools,
+    parallel: parallelAgentTools,
+    llm: llmTools,
+    security: securityTools,
+    platform: platformTools,
+    research_writing: researchWritingTools,
+    flicker_detection: flickerDetectionTools,
+    figma_flow: figmaFlowTools,
+    boilerplate: boilerplateTools,
+    benchmark: cCompilerBenchmarkTools,
+    session_memory: sessionMemoryTools,
+    gaia_solvers: gaiaMediaSolvers,
+};
+const PRESETS = {
+    meta: [],
+    lite: ["verification", "eval", "quality_gate", "learning", "flywheel", "recon", "security", "boilerplate"],
+    core: [
+        "verification", "eval", "quality_gate", "learning", "flywheel", "recon",
+        "bootstrap", "self_eval", "llm", "security", "platform", "research_writing",
+        "flicker_detection", "figma_flow", "boilerplate", "benchmark", "session_memory",
+    ],
+    full: Object.keys(TOOLSET_MAP),
+};
+function buildToolset(preset) {
+    const keys = PRESETS[preset];
+    const domain = keys.flatMap((k) => TOOLSET_MAP[k] ?? []);
+    const metaTools = createMetaTools(domain);
+    const allForDiscovery = [...domain, ...metaTools];
+    const discoveryTools = createProgressiveDiscoveryTools(allForDiscovery);
+    return [...domain, ...metaTools, ...discoveryTools];
+}
+// ═══════════════════════════════════════════════════════════════════════════
+// HELPER: Execute a scenario against a preset
+// ═══════════════════════════════════════════════════════════════════════════
+async function executeScenario(scenario, preset) {
+    const tools = buildToolset(preset);
+    const toolMap = new Map(tools.map((t) => [t.name, t]));
+    const startTime = Date.now();
+    const phases = [];
+    let totalCalls = 0;
+    let totalMissing = 0;
+    let totalErrors = 0;
+    let knowledgeRecorded = false;
+    let knowledgeReused = false;
+    let discoveryUsed = false;
+    for (const phaseSpec of scenario.phases) {
+        const phaseStart = Date.now();
+        const called = [];
+        const missing = [];
+        const failed = [];
+        for (const attempt of phaseSpec.tools) {
+            const tool = toolMap.get(attempt.name);
+            if (!tool) {
+                missing.push(attempt.name);
+                totalMissing++;
+                continue;
+            }
+            try {
+                await tool.handler(attempt.args);
+                called.push(attempt.name);
+                totalCalls++;
+                if (attempt.name === "record_learning")
+                    knowledgeRecorded = true;
+                if (attempt.name === "search_all_knowledge")
+                    knowledgeReused = true;
+                if (attempt.name === "discover_tools" || attempt.name === "get_workflow_chain")
+                    discoveryUsed = true;
+            }
+            catch {
+                failed.push(attempt.name);
+                totalErrors++;
+            }
+        }
+        phases.push({
+            phase: phaseSpec.name,
+            toolsCalled: called,
+            toolsMissing: missing,
+            toolsFailed: failed,
+            success: missing.length === 0 && (called.length > 0 || failed.length > 0),
+            durationMs: Date.now() - phaseStart,
+        });
+    }
+    return {
+        preset,
+        scenarioId: scenario.id,
+        scenarioName: scenario.name,
+        inspiredBy: scenario.inspiredBy,
+        gapFilled: scenario.gapFilled,
+        toolCount: tools.length,
+        estimatedSchemaTokens: tools.length * 200,
+        phases,
+        phasesCompleted: phases.filter((p) => p.success).length,
+        phasesSkipped: phases.filter((p) => !p.success).length,
+        totalToolCalls: totalCalls,
+        totalToolMissing: totalMissing,
+        totalToolErrors: totalErrors,
+        knowledgeRecorded,
+        knowledgeReused,
+        discoveryUsed,
+        durationMs: Date.now() - startTime,
+    };
+}
+// ═══════════════════════════════════════════════════════════════════════════
+// 8 REAL-WORLD SCENARIOS — Inspired by open-source ecosystem repos
+// ═══════════════════════════════════════════════════════════════════════════
+const SCENARIOS = [
+    // ─── Scenario 1: Cold Start Self-Setup ───────────────────────────────
+    // Inspired by: superpowers (mandatory skill-check), oh-my-claudecode (zero-learning-curve)
+    // Gap filled: Bootstrap cold-start (GAP 6)
+    {
+        id: "cold-start-self-setup",
+        name: "Cold Start: Agent Onboarding via Discovery",
+        inspiredBy: "obra/superpowers + Yeachan-Heo/oh-my-claudecode",
+        gapFilled: "GAP 6: Bootstrap cold-start",
+        prompt: "You are a new agent. Discover available tools, find the right methodology, and set up your working environment.",
+        category: "cold_start",
+        phases: [
+            {
+                name: "discovery",
+                tools: [
+                    { name: "discover_tools", args: { query: "getting started setup bootstrap" }, domain: "progressive_discovery" },
+                    { name: "get_workflow_chain", args: { workflow: "self_setup" }, domain: "progressive_discovery" },
+                    { name: "findTools", args: { query: "verify" }, domain: "meta" },
+                ],
+            },
+            {
+                name: "methodology",
+                tools: [
+                    { name: "getMethodology", args: { topic: "verification" }, domain: "meta" },
+                    { name: "getMethodology", args: { topic: "agent_contract" }, domain: "meta" },
+                ],
+            },
+            {
+                name: "bootstrap",
+                optionalForMeta: true,
+                optionalForLite: true,
+                tools: [
+                    { name: "discover_infrastructure", args: { targetDir: "." }, domain: "bootstrap" },
+                    { name: "triple_verify", args: { component: "database", claims: ["SQLite exists"] }, domain: "bootstrap" },
+                    { name: "generate_self_instructions", args: { targetDir: ".", existingCapabilities: ["verification", "eval"] }, domain: "bootstrap" },
+                ],
+            },
+            {
+                name: "knowledge_seed",
+                optionalForMeta: true,
+                tools: [
+                    { name: "search_all_knowledge", args: { query: "setup patterns" }, domain: "learning" },
+                    { name: "record_learning", args: { key: "bench-cold-start", content: "Agent bootstrap completed via discovery-first pattern", category: "pattern" }, domain: "learning" },
+                ],
+            },
+        ],
+    },
+    // ─── Scenario 2: 4-Phase Bug Fix Pipeline ───────────────────────────
+    // Inspired by: superpowers (4-phase root cause analysis)
+    // Gap filled: Cross-domain workflows (GAP 1)
+    {
+        id: "four-phase-bug-fix",
+        name: "4-Phase Bug Fix: Root Cause → Verify → Eval → Learn",
+        inspiredBy: "obra/superpowers (systematic debugging)",
+        gapFilled: "GAP 1: Cross-domain workflows",
+        prompt: "Fix a production bug: the daily cron job silently fails when the API returns 429 rate-limit responses.",
+        category: "bug_fix",
+        phases: [
+            {
+                name: "investigate",
+                tools: [
+                    { name: "search_all_knowledge", args: { query: "rate limit 429 cron failure" }, domain: "learning" },
+                    { name: "run_recon", args: { target: "cron-rate-limit-bug", scope: "code", maxFindings: 5 }, domain: "recon" },
+                    { name: "log_recon_finding", args: { sessionId: "bench-recon", category: "bug", summary: "429 not retried: Cron ignores HTTP 429 responses from upstream API" }, domain: "recon" },
+                ],
+            },
+            {
+                name: "verify",
+                tools: [
+                    { name: "start_verification_cycle", args: { title: "Fix cron 429 handling" }, domain: "verification" },
+                    { name: "log_phase_findings", args: { cycleId: "bench-cycle", phase: 1, summary: "Root cause: missing retry logic for 429", passed: true }, domain: "verification" },
+                    { name: "log_gap", args: { cycleId: "bench-cycle", description: "No exponential backoff on 429", severity: "critical", phase: 2 }, domain: "verification" },
+                    { name: "resolve_gap", args: { gapId: "bench-gap", resolution: "Added exponential backoff with jitter" }, domain: "verification" },
+                    { name: "log_test_result", args: { cycleId: "bench-cycle", label: "unit-retry-429", layer: "unit", passed: true }, domain: "verification" },
+                ],
+            },
+            {
+                name: "eval",
+                tools: [
+                    { name: "start_eval_run", args: { name: "cron-429-fix-eval" }, domain: "eval" },
+                    { name: "record_eval_result", args: { runId: "bench-eval", case: "retry-backoff", passed: true, notes: "429 now triggers 3 retries with exponential backoff" }, domain: "eval" },
+                    { name: "complete_eval_run", args: { runId: "bench-eval" }, domain: "eval" },
+                ],
+            },
+            {
+                name: "quality_gate",
+                tools: [
+                    { name: "run_quality_gate", args: { targetId: "cron-fix", rules: [{ name: "test-coverage", threshold: 80 }] }, domain: "quality_gate" },
+                    { name: "run_closed_loop", args: { targetId: "cron-fix", command: "npm test", expectedPattern: "PASS" }, domain: "quality_gate" },
+                ],
+            },
+            {
+                name: "learn",
+                tools: [
+                    { name: "record_learning", args: { key: "bench-429-retry", content: "Always add exponential backoff for HTTP 429 in cron jobs", category: "gotcha", tags: ["http", "retry", "cron"] }, domain: "learning" },
+                ],
+            },
+        ],
+    },
+    // ─── Scenario 3: Feature Dev (Context → Plan → Implement → Ship) ──────
+    // Inspired by: wshobson/agents Conductor pattern
+    // Gap filled: Cross-domain end-to-end (GAP 1)
+    {
+        id: "conductor-feature-dev",
+        name: "Conductor-Style Feature: Context → Spec → Implement → Ship",
+        inspiredBy: "wshobson/agents (Conductor plugin)",
+        gapFilled: "GAP 1: Cross-domain workflows",
+        prompt: "Implement a new dark mode toggle feature following the Conductor workflow: gather context, spec, plan, implement, verify, ship.",
+        category: "feature_dev",
+        phases: [
+            {
+                name: "context_gathering",
+                tools: [
+                    { name: "search_all_knowledge", args: { query: "dark mode theme UI toggle" }, domain: "learning" },
+                    { name: "run_recon", args: { target: "dark-mode-feature", scope: "architecture", maxFindings: 5 }, domain: "recon" },
+                    { name: "get_recon_summary", args: { sessionId: "bench-recon" }, domain: "recon" },
+                ],
+            },
+            {
+                name: "specification",
+                tools: [
+                    { name: "start_verification_cycle", args: { title: "Dark Mode Toggle Implementation" }, domain: "verification" },
+                    { name: "log_phase_findings", args: { cycleId: "bench-cycle", phase: 1, summary: "Architecture review: component tree supports theme prop injection", passed: true }, domain: "verification" },
+                ],
+            },
+            {
+                name: "implement_and_test",
+                tools: [
+                    { name: "log_phase_findings", args: { cycleId: "bench-cycle", phase: 2, summary: "Implementation: ThemeProvider + useTheme hook + toggle component", passed: true }, domain: "verification" },
+                    { name: "log_test_result", args: { cycleId: "bench-cycle", label: "dark-mode-unit", layer: "unit", passed: true }, domain: "verification" },
+                    { name: "log_test_result", args: { cycleId: "bench-cycle", label: "dark-mode-integration", layer: "integration", passed: true }, domain: "verification" },
+                    { name: "run_closed_loop", args: { targetId: "dark-mode", command: "npm test -- --grep theme", expectedPattern: "PASS" }, domain: "quality_gate" },
+                ],
+            },
+            {
+                name: "flywheel",
+                optionalForLite: true,
+                optionalForMeta: true,
+                tools: [
+                    { name: "get_flywheel_status", args: {}, domain: "flywheel" },
+                    { name: "trigger_investigation", args: { evalRunId: "bench-eval", regressionDescription: "dark-mode-accessibility regression detected" }, domain: "flywheel" },
+                ],
+            },
+            {
+                name: "ship",
+                tools: [
+                    { name: "run_quality_gate", args: { targetId: "dark-mode", rules: [{ name: "all-tests-pass", threshold: 100 }] }, domain: "quality_gate" },
+                    { name: "record_learning", args: { key: "bench-dark-mode-pattern", content: "ThemeProvider + useTheme hook pattern works well for dark mode", category: "pattern", tags: ["ui", "theme", "dark-mode"] }, domain: "learning" },
+                ],
+            },
+        ],
+    },
+    // ─── Scenario 4: Multi-Agent Coordination ────────────────────────────
+    // Inspired by: ruvnet/claude-flow (queen-led swarm), wshobson/agents (agent-teams)
+    // Gap filled: Parallel agents at scale (GAP 7)
+    {
+        id: "multi-agent-swarm",
+        name: "Multi-Agent Swarm: Coordinator + 3 Parallel Workers",
+        inspiredBy: "ruvnet/claude-flow + wshobson/agents (agent-teams)",
+        gapFilled: "GAP 7: Multi-agent coordination at scale",
+        prompt: "Coordinate 3 parallel agents: backend-api, frontend-ui, and testing-agent working on a full-stack feature.",
+        category: "multi_agent",
+        phases: [
+            {
+                name: "coordinator_setup",
+                optionalForMeta: true,
+                optionalForLite: true,
+                tools: [
+                    { name: "bootstrap_parallel_agents", args: {}, domain: "parallel" },
+                    { name: "assign_agent_role", args: { role: "backend" }, domain: "parallel" },
+                    { name: "assign_agent_role", args: { role: "frontend" }, domain: "parallel" },
+                    { name: "assign_agent_role", args: { role: "testing" }, domain: "parallel" },
+                ],
+            },
+            {
+                name: "task_assignment",
+                optionalForMeta: true,
+                optionalForLite: true,
+                tools: [
+                    { name: "claim_agent_task", args: { taskKey: "backend-api-endpoints" }, domain: "parallel" },
+                    { name: "claim_agent_task", args: { taskKey: "frontend-dark-mode" }, domain: "parallel" },
+                    { name: "claim_agent_task", args: { taskKey: "e2e-tests" }, domain: "parallel" },
+                ],
+            },
+            {
+                name: "context_budget",
+                optionalForMeta: true,
+                optionalForLite: true,
+                tools: [
+                    { name: "log_context_budget", args: { eventType: "checkpoint", tokensUsed: 15000 }, domain: "parallel" },
+                    { name: "log_context_budget", args: { eventType: "checkpoint", tokensUsed: 12000 }, domain: "parallel" },
+                    { name: "log_context_budget", args: { eventType: "checkpoint", tokensUsed: 8000 }, domain: "parallel" },
+                ],
+            },
+            {
+                name: "oracle_comparison",
+                optionalForMeta: true,
+                optionalForLite: true,
+                tools: [
+                    { name: "run_oracle_comparison", args: { testLabel: "fullstack-integration", actualOutput: "API endpoints created + UI renders", expectedOutput: "API endpoints created + UI renders", oracleSource: "manual_review" }, domain: "parallel" },
+                ],
+            },
+            {
+                name: "knowledge_banking",
+                optionalForMeta: true,
+                tools: [
+                    { name: "record_learning", args: { key: "bench-parallel-fullstack", content: "3-agent fullstack pattern: backend+frontend+testing agents with coordinator reduces merge conflicts", category: "pattern", tags: ["parallel", "fullstack", "coordination"] }, domain: "learning" },
+                ],
+            },
+        ],
+    },
+    // ─── Scenario 5: Research & Academic Writing Pipeline ─────────────────
+    // Inspired by: K-Dense-AI/claude-scientific-skills, planning-with-files
+    // Gap filled: Research writing 0% coverage (GAP 5)
+    {
+        id: "research-writing-pipeline",
+        name: "Research Paper: Outline → Draft → Polish → Review",
+        inspiredBy: "K-Dense-AI/claude-scientific-skills + planning-with-files",
+        gapFilled: "GAP 5: Research writing workflows",
+        prompt: "Write a research paper on 'Multi-Agent Coordination in AI-Assisted Development' with proper citations and peer review simulation.",
+        category: "research",
+        phases: [
+            {
+                name: "literature_review",
+                tools: [
+                    { name: "search_all_knowledge", args: { query: "multi-agent coordination research" }, domain: "learning" },
+                    { name: "run_recon", args: { target: "multi-agent-research", scope: "literature", maxFindings: 10 }, domain: "recon" },
+                ],
+            },
+            {
+                name: "outline_and_draft",
+                optionalForMeta: true,
+                optionalForLite: true,
+                tools: [
+                    { name: "check_paper_logic", args: { text: "Multi-agent coordination enables parallel task execution. Our approach uses a coordinator pattern to assign roles and manage context budgets across agents." }, domain: "research_writing" },
+                    { name: "generate_academic_caption", args: { description: "System architecture showing coordinator agent distributing tasks to 3 worker agents", figureType: "diagram" }, domain: "research_writing" },
+                ],
+            },
+            {
+                name: "polish_and_review",
+                optionalForMeta: true,
+                optionalForLite: true,
+                tools: [
+                    { name: "polish_academic_text", args: { text: "Multi-agent systems enable parallel task execution. This improves throughput and reduces context window pressure." }, domain: "research_writing" },
+                    { name: "review_paper_as_reviewer", args: { text: "We propose a coordinator pattern for multi-agent AI development. Our approach distributes tasks to specialized agents.", venue: "ICSE" }, domain: "research_writing" },
+                ],
+            },
+            {
+                name: "record_findings",
+                optionalForMeta: true,
+                tools: [
+                    { name: "record_learning", args: { key: "bench-research-pattern", content: "4-phase research pipeline: literature review → outline → draft → polish works well for academic papers", category: "pattern", tags: ["research", "writing", "academic"] }, domain: "learning" },
+                ],
+            },
+        ],
+    },
+    // ─── Scenario 6: Cross-Domain Investigation ──────────────────────────
+    // Inspired by: claude-mem (multi-source observations), oh-my-claudecode (5 modes)
+    // Gap filled: Cross-domain silos (GAP 1)
+    {
+        id: "cross-domain-investigation",
+        name: "Cross-Domain: Recon → Local Files → Vision → Quality Gate",
+        inspiredBy: "thedotmack/claude-mem + Yeachan-Heo/oh-my-claudecode",
+        gapFilled: "GAP 1: Cross-domain workflows (break silos)",
+        prompt: "Investigate a UI rendering issue: parse local config files, analyze screenshot, search codebase, and verify the fix.",
+        category: "cross_domain",
+        phases: [
+            {
+                name: "recon",
+                tools: [
+                    { name: "run_recon", args: { target: "ui-rendering-bug", scope: "code", maxFindings: 5 }, domain: "recon" },
+                    { name: "search_all_knowledge", args: { query: "UI rendering CSS layout issue" }, domain: "learning" },
+                ],
+            },
+            {
+                name: "local_file_analysis",
+                tools: [
+                    { name: "read_json_file", args: { filePath: "test_config.json" }, domain: "local_file" },
+                    { name: "extract_structured_data", args: { text: "Error: flex container overflow at line 42 in MainLayout.tsx. Component tree depth: 8. Render time: 340ms.", fields: ["error_type", "file", "line", "render_time"] }, domain: "local_file" },
+                ],
+            },
+            {
+                name: "verification",
+                tools: [
+                    { name: "start_verification_cycle", args: { title: "UI rendering fix" }, domain: "verification" },
+                    { name: "log_gap", args: { cycleId: "bench-cycle", description: "CSS flex overflow not handled", severity: "high", phase: 2 }, domain: "verification" },
+                    { name: "resolve_gap", args: { gapId: "bench-gap", resolution: "Added overflow-x: hidden to container" }, domain: "verification" },
+                ],
+            },
+            {
+                name: "quality_gate",
+                tools: [
+                    { name: "run_quality_gate", args: { targetId: "ui-fix", rules: [{ name: "visual-regression", threshold: 95 }] }, domain: "quality_gate" },
+                ],
+            },
+            {
+                name: "learn",
+                tools: [
+                    { name: "record_learning", args: { key: "bench-flex-overflow", content: "Flex container overflow: always set overflow-x on deeply nested component trees", category: "gotcha", tags: ["css", "flex", "overflow", "ui"] }, domain: "learning" },
+                ],
+            },
+        ],
+    },
+    // ─── Scenario 7: Error Recovery & Resilience ──────────────────────────
+    // Inspired by: claude-flow (Byzantine fault tolerance), oh-my-claudecode (compaction-resilient)
+    // Gap filled: Error recovery (GAP 2)
+    {
+        id: "error-recovery-resilience",
+        name: "Error Recovery: Failure at Each Phase → Graceful Degradation",
+        inspiredBy: "ruvnet/claude-flow (fault tolerance) + oh-my-claudecode (resilience)",
+        gapFilled: "GAP 2: Error recovery & failure paths",
+        prompt: "Handle a scenario where tools fail mid-workflow: recon times out, eval has stale data, and gate rules are violated.",
+        category: "error_recovery",
+        phases: [
+            {
+                name: "safe_recon",
+                tools: [
+                    { name: "run_recon", args: { target: "resilience-test", scope: "code", maxFindings: 3 }, domain: "recon" },
+                ],
+            },
+            {
+                name: "verification_with_errors",
+                tools: [
+                    { name: "start_verification_cycle", args: { title: "Resilience test cycle" }, domain: "verification" },
+                    { name: "log_phase_findings", args: { cycleId: "bench-cycle", phase: 1, summary: "Phase 1 passed under degraded conditions", passed: true }, domain: "verification" },
+                    // Intentionally log a gap that stays open (simulates partial recovery)
+                    { name: "log_gap", args: { cycleId: "bench-cycle", description: "Stale cache detected but non-critical", severity: "medium", phase: 2 }, domain: "verification" },
+                ],
+            },
+            {
+                name: "eval_despite_gaps",
+                tools: [
+                    { name: "start_eval_run", args: { name: "resilience-eval" }, domain: "eval" },
+                    { name: "record_eval_result", args: { runId: "bench-eval", case: "graceful-degradation", passed: true, notes: "System operates correctly despite stale cache" }, domain: "eval" },
+                    { name: "complete_eval_run", args: { runId: "bench-eval" }, domain: "eval" },
+                ],
+            },
+            {
+                name: "gate_with_violations",
+                tools: [
+                    // Gate with a very high threshold that will "fail" (simulates gate violation)
+                    { name: "run_quality_gate", args: { targetId: "resilience-check", rules: [{ name: "zero-open-gaps", threshold: 100 }] }, domain: "quality_gate" },
+                ],
+            },
+            {
+                name: "learn_from_failure",
+                tools: [
+                    { name: "record_learning", args: { key: "bench-resilience-pattern", content: "Graceful degradation: continue eval even with open medium-severity gaps. Only block on critical.", category: "pattern", tags: ["resilience", "error-recovery", "degradation"] }, domain: "learning" },
+                ],
+            },
+        ],
+    },
+    // ─── Scenario 8: Knowledge Lifecycle ──────────────────────────────────
+    // Inspired by: thedotmack/claude-mem (session compression + token economics)
+    // Gap filled: Knowledge lifecycle (GAP 4)
+    {
+        id: "knowledge-lifecycle",
+        name: "Knowledge Lifecycle: Record → Search → Synthesize → Reuse",
+        inspiredBy: "thedotmack/claude-mem (context compression + observations)",
+        gapFilled: "GAP 4: Knowledge lifecycle (record → reuse)",
+        prompt: "Exercise the full knowledge lifecycle: record learnings from past work, search for relevant knowledge, synthesize findings, and verify reuse improves outcomes.",
+        category: "knowledge_lifecycle",
+        phases: [
+            {
+                name: "seed_knowledge",
+                tools: [
+                    { name: "record_learning", args: { key: "bench-kl-pattern-1", content: "Always check for null pointers before accessing nested properties", category: "gotcha", tags: ["null", "safety", "typescript"] }, domain: "learning" },
+                    { name: "record_learning", args: { key: "bench-kl-pattern-2", content: "Use zod schemas for API input validation at system boundaries", category: "pattern", tags: ["validation", "zod", "api"] }, domain: "learning" },
+                    { name: "record_learning", args: { key: "bench-kl-edge-1", content: "SQLite FTS5 requires rebuilding index after schema changes", category: "edge_case", tags: ["sqlite", "fts5", "migration"] }, domain: "learning" },
+                ],
+            },
+            {
+                name: "search_and_retrieve",
+                tools: [
+                    { name: "search_all_knowledge", args: { query: "typescript null safety" }, domain: "learning" },
+                    { name: "search_all_knowledge", args: { query: "API validation" }, domain: "learning" },
+                    { name: "search_all_knowledge", args: { query: "database migration" }, domain: "learning" },
+                ],
+            },
+            {
+                name: "apply_knowledge",
+                tools: [
+                    { name: "start_verification_cycle", args: { title: "Apply prior learnings to new task" }, domain: "verification" },
+                    { name: "log_phase_findings", args: { cycleId: "bench-cycle", phase: 1, summary: "Prior knowledge applied: null checks + zod validation added", passed: true }, domain: "verification" },
+                ],
+            },
+            {
+                name: "synthesize",
+                optionalForMeta: true,
+                optionalForLite: true,
+                tools: [
+                    { name: "synthesize_recon_to_learnings", args: {}, domain: "self_eval" },
+                    { name: "get_improvement_recommendations", args: {}, domain: "self_eval" },
+                ],
+            },
+            {
+                name: "verify_reuse",
+                tools: [
+                    { name: "record_learning", args: { key: "bench-kl-meta-learning", content: "Knowledge reuse reduced verification time by ~30%: prior learnings prevented 3 known gotchas", category: "pattern", tags: ["knowledge", "reuse", "efficiency"] }, domain: "learning" },
+                ],
+            },
+        ],
+    },
+];
+// ═══════════════════════════════════════════════════════════════════════════
+// PARALLEL EXECUTION — All 4 presets run concurrently per scenario
+// ═══════════════════════════════════════════════════════════════════════════
+const allTrajectories = [];
+const PRESET_NAMES = ["meta", "lite", "core", "full"];
+describe("Preset Real-World Benchmark", () => {
+    // ─── Per-scenario tests: 4 presets run in parallel ─────────────────
+    for (const scenario of SCENARIOS) {
+        describe(`Scenario: ${scenario.name}`, () => {
+            const scenarioTrajectories = [];
+            it(`runs all 4 presets in parallel for ${scenario.id}`, async () => {
+                // Execute all 4 presets concurrently (simulates parallel subagents)
+                const results = await Promise.all(PRESET_NAMES.map((preset) => executeScenario(scenario, preset)));
+                scenarioTrajectories.push(...results);
+                allTrajectories.push(...results);
+                // Basic sanity: every preset produced a trajectory
+                expect(results.length).toBe(4);
+                for (const r of results) {
+                    expect(r.scenarioId).toBe(scenario.id);
+                    expect(r.phases.length).toBe(scenario.phases.length);
+                }
+            });
+            it(`full preset has no missing tools for ${scenario.id}`, () => {
+                const full = scenarioTrajectories.find((t) => t.preset === "full");
+                if (!full)
+                    return; // depends on previous test
+                for (const phase of full.phases) {
+                    expect(phase.toolsMissing).toEqual([]);
+                }
+                // All phases complete (tools found, even if some errored on stale IDs)
+                expect(full.phasesCompleted).toBe(scenario.phases.length);
+            });
+            it(`meta preset discovers tools but hits domain limits for ${scenario.id}`, () => {
+                const meta = scenarioTrajectories.find((t) => t.preset === "meta");
+                if (!meta)
+                    return;
+                // Meta should always have discovery tools
+                expect(meta.toolCount).toBe(5); // 2 meta + 3 discovery
+                // Meta should succeed at discovery/methodology phases
+                const discoveryPhase = meta.phases.find((p) => p.phase === "discovery" || p.phase === "methodology");
+                if (discoveryPhase) {
+                    expect(discoveryPhase.toolsCalled.length).toBeGreaterThan(0);
+                }
+            });
+            it(`lite has fewer tools but covers core verification for ${scenario.id}`, () => {
+                const lite = scenarioTrajectories.find((t) => t.preset === "lite");
+                const full = scenarioTrajectories.find((t) => t.preset === "full");
+                if (!lite || !full)
+                    return;
+                expect(lite.toolCount).toBeLessThan(full.toolCount);
+                // Lite should always have verification, eval, learning, recon
+                const verifyPhase = lite.phases.find((p) => p.phase === "verify" || p.phase === "verification");
+                if (verifyPhase) {
+                    expect(verifyPhase.toolsMissing.length).toBe(0);
+                }
+            });
+            it(`core covers most phases, loses only full-exclusive domains for ${scenario.id}`, () => {
+                const core = scenarioTrajectories.find((t) => t.preset === "core");
+                if (!core)
+                    return;
+                // Core should complete most phases (may miss ui_capture, vision, web, github, parallel, docs)
+                const coreCompleted = core.phasesCompleted;
+                expect(coreCompleted).toBeGreaterThanOrEqual(scenario.phases.filter((p) => !p.optionalForLite).length - 1);
+            });
+        });
+    }
+    // ═══════════════════════════════════════════════════════════════════════
+    // CROSS-SCENARIO ANALYSIS
+    // ═══════════════════════════════════════════════════════════════════════
+    describe("Cross-Scenario Analysis", () => {
+        it("generated 32 trajectories (8 scenarios × 4 presets)", () => {
+            expect(allTrajectories.length).toBe(32);
+        });
+        it("full preset has most successful tool executions (calls + errors) across all scenarios", () => {
+            const byPreset = (p) => allTrajectories.filter((t) => t.preset === p).reduce((sum, t) => sum + t.totalToolCalls + t.totalToolErrors, 0);
+            expect(byPreset("full")).toBeGreaterThanOrEqual(byPreset("core"));
+            expect(byPreset("core")).toBeGreaterThanOrEqual(byPreset("lite"));
+            expect(byPreset("lite")).toBeGreaterThan(byPreset("meta"));
+        });
+        it("presets are strictly ordered by tool count: meta < lite < core < full", () => {
+            const counts = PRESET_NAMES.map((p) => {
+                const t = allTrajectories.find((tr) => tr.preset === p);
+                return t?.toolCount ?? 0;
+            });
+            expect(counts[0]).toBeLessThan(counts[1]); // meta < lite
+            expect(counts[1]).toBeLessThan(counts[2]); // lite < core
+            expect(counts[2]).toBeLessThan(counts[3]); // core < full
+        });
+        it("meta preset token overhead is <5% of full preset", () => {
+            const metaTokens = allTrajectories.find((t) => t.preset === "meta")?.estimatedSchemaTokens ?? 0;
+            const fullTokens = allTrajectories.find((t) => t.preset === "full")?.estimatedSchemaTokens ?? 0;
+            expect(metaTokens / fullTokens).toBeLessThan(0.05);
+        });
+        it("knowledge is recorded in at least 6/8 scenarios for full preset", () => {
+            const fullTrajectories = allTrajectories.filter((t) => t.preset === "full");
+            const withKnowledge = fullTrajectories.filter((t) => t.knowledgeRecorded);
+            expect(withKnowledge.length).toBeGreaterThanOrEqual(6);
+        });
+        it("knowledge is reused (searched) in at least 5/8 scenarios for full preset", () => {
+            const fullTrajectories = allTrajectories.filter((t) => t.preset === "full");
+            const withReuse = fullTrajectories.filter((t) => t.knowledgeReused);
+            expect(withReuse.length).toBeGreaterThanOrEqual(5);
+        });
+        it("discovery tools are used in cold-start scenario for all presets", () => {
+            const coldStartTrajectories = allTrajectories.filter((t) => t.scenarioId === "cold-start-self-setup");
+            for (const t of coldStartTrajectories) {
+                expect(t.discoveryUsed).toBe(true);
+            }
+        });
+        it("lite catches verification gaps in bug-fix and feature-dev scenarios", () => {
+            const liteTrajectories = allTrajectories.filter((t) => t.preset === "lite" && (t.scenarioId === "four-phase-bug-fix" || t.scenarioId === "conductor-feature-dev"));
+            for (const t of liteTrajectories) {
+                const verifyPhase = t.phases.find((p) => p.phase === "verify" || p.phase === "specification" || p.phase === "implement_and_test");
+                if (verifyPhase) {
+                    expect(verifyPhase.toolsMissing.length).toBe(0);
+                }
+            }
+        });
+        it("multi-agent scenario requires full or core preset (lite/meta skip parallel)", () => {
+            const metaSwarm = allTrajectories.find((t) => t.preset === "meta" && t.scenarioId === "multi-agent-swarm");
+            const liteSwarm = allTrajectories.find((t) => t.preset === "lite" && t.scenarioId === "multi-agent-swarm");
+            const fullSwarm = allTrajectories.find((t) => t.preset === "full" && t.scenarioId === "multi-agent-swarm");
+            expect(metaSwarm.totalToolMissing).toBeGreaterThan(0);
+            expect(liteSwarm.totalToolMissing).toBeGreaterThan(0);
+            // Full should have all parallel tools
+            const coordPhase = fullSwarm.phases.find((p) => p.phase === "coordinator_setup");
+            expect(coordPhase?.toolsMissing.length).toBe(0);
+        });
+        it("research-writing scenario needs core+ (lite/meta missing research_writing tools)", () => {
+            const liteResearch = allTrajectories.find((t) => t.preset === "lite" && t.scenarioId === "research-writing-pipeline");
+            const coreResearch = allTrajectories.find((t) => t.preset === "core" && t.scenarioId === "research-writing-pipeline");
+            expect(liteResearch.totalToolMissing).toBeGreaterThan(0);
+            // Core should have research_writing
+            const draftPhase = coreResearch.phases.find((p) => p.phase === "outline_and_draft");
+            expect(draftPhase?.toolsMissing.length).toBe(0);
+        });
+        it("error-recovery scenario completes for all presets with verification", () => {
+            for (const preset of ["lite", "core", "full"]) {
+                const t = allTrajectories.find((tr) => tr.preset === preset && tr.scenarioId === "error-recovery-resilience");
+                expect(t.phasesCompleted).toBeGreaterThanOrEqual(3);
+            }
+        });
+    });
+    // ═══════════════════════════════════════════════════════════════════════
+    // GAP COVERAGE REPORT
+    // ═══════════════════════════════════════════════════════════════════════
+    describe("Gap Coverage Verification", () => {
+        it("GAP 1 (cross-domain) is exercised by 3 scenarios", () => {
+            const crossDomain = allTrajectories.filter((t) => t.preset === "full" &&
+                ["four-phase-bug-fix", "conductor-feature-dev", "cross-domain-investigation"].includes(t.scenarioId));
+            expect(crossDomain.length).toBe(3);
+            for (const t of crossDomain) {
+                // Each should call tools from 3+ domains
+                const domainsUsed = new Set(t.phases.flatMap((p) => p.toolsCalled));
+                expect(domainsUsed.size).toBeGreaterThanOrEqual(3);
+            }
+        });
+        it("GAP 2 (error recovery) is exercised", () => {
+            const recovery = allTrajectories.find((t) => t.preset === "full" && t.scenarioId === "error-recovery-resilience");
+            expect(recovery).toBeDefined();
+            expect(recovery.totalToolCalls).toBeGreaterThan(0);
+        });
+        it("GAP 4 (knowledge lifecycle) exercises record→search→synthesize", () => {
+            const kl = allTrajectories.find((t) => t.preset === "full" && t.scenarioId === "knowledge-lifecycle");
+            expect(kl).toBeDefined();
+            expect(kl.knowledgeRecorded).toBe(true);
+            expect(kl.knowledgeReused).toBe(true);
+            // Synthesize phase should complete for full
+            const synthPhase = kl.phases.find((p) => p.phase === "synthesize");
+            expect(synthPhase?.toolsMissing.length).toBe(0);
+        });
+        it("GAP 5 (research writing) exercises outline→draft→polish→review", () => {
+            const rw = allTrajectories.find((t) => t.preset === "full" && t.scenarioId === "research-writing-pipeline");
+            expect(rw).toBeDefined();
+            const phases = rw.phases.map((p) => p.phase);
+            expect(phases).toContain("outline_and_draft");
+            expect(phases).toContain("polish_and_review");
+        });
+        it("GAP 6 (bootstrap cold-start) exercises discovery→bootstrap→seed", () => {
+            const cs = allTrajectories.find((t) => t.preset === "full" && t.scenarioId === "cold-start-self-setup");
+            expect(cs).toBeDefined();
+            expect(cs.discoveryUsed).toBe(true);
+            const bootstrapPhase = cs.phases.find((p) => p.phase === "bootstrap");
+            expect(bootstrapPhase?.toolsMissing.length).toBe(0);
+        });
+        it("GAP 7 (multi-agent) exercises coordinator→assign→budget→oracle", () => {
+            const ma = allTrajectories.find((t) => t.preset === "full" && t.scenarioId === "multi-agent-swarm");
+            expect(ma).toBeDefined();
+            // Full preset has all parallel tools, so no missing tools in any phase
+            expect(ma.totalToolMissing).toBe(0);
+            expect(ma.phasesCompleted).toBe(5);
+        });
+    });
+    // ═══════════════════════════════════════════════════════════════════════
+    // FINAL REPORT (printed to console after all tests)
+    // ═══════════════════════════════════════════════════════════════════════
+    afterAll(() => {
+        if (allTrajectories.length === 0)
+            return;
+        console.log("\n╔══════════════════════════════════════════════════════════════════════════╗");
+        console.log("║          PRESET REAL-WORLD BENCHMARK — IMPACT REPORT                   ║");
+        console.log("║  8 scenarios × 4 presets = 32 trajectories                             ║");
+        console.log("║  Inspired by: superpowers, agents, claude-flow, oh-my-claudecode,      ║");
+        console.log("║               claude-mem, planning-with-files, scientific-skills,       ║");
+        console.log("║               claude-code-guide                                        ║");
+        console.log("╠══════════════════════════════════════════════════════════════════════════╣");
+        // Per-preset summary
+        for (const preset of PRESET_NAMES) {
+            const trajectories = allTrajectories.filter((t) => t.preset === preset);
+            const totalCalls = trajectories.reduce((s, t) => s + t.totalToolCalls, 0);
+            const totalMissing = trajectories.reduce((s, t) => s + t.totalToolMissing, 0);
+            const totalErrors = trajectories.reduce((s, t) => s + t.totalToolErrors, 0);
+            const completedPhases = trajectories.reduce((s, t) => s + t.phasesCompleted, 0);
+            const totalPhases = trajectories.reduce((s, t) => s + t.phases.length, 0);
+            const toolCount = trajectories[0]?.toolCount ?? 0;
+            const tokens = trajectories[0]?.estimatedSchemaTokens ?? 0;
+            const knowledgeCount = trajectories.filter((t) => t.knowledgeRecorded).length;
+            const duration = trajectories.reduce((s, t) => s + t.durationMs, 0);
+            console.log(`║                                                                          ║`);
+            console.log(`║  --preset ${preset.padEnd(6)} (${String(toolCount).padStart(3)} tools, ~${String(tokens).padStart(5)} schema tokens)               ║`);
+            console.log(`║    Phases: ${completedPhases}/${totalPhases} completed                                           ║`);
+            console.log(`║    Tools:  ${totalCalls} called, ${totalMissing} missing, ${totalErrors} errors                            ║`);
+            console.log(`║    Knowledge: ${knowledgeCount}/8 scenarios recorded learnings                      ║`);
+            console.log(`║    Duration: ${duration}ms total                                             ║`);
+        }
+        // Per-scenario breakdown
+        console.log("║                                                                          ║");
+        console.log("╠══════════════════════════════════════════════════════════════════════════╣");
+        console.log("║  PER-SCENARIO BREAKDOWN                                                 ║");
+        console.log("╠══════════════════════════════════════════════════════════════════════════╣");
+        for (const scenario of SCENARIOS) {
+            console.log(`║                                                                          ║`);
+            console.log(`║  ${scenario.id.padEnd(40)} [${scenario.gapFilled}]  ║`);
+            for (const preset of PRESET_NAMES) {
+                const t = allTrajectories.find((tr) => tr.preset === preset && tr.scenarioId === scenario.id);
+                if (t) {
+                    const status = t.phasesCompleted === t.phases.length ? "PASS" : `${t.phasesCompleted}/${t.phases.length}`;
+                    console.log(`║    ${preset.padEnd(6)}: ${status.padEnd(6)} | calls=${String(t.totalToolCalls).padStart(3)} missing=${String(t.totalToolMissing).padStart(2)} | ${t.durationMs}ms  ║`);
+                }
+            }
+        }
+        // Recommendations
+        console.log("║                                                                          ║");
+        console.log("╠══════════════════════════════════════════════════════════════════════════╣");
+        console.log("║  RECOMMENDATIONS                                                        ║");
+        console.log("╠══════════════════════════════════════════════════════════════════════════╣");
+        console.log("║                                                                          ║");
+        console.log("║  Discovery-first / new agents  → --preset meta  (self-escalate)         ║");
+        console.log("║  Solo dev, bug fixes, features → --preset lite  (fast, core coverage)   ║");
+        console.log("║  Research + multi-agent teams   → --preset core  (full methodology)     ║");
+        console.log("║  Full pipeline + all domains    → --preset full  (zero blind spots)     ║");
+        console.log("║                                                                          ║");
+        console.log("╚══════════════════════════════════════════════════════════════════════════╝");
+    });
+});
+//# sourceMappingURL=presetRealWorldBench.test.js.map