@toyako/hive 1.0.0

package/src/evaluation/GroundTruthDataset.ts

@@ -0,0 +1,84 @@
+/**
+ * Ground Truth Dataset — 50 benchmark tasks with expected outcomes.
+ */
+export interface BenchmarkTask {
+  id: string;
+  category: string;
+  task: string;
+  expected: {
+    files: string[];
+    tests: string[];
+    acceptanceCriteria: string[];
+    minScore: number;
+  };
+}
+
+export const BENCHMARK_DATASET: BenchmarkTask[] = [
+  // CLI Refactor (10)
+  { id: "cli-01", category: "cli-refactor", task: "Extract command handler functions from a monolithic switch statement", expected: { files: ["commands/"], tests: ["handler extraction"], acceptanceCriteria: ["Each command in separate function", "No switch statement > 50 lines"], minScore: 70 } },
+  { id: "cli-02", category: "cli-refactor", task: "Add proper error handling to CLI commands", expected: { files: ["commands/"], tests: ["error handling"], acceptanceCriteria: ["Try-catch on all async commands", "User-friendly error messages"], minScore: 70 } },
+  { id: "cli-03", category: "cli-refactor", task: "Implement consistent help text for all commands", expected: { files: ["commands/"], tests: ["help text"], acceptanceCriteria: ["All commands have --help", "Consistent format"], minScore: 60 } },
+  { id: "cli-04", category: "cli-refactor", task: "Add input validation to CLI arguments", expected: { files: ["commands/"], tests: ["validation"], acceptanceCriteria: ["Required args checked", "Type validation"], minScore: 70 } },
+  { id: "cli-05", category: "cli-refactor", task: "Refactor config loading to use a centralized config module", expected: { files: ["config/"], tests: ["config loading"], acceptanceCriteria: ["Single config source", "No hardcoded values"], minScore: 70 } },
+  { id: "cli-06", category: "cli-refactor", task: "Add verbose/debug output mode to CLI", expected: { files: ["commands/"], tests: ["debug mode"], acceptanceCriteria: ["--verbose flag", "Debug output"], minScore: 60 } },
+  { id: "cli-07", category: "cli-refactor", task: "Implement command aliases for frequently used commands", expected: { files: ["commands/"], tests: ["aliases"], acceptanceCriteria: ["Short aliases work", "Documented"], minScore: 50 } },
+  { id: "cli-08", category: "cli-refactor", task: "Add progress indicators for long-running commands", expected: { files: ["commands/"], tests: ["progress"], acceptanceCriteria: ["Spinner or progress bar", "Non-blocking"], minScore: 60 } },
+  { id: "cli-09", category: "cli-refactor", task: "Refactor output formatting to support JSON and table modes", expected: { files: ["commands/"], tests: ["output format"], acceptanceCriteria: ["--json flag", "Table default"], minScore: 70 } },
+  { id: "cli-10", category: "cli-refactor", task: "Add shell completion support", expected: { files: ["commands/"], tests: ["completion"], acceptanceCriteria: ["bash/zsh completion", "Dynamic suggestions"], minScore: 50 } },
+
+  // Bug Fix (10)
+  { id: "bug-01", category: "bug-fix", task: "Fix race condition in parallel task execution", expected: { files: ["orchestration/DAGExecutor.ts"], tests: ["race condition"], acceptanceCriteria: ["No data corruption", "Consistent results"], minScore: 80 } },
+  { id: "bug-02", category: "bug-fix", task: "Fix memory leak in EventBus event storage", expected: { files: ["runtime/EventBus.ts"], tests: ["memory leak"], acceptanceCriteria: ["Events cleaned up", "No unbounded growth"], minScore: 80 } },
+  { id: "bug-03", category: "bug-fix", task: "Fix timeout handling in ExecutionEngine", expected: { files: ["engine/ExecutionEngine.ts"], tests: ["timeout"], acceptanceCriteria: ["Clean timeout", "No zombie processes"], minScore: 80 } },
+  { id: "bug-04", category: "bug-fix", task: "Fix path normalization on Windows", expected: { files: ["engine/ResultNormalizer.ts"], tests: ["path normalization"], acceptanceCriteria: ["Windows paths work", "No backslash issues"], minScore: 70 } },
+  { id: "bug-05", category: "bug-fix", task: "Fix concurrent file writes in StructuredLogger", expected: { files: ["engine/StructuredLogger.ts"], tests: ["concurrent write"], acceptanceCriteria: ["No file corruption", "All entries saved"], minScore: 80 } },
+  { id: "bug-06", category: "bug-fix", task: "Fix agent router returning null for valid tasks", expected: { files: ["orchestration/AgentRouter.ts"], tests: ["router null"], acceptanceCriteria: ["Valid tasks always routed", "No silent fail"], minScore: 80 } },
+  { id: "bug-07", category: "bug-fix", task: "Fix planner generating duplicate task IDs", expected: { files: ["orchestration/Planner.ts"], tests: ["duplicate IDs"], acceptanceCriteria: ["All IDs unique", "No collisions"], minScore: 70 } },
+  { id: "bug-08", category: "bug-fix", task: "Fix reviewer accepting empty outputs", expected: { files: ["orchestration/ReviewerAgent.ts"], tests: ["empty output"], acceptanceCriteria: ["Empty output rejected", "Score < 60"], minScore: 70 } },
+  { id: "bug-09", category: "bug-fix", task: "Fix DAG executor not handling circular dependencies", expected: { files: ["orchestration/DAGExecutor.ts"], tests: ["circular deps"], acceptanceCriteria: ["Detection", "Error message"], minScore: 80 } },
+  { id: "bug-10", category: "bug-fix", task: "Fix replay engine missing events from crashed agents", expected: { files: ["runtime/ReplayEngine.ts"], tests: ["missing events"], acceptanceCriteria: ["Partial replay works", "Graceful degradation"], minScore: 70 } },
+
+  // Feature (10)
+  { id: "feat-01", category: "feature", task: "Implement agent priority-based routing", expected: { files: ["orchestration/AgentRouter.ts"], tests: ["priority routing"], acceptanceCriteria: ["Higher priority preferred", "Tie-breaking"], minScore: 70 } },
+  { id: "feat-02", category: "feature", task: "Add retry logic with exponential backoff", expected: { files: ["orchestration/DAGExecutor.ts"], tests: ["retry logic"], acceptanceCriteria: ["Exponential backoff", "Max retries"], minScore: 70 } },
+  { id: "feat-03", category: "feature", task: "Implement agent health monitoring", expected: { files: ["orchestration/registry/"], tests: ["health check"], acceptanceCriteria: ["Periodic checks", "Unhealthy agents disabled"], minScore: 60 } },
+  { id: "feat-04", category: "feature", task: "Add task result caching", expected: { files: ["orchestration/"], tests: ["caching"], acceptanceCriteria: ["Cache hit", "TTL support"], minScore: 60 } },
+  { id: "feat-05", category: "feature", task: "Implement conversation context compression", expected: { files: ["memory/ContextCompressor.ts"], tests: ["compression"], acceptanceCriteria: ["50%+ compression", "No info loss"], minScore: 70 } },
+  { id: "feat-06", category: "feature", task: "Add agent capability discovery", expected: { files: ["orchestration/registry/"], tests: ["capability discovery"], acceptanceCriteria: ["Auto-detect capabilities", "Update registry"], minScore: 60 } },
+  { id: "feat-07", category: "feature", task: "Implement task dependency visualization", expected: { files: ["orchestration/"], tests: ["visualization"], acceptanceCriteria: ["DAG output", "Mermaid format"], minScore: 50 } },
+  { id: "feat-08", category: "feature", task: "Add cost tracking per task", expected: { files: ["evaluation/"], tests: ["cost tracking"], acceptanceCriteria: ["Token count", "Cost estimate"], minScore: 60 } },
+  { id: "feat-09", category: "feature", task: "Implement agent load balancing", expected: { files: ["orchestration/AgentRouter.ts"], tests: ["load balancing"], acceptanceCriteria: ["Even distribution", "No overload"], minScore: 60 } },
+  { id: "feat-10", category: "feature", task: "Add execution trace export to JSON", expected: { files: ["runtime/"], tests: ["export"], acceptanceCriteria: ["Valid JSON", "Complete trace"], minScore: 60 } },
+
+  // Architecture (10)
+  { id: "arch-01", category: "architecture", task: "Review and optimize module dependency graph", expected: { files: ["docs/"], tests: ["dependency review"], acceptanceCriteria: ["No circular deps", "Clean layering"], minScore: 70 } },
+  { id: "arch-02", category: "architecture", task: "Design plugin system for custom agents", expected: { files: ["orchestration/"], tests: ["plugin system"], acceptanceCriteria: ["Plugin interface", "Dynamic loading"], minScore: 60 } },
+  { id: "arch-03", category: "architecture", task: "Implement event-driven architecture for agent communication", expected: { files: ["runtime/EventBus.ts"], tests: ["event-driven"], acceptanceCriteria: ["Pub/sub", "Decoupled agents"], minScore: 70 } },
+  { id: "arch-04", category: "architecture", task: "Design scalable task queue with priority support", expected: { files: ["orchestration/"], tests: ["task queue"], acceptanceCriteria: ["Priority ordering", "Persistence"], minScore: 70 } },
+  { id: "arch-05", category: "architecture", task: "Implement configuration validation layer", expected: { files: ["config/"], tests: ["config validation"], acceptanceCriteria: ["Schema validation", "Error messages"], minScore: 60 } },
+  { id: "arch-06", category: "architecture", task: "Design multi-tenant agent isolation", expected: { files: ["orchestration/"], tests: ["isolation"], acceptanceCriteria: ["Tenant separation", "No cross-talk"], minScore: 60 } },
+  { id: "arch-07", category: "architecture", task: "Implement graceful degradation for agent failures", expected: { files: ["orchestration/"], tests: ["degradation"], acceptanceCriteria: ["Fallback agents", "Reduced capability"], minScore: 70 } },
+  { id: "arch-08", category: "architecture", task: "Design audit trail for all agent actions", expected: { files: ["runtime/"], tests: ["audit trail"], acceptanceCriteria: ["Complete logging", "Tamper-proof"], minScore: 70 } },
+  { id: "arch-09", category: "architecture", task: "Implement rate limiting for API calls", expected: { files: ["engine/"], tests: ["rate limiting"], acceptanceCriteria: ["Token bucket", "Graceful queuing"], minScore: 60 } },
+  { id: "arch-10", category: "architecture", task: "Design horizontal scaling strategy", expected: { files: ["docs/"], tests: ["scaling"], acceptanceCriteria: ["Stateless design", "Load balancer compatible"], minScore: 50 } },
+
+  // Recovery (10)
+  { id: "recv-01", category: "recovery", task: "Recover from agent timeout during execution", expected: { files: ["orchestration/DAGExecutor.ts"], tests: ["timeout recovery"], acceptanceCriteria: ["Clean timeout", "Retry or skip"], minScore: 70 } },
+  { id: "recv-02", category: "recovery", task: "Recover from network failure during API call", expected: { files: ["engine/"], tests: ["network recovery"], acceptanceCriteria: ["Retry with backoff", "Error reported"], minScore: 70 } },
+  { id: "recv-03", category: "recovery", task: "Recover from corrupted event log", expected: { files: ["runtime/EventBus.ts"], tests: ["corrupt log"], acceptanceCriteria: ["Partial recovery", "Error logged"], minScore: 70 } },
+  { id: "recv-04", category: "recovery", task: "Recover from agent crash mid-task", expected: { files: ["orchestration/"], tests: ["crash recovery"], acceptanceCriteria: ["Task re-assigned", "No data loss"], minScore: 70 } },
+  { id: "recv-05", category: "recovery", task: "Recover from disk full during logging", expected: { files: ["engine/StructuredLogger.ts"], tests: ["disk full"], acceptanceCriteria: ["Graceful degradation", "In-memory fallback"], minScore: 60 } },
+  { id: "recv-06", category: "recovery", task: "Recover from invalid AST in command", expected: { files: ["sandbox/ExecutionSandbox.ts"], tests: ["invalid AST"], acceptanceCriteria: ["Validation error", "No crash"], minScore: 70 } },
+  { id: "recv-07", category: "recovery", task: "Recover from reviewer producing invalid score", expected: { files: ["orchestration/ReviewerAgent.ts"], tests: ["invalid score"], acceptanceCriteria: ["Score clamped", "Default to FAIL"], minScore: 70 } },
+  { id: "recv-08", category: "recovery", task: "Recover from DAG with missing dependencies", expected: { files: ["orchestration/DAGExecutor.ts"], tests: ["missing deps"], acceptanceCriteria: ["Error message", "No infinite loop"], minScore: 80 } },
+  { id: "recv-09", category: "recovery", task: "Recover from memory pressure during large DAG", expected: { files: ["orchestration/DAGExecutor.ts"], tests: ["memory pressure"], acceptanceCriteria: ["Batch execution", "No OOM"], minScore: 60 } },
+  { id: "recv-10", category: "recovery", task: "Recover from concurrent modification of agent registry", expected: { files: ["orchestration/registry/AgentRegistry.ts"], tests: ["concurrent mod"], acceptanceCriteria: ["No corruption", "Consistent state"], minScore: 70 } },
+];
+
+export function getBenchmarksByCategory(category: string): BenchmarkTask[] {
+  return BENCHMARK_DATASET.filter(t => t.category === category);
+}
+
+export function getBenchmarkById(id: string): BenchmarkTask | undefined {
+  return BENCHMARK_DATASET.find(t => t.id === id);
+}

package/src/evaluation/RealWorldBenchmark.ts

@@ -0,0 +1,91 @@
+/**
+ * RealWorldBenchmark — 50 real-world tasks with difficulty tiers.
+ */
+export type Difficulty = "L1" | "L2" | "L3" | "L4";
+export type Category = "self-development" | "bug-fix" | "feature-delivery" | "adversarial" | "exploratory";
+
+export interface RealWorldTask {
+  id: string;
+  category: Category;
+  difficulty: Difficulty;
+  task: string;
+  expected: {
+    outputs: string[];
+    tests: string[];
+    acceptanceCriteria: string[];
+  };
+}
+
+export const REAL_WORLD_BENCHMARKS: RealWorldTask[] = [
+  // === Self Development (10) ===
+  { id: "rw-01", category: "self-development", difficulty: "L2", task: "Add plugin loading system to Agent Hive CLI", expected: { outputs: ["src/plugins/PluginLoader.ts"], tests: ["plugin load", "plugin not found"], acceptanceCriteria: ["Dynamic plugin import", "Error on missing plugin"] } },
+  { id: "rw-02", category: "self-development", difficulty: "L3", task: "Implement config validation with schema for .agent-hive/config.json", expected: { outputs: ["src/config/ConfigValidator.ts"], tests: ["valid config", "invalid config", "missing fields"], acceptanceCriteria: ["Schema validation", "Descriptive errors"] } },
+  { id: "rw-03", category: "self-development", difficulty: "L2", task: "Add workflow DSL parser for task definitions", expected: { outputs: ["src/workflow/WorkflowDSL.ts"], tests: ["parse simple", "parse complex"], acceptanceCriteria: ["DSL syntax", "Error on invalid"] } },
+  { id: "rw-04", category: "self-development", difficulty: "L3", task: "Build replay visualization showing agent message flow", expected: { outputs: ["src/runtime/ReplayVisualizer.ts"], tests: ["generate mermaid", "generate timeline"], acceptanceCriteria: ["Mermaid output", "Timeline output"] } },
+  { id: "rw-05", category: "self-development", difficulty: "L2", task: "Add task dashboard showing active/completed/failed tasks", expected: { outputs: ["src/commands/dashboard.ts"], tests: ["show active", "show completed"], acceptanceCriteria: ["Task counts", "Status display"] } },
+  { id: "rw-06", category: "self-development", difficulty: "L1", task: "Add --json output flag to all CLI commands", expected: { outputs: ["src/commands/cli.ts"], tests: ["json output", "default output"], acceptanceCriteria: ["Valid JSON", "All commands supported"] } },
+  { id: "rw-07", category: "self-development", difficulty: "L4", task: "Implement hot-reload for agent configurations", expected: { outputs: ["src/orchestration/registry/HotReloader.ts"], tests: ["reload on change", "no restart"], acceptanceCriteria: ["File watcher", "Config update"] } },
+  { id: "rw-08", category: "self-development", difficulty: "L2", task: "Add agent performance profiling per task", expected: { outputs: ["src/evaluation/AgentProfiler.ts"], tests: ["profile task", "aggregate stats"], acceptanceCriteria: ["Per-agent stats", "Duration tracking"] } },
+  { id: "rw-09", category: "self-development", difficulty: "L3", task: "Implement task priority queue with weighted scheduling", expected: { outputs: ["src/orchestration/TaskQueue.ts"], tests: ["priority order", "weight balance"], acceptanceCriteria: ["Priority respected", "Fair scheduling"] } },
+  { id: "rw-10", category: "self-development", difficulty: "L1", task: "Add version check command showing all component versions", expected: { outputs: ["src/commands/version.ts"], tests: ["show versions"], acceptanceCriteria: ["All components listed"] } },
+
+  // === Bug Fix (10) ===
+  { id: "rw-11", category: "bug-fix", difficulty: "L2", task: "Fix Windows path separator in log output (backslash vs forward slash)", expected: { outputs: ["src/engine/ResultNormalizer.ts"], tests: ["windows path normalized"], acceptanceCriteria: ["Consistent path format"] } },
+  { id: "rw-12", category: "bug-fix", difficulty: "L3", task: "Fix CLI hanging when OpenAI SDK keeps event loop alive", expected: { outputs: ["src/commands/cli.ts"], tests: ["clean exit"], acceptanceCriteria: ["Process exits after command"] } },
+  { id: "rw-13", category: "bug-fix", difficulty: "L2", task: "Fix EventBus file write race condition on concurrent publishes", expected: { outputs: ["src/runtime/EventBus.ts"], tests: ["concurrent write"], acceptanceCriteria: ["No data corruption"] } },
+  { id: "rw-14", category: "bug-fix", difficulty: "L1", task: "Fix --version flag being treated as task instead of command", expected: { outputs: ["src/commands/cli.ts"], tests: ["--version works"], acceptanceCriteria: ["Shows version, not task"] } },
+  { id: "rw-15", category: "bug-fix", difficulty: "L3", task: "Fix DAG executor infinite loop when all tasks have unmet dependencies", expected: { outputs: ["src/orchestration/DAGExecutor.ts"], tests: ["deadlock detection"], acceptanceCriteria: ["Error on deadlock"] } },
+  { id: "rw-16", category: "bug-fix", difficulty: "L2", task: "Fix reviewer accepting empty string output as valid", expected: { outputs: ["src/orchestration/ReviewerAgent.ts"], tests: ["empty rejected"], acceptanceCriteria: ["Score < 60 for empty"] } },
+  { id: "rw-17", category: "bug-fix", difficulty: "L1", task: "Fix missing .agent-hive directory causing crash on first run", expected: { outputs: ["src/commands/cli.ts"], tests: ["auto create dirs"], acceptanceCriteria: ["No crash, dirs created"] } },
+  { id: "rw-18", category: "bug-fix", difficulty: "L3", task: "Fix sandbox not detecting pipe character in command name", expected: { outputs: ["src/sandbox/ExecutionSandbox.ts"], tests: ["pipe rejected"], acceptanceCriteria: ["| in cmd rejected"] } },
+  { id: "rw-19", category: "bug-fix", difficulty: "L2", task: "Fix memory search returning results from wrong project", expected: { outputs: ["src/memory/MemorySearch.ts"], tests: ["project scoping"], acceptanceCriteria: ["Results filtered by project"] } },
+  { id: "rw-20", category: "bug-fix", difficulty: "L1", task: "Fix doctor command not checking API connectivity", expected: { outputs: ["src/commands/cli.ts"], tests: ["api check"], acceptanceCriteria: ["API status shown"] } },
+
+  // === Feature Delivery (10) ===
+  { id: "rw-21", category: "feature-delivery", difficulty: "L3", task: "Implement agent template system for creating new agents", expected: { outputs: ["src/orchestration/templates/AgentTemplate.ts"], tests: ["create from template", "custom template"], acceptanceCriteria: ["Template loading", "Agent creation"] } },
+  { id: "rw-22", category: "feature-delivery", difficulty: "L2", task: "Add workflow template library (code-review, bug-fix, feature)", expected: { outputs: ["src/orchestration/templates/WorkflowTemplate.ts"], tests: ["load template", "apply template"], acceptanceCriteria: ["Template library", "DAG generation"] } },
+  { id: "rw-23", category: "feature-delivery", difficulty: "L3", task: "Implement CLI extension system for custom commands", expected: { outputs: ["src/commands/ExtensionLoader.ts"], tests: ["load extension", "register command"], acceptanceCriteria: ["Dynamic commands", "Error handling"] } },
+  { id: "rw-24", category: "feature-delivery", difficulty: "L4", task: "Build plugin marketplace listing available plugins", expected: { outputs: ["src/plugins/Marketplace.ts"], tests: ["list plugins", "install plugin"], acceptanceCriteria: ["Plugin registry", "Install flow"] } },
+  { id: "rw-25", category: "feature-delivery", difficulty: "L2", task: "Add task export to JSON/CSV formats", expected: { outputs: ["src/commands/export.ts"], tests: ["export json", "export csv"], acceptanceCriteria: ["Valid formats", "All fields"] } },
+  { id: "rw-26", category: "feature-delivery", difficulty: "L3", task: "Implement agent-to-agent direct messaging", expected: { outputs: ["src/orchestration/AgentMessenger.ts"], tests: ["send message", "receive message"], acceptanceCriteria: ["Direct channel", "Delivery confirmation"] } },
+  { id: "rw-27", category: "feature-delivery", difficulty: "L2", task: "Add execution cost estimation before running tasks", expected: { outputs: ["src/evaluation/CostEstimator.ts"], tests: ["estimate tokens", "estimate cost"], acceptanceCriteria: ["Token estimate", "Cost estimate"] } },
+  { id: "rw-28", category: "feature-delivery", difficulty: "L1", task: "Add dry-run mode showing what would execute without running", expected: { outputs: ["src/commands/cli.ts"], tests: ["dry run output"], acceptanceCriteria: ["Plan shown", "No execution"] } },
+  { id: "rw-29", category: "feature-delivery", difficulty: "L3", task: "Implement task scheduling with cron-like syntax", expected: { outputs: ["src/orchestration/TaskScheduler.ts"], tests: ["schedule task", "cron parse"], acceptanceCriteria: ["Cron syntax", "Auto-trigger"] } },
+  { id: "rw-30", category: "feature-delivery", difficulty: "L2", task: "Add agent capability auto-discovery from execution history", expected: { outputs: ["src/orchestration/registry/CapabilityDiscovery.ts"], tests: ["discover caps", "update registry"], acceptanceCriteria: ["Auto-detect", "Registry update"] } },
+
+  // === Adversarial (10) ===
+  { id: "rw-31", category: "adversarial", difficulty: "L3", task: "Detect and reject task that claims success but produces no output", expected: { outputs: ["src/orchestration/ReviewerAgent.ts"], tests: ["fake success detected"], acceptanceCriteria: ["Empty output = FAIL"] } },
+  { id: "rw-32", category: "adversarial", difficulty: "L4", task: "Detect task that produces fake test results", expected: { outputs: ["src/evaluation/FakeTestDetector.ts"], tests: ["fake test detected"], acceptanceCriteria: ["Test output validation"] } },
+  { id: "rw-33", category: "adversarial", difficulty: "L3", task: "Detect task that references non-existent files as output", expected: { outputs: ["src/evaluation/OutputValidator.ts"], tests: ["missing file detected"], acceptanceCriteria: ["File existence check"] } },
+  { id: "rw-34", category: "adversarial", difficulty: "L2", task: "Detect task that produces incorrect logic (wrong algorithm)", expected: { outputs: ["src/evaluation/LogicValidator.ts"], tests: ["logic error detected"], acceptanceCriteria: ["Output correctness"] } },
+  { id: "rw-35", category: "adversarial", difficulty: "L3", task: "Detect judge collusion with reviewer (always same decision)", expected: { outputs: ["src/evaluation/CollusionDetector.ts"], tests: ["collusion detected"], acceptanceCriteria: ["Independence check"] } },
+  { id: "rw-36", category: "adversarial", difficulty: "L2", task: "Detect agent that takes credit for another agent's work", expected: { outputs: ["src/evaluation/AttributionChecker.ts"], tests: ["attribution verified"], acceptanceCriteria: ["Agent attribution"] } },
+  { id: "rw-37", category: "adversarial", difficulty: "L4", task: "Detect task that passes tests but has hidden security vulnerability", expected: { outputs: ["src/evaluation/SecurityScanner.ts"], tests: ["vuln detected"], acceptanceCriteria: ["Security pattern check"] } },
+  { id: "rw-38", category: "adversarial", difficulty: "L2", task: "Detect task that modifies benchmark files to pass", expected: { outputs: ["src/evaluation/BenchmarkGuard.ts"], tests: ["benchmark tamper detected"], acceptanceCriteria: ["Checksum verification"] } },
+  { id: "rw-39", category: "adversarial", difficulty: "L3", task: "Detect task that times out but reports success", expected: { outputs: ["src/evaluation/TimeoutValidator.ts"], tests: ["timeout success detected"], acceptanceCriteria: ["Time consistency"] } },
+  { id: "rw-40", category: "adversarial", difficulty: "L1", task: "Detect task with empty instruction", expected: { outputs: ["src/sandbox/ExecutionSandbox.ts"], tests: ["empty instruction rejected"], acceptanceCriteria: ["Validation error"] } },
+
+  // === Exploratory (10) ===
+  { id: "rw-41", category: "exploratory", difficulty: "L2", task: "Explore: Can Agent Hive refactor its own Planner?", expected: { outputs: ["docs/self-refactor-report.md"], tests: ["planner analysis"], acceptanceCriteria: ["Analysis complete", "Recommendations"] } },
+  { id: "rw-42", category: "exploratory", difficulty: "L3", task: "Explore: What happens when all agents fail simultaneously?", expected: { outputs: ["docs/all-fail-scenario.md"], tests: ["all fail handled"], acceptanceCriteria: ["Graceful degradation"] } },
+  { id: "rw-43", category: "exploratory", difficulty: "L2", task: "Explore: Can we add a new agent type without code changes?", expected: { outputs: ["docs/dynamic-agent-report.md"], tests: ["dynamic agent"], acceptanceCriteria: ["Config-based agent"] } },
+  { id: "rw-44", category: "exploratory", difficulty: "L3", task: "Explore: Performance characteristics under 100 concurrent tasks", expected: { outputs: ["docs/performance-report.md"], tests: ["load test"], acceptanceCriteria: ["No crash", "Metrics collected"] } },
+  { id: "rw-45", category: "exploratory", difficulty: "L1", task: "Explore: What CLI commands do users actually use?", expected: { outputs: ["docs/usage-analysis.md"], tests: ["usage tracking"], acceptanceCriteria: ["Command frequency"] } },
+  { id: "rw-46", category: "exploratory", difficulty: "L4", task: "Explore: Can Agent Hive generate its own test suite?", expected: { outputs: ["docs/auto-test-report.md"], tests: ["generated tests"], acceptanceCriteria: ["Test generation", "Tests pass"] } },
+  { id: "rw-47", category: "exploratory", difficulty: "L2", task: "Explore: What is the optimal number of agents for a task?", expected: { outputs: ["docs/agent-scaling-report.md"], tests: ["scaling analysis"], acceptanceCriteria: ["Recommendation"] } },
+  { id: "rw-48", category: "exploratory", difficulty: "L3", task: "Explore: Can replay data be used to train better routers?", expected: { outputs: ["docs/replay-training-report.md"], tests: ["training analysis"], acceptanceCriteria: ["Training feasibility"] } },
+  { id: "rw-49", category: "exploratory", difficulty: "L1", task: "Explore: What error messages do users see most often?", expected: { outputs: ["docs/error-analysis.md"], tests: ["error frequency"], acceptanceCriteria: ["Top errors listed"] } },
+  { id: "rw-50", category: "exploratory", difficulty: "L3", task: "Explore: Can Agent Hive work with non-OpenAI providers?", expected: { outputs: ["docs/provider-compat-report.md"], tests: ["provider test"], acceptanceCriteria: ["Compatibility matrix"] } },
+];
+
+export function getByCategory(cat: Category): RealWorldTask[] {
+  return REAL_WORLD_BENCHMARKS.filter(t => t.category === cat);
+}
+
+export function getByDifficulty(diff: Difficulty): RealWorldTask[] {
+  return REAL_WORLD_BENCHMARKS.filter(t => t.difficulty === diff);
+}
+
+export function getById(id: string): RealWorldTask | undefined {
+  return REAL_WORLD_BENCHMARKS.find(t => t.id === id);
+}

package/src/evaluation/RealWorldRunner.ts

@@ -0,0 +1,171 @@
+/**
+ * RealWorldRunner — runs real-world benchmarks through the full orchestration pipeline.
+ */
+import { RealWorldTask, REAL_WORLD_BENCHMARKS, Difficulty } from "./RealWorldBenchmark";
+import { DecisionTraceCollector, DecisionTrace } from "./DecisionTrace";
+import { Planner } from "../orchestration/Planner";
+import { AgentRouter } from "../orchestration/AgentRouter";
+import { AgentRegistry } from "../orchestration/registry/AgentRegistry";
+import { ReviewerAgent } from "../orchestration/ReviewerAgent";
+import { JudgeAgent } from "../orchestration/JudgeAgent";
+import { Agent, Task, TaskResult, ReviewResult, JudgeResult } from "../orchestration/contracts/Agent";
+
+export interface SuccessCriteria {
+  implementation: boolean;
+  testsPassed: boolean;
+  reviewerApproved: boolean;
+  judgeApproved: boolean;
+}
+
+export interface RealWorldResult {
+  taskId: string;
+  category: string;
+  difficulty: Difficulty;
+  success: boolean;
+  criteria: SuccessCriteria;
+  plannerScore: number;
+  routerScore: number;
+  reviewScore: number;
+  judgeScore: number;
+  executionTimeMs: number;
+  decisionTraces: DecisionTrace[];
+}
+
+export interface RealWorldMetrics {
+  taskCompletionRate: number;
+  taskSuccessRate: number;
+  reviewerAccuracy: number;
+  judgeAccuracy: number;
+  retryRate: number;
+  failureRate: number;
+  averageExecutionTimeMs: number;
+  difficultyBreakdown: Record<Difficulty, { total: number; passed: number; rate: number }>;
+}
+
+export class RealWorldRunner {
+  private planner: Planner;
+  private router: AgentRouter;
+  private reviewer: ReviewerAgent;
+  private judge: JudgeAgent;
+  private traceCollector: DecisionTraceCollector;
+
+  constructor(registry: AgentRegistry) {
+    this.planner = new Planner();
+    this.router = new AgentRouter(registry);
+    this.reviewer = new ReviewerAgent();
+    this.judge = new JudgeAgent();
+    this.traceCollector = new DecisionTraceCollector();
+  }
+
+  async runTask(task: RealWorldTask): Promise<RealWorldResult> {
+    const startTime = Date.now();
+    const traceId = `rw-${task.id}`;
+
+    // 1. Plan
+    const plan = this.planner.plan(task.task);
+    this.traceCollector.record({
+      traceId, agent: "planner", decision: `Split into ${plan.length} tasks`,
+      rationale: `Task type detection: ${task.task.slice(0, 50)}`, confidence: 0.8, timestamp: Date.now(),
+    });
+    const plannerScore = this.evaluatePlan(plan, task);
+
+    // 2. Route
+    let routerScore = 100;
+    for (const t of plan) {
+      const route = this.router.route(t);
+      if (!route.assignedAgent) routerScore -= 20;
+      this.traceCollector.record({
+        traceId, agent: "router", decision: `Route ${t.id} → ${route.assignedAgent?.id || "none"}`,
+        rationale: route.reason, confidence: route.assignedAgent ? 0.9 : 0.1, timestamp: Date.now(),
+      });
+    }
+    routerScore = Math.max(0, routerScore);
+
+    // 3. Execute (simulated)
+    const result: TaskResult = {
+      taskId: task.id, agentId: "multi-agent", success: true,
+      output: `Real-world output for: ${task.task}`, durationMs: Date.now() - startTime + 20,
+    };
+
+    // 4. Review
+    const review = await this.reviewer.review(result);
+    this.traceCollector.record({
+      traceId, agent: "reviewer", decision: review.approved ? "APPROVE" : "REJECT",
+      rationale: `Score: ${review.score}, Findings: ${review.findings.length}`, confidence: review.score / 100, timestamp: Date.now(),
+    });
+
+    // 5. Judge
+    const judgeResult = this.judge.judge(result, review);
+    this.traceCollector.record({
+      traceId, agent: "judge", decision: judgeResult.accepted ? "ACCEPT" : "REJECT",
+      rationale: judgeResult.reason, confidence: judgeResult.score / 100, timestamp: Date.now(),
+    });
+
+    // 6. Success criteria
+    const criteria: SuccessCriteria = {
+      implementation: result.success,
+      testsPassed: result.success,
+      reviewerApproved: review.approved,
+      judgeApproved: judgeResult.accepted,
+    };
+    const success = criteria.implementation && criteria.testsPassed && criteria.reviewerApproved && criteria.judgeApproved;
+
+    return {
+      taskId: task.id,
+      category: task.category,
+      difficulty: task.difficulty,
+      success,
+      criteria,
+      plannerScore,
+      routerScore,
+      reviewScore: review.score,
+      judgeScore: judgeResult.score,
+      executionTimeMs: Date.now() - startTime,
+      decisionTraces: this.traceCollector.getByTraceId(traceId),
+    };
+  }
+
+  async runAll(): Promise<{ results: RealWorldResult[]; metrics: RealWorldMetrics }> {
+    const results: RealWorldResult[] = [];
+    for (const task of REAL_WORLD_BENCHMARKS) {
+      results.push(await this.runTask(task));
+    }
+    return { results, metrics: this.computeMetrics(results) };
+  }
+
+  private evaluatePlan(plan: Task[], task: RealWorldTask): number {
+    let score = 100;
+    if (plan.length > 6) score -= (plan.length - 6) * 10;
+    if (plan.length === 1 && task.task.length > 50) score -= 20;
+    if (!plan.some(t => t.dependencies.length > 0) && plan.length > 1) score -= 15;
+    return Math.max(0, score);
+  }
+
+  private computeMetrics(results: RealWorldResult[]): RealWorldMetrics {
+    const n = results.length || 1;
+    const diffBreakdown: Record<Difficulty, { total: number; passed: number; rate: number }> = {
+      L1: { total: 0, passed: 0, rate: 0 },
+      L2: { total: 0, passed: 0, rate: 0 },
+      L3: { total: 0, passed: 0, rate: 0 },
+      L4: { total: 0, passed: 0, rate: 0 },
+    };
+    for (const r of results) {
+      diffBreakdown[r.difficulty].total++;
+      if (r.success) diffBreakdown[r.difficulty].passed++;
+    }
+    for (const d of ["L1", "L2", "L3", "L4"] as Difficulty[]) {
+      diffBreakdown[d].rate = diffBreakdown[d].total > 0 ? diffBreakdown[d].passed / diffBreakdown[d].total : 0;
+    }
+
+    return {
+      taskCompletionRate: results.filter(r => r.criteria.implementation).length / n,
+      taskSuccessRate: results.filter(r => r.success).length / n,
+      reviewerAccuracy: results.filter(r => r.criteria.reviewerApproved === r.success).length / n,
+      judgeAccuracy: results.filter(r => r.criteria.judgeApproved === r.success).length / n,
+      retryRate: 0,
+      failureRate: results.filter(r => !r.success).length / n,
+      averageExecutionTimeMs: results.reduce((s, r) => s + r.executionTimeMs, 0) / n,
+      difficultyBreakdown: diffBreakdown,
+    };
+  }
+}

package/src/evaluation/ValidationRunner.ts

@@ -0,0 +1,156 @@
+/**
+ * ValidationRunner — runs benchmarks and produces validation reports.
+ */
+import { BenchmarkTask, BENCHMARK_DATASET } from "./GroundTruthDataset";
+import { EvaluationMetricsCollector, EvaluationMetrics } from "./EvaluationMetrics";
+import { Planner } from "../orchestration/Planner";
+import { AgentRouter } from "../orchestration/AgentRouter";
+import { AgentRegistry } from "../orchestration/registry/AgentRegistry";
+import { EventBus } from "../runtime/EventBus";
+import { DAGExecutor } from "../orchestration/DAGExecutor";
+import { ReviewerAgent } from "../orchestration/ReviewerAgent";
+import { JudgeAgent } from "../orchestration/JudgeAgent";
+import { Agent, Task, TaskResult, ReviewResult, JudgeResult } from "../orchestration/contracts/Agent";
+
+export interface ValidationResult {
+  taskId: string;
+  category: string;
+  task: string;
+  plannerScore: number;
+  routerScore: number;
+  reviewScore: number;
+  judgeScore: number;
+  passed: boolean;
+  durationMs: number;
+}
+
+export interface ValidationReport {
+  timestamp: string;
+  totalTasks: number;
+  passed: number;
+  failed: number;
+  metrics: EvaluationMetrics;
+  results: ValidationResult[];
+}
+
+export class ValidationRunner {
+  private planner: Planner;
+  private router: AgentRouter;
+  private dag: DAGExecutor;
+  private reviewer: ReviewerAgent;
+  private judge: JudgeAgent;
+  private metrics: EvaluationMetricsCollector;
+
+  constructor(agents: Map<string, Agent>, registry: AgentRegistry, eventBus: EventBus) {
+    this.planner = new Planner();
+    this.router = new AgentRouter(registry);
+    this.dag = new DAGExecutor(this.router, agents, eventBus);
+    this.reviewer = new ReviewerAgent();
+    this.judge = new JudgeAgent();
+    this.metrics = new EvaluationMetricsCollector();
+  }
+
+  /** Run a single benchmark task */
+  async runTask(benchmark: BenchmarkTask): Promise<ValidationResult> {
+    const startTime = Date.now();
+
+    // 1. Plan
+    const plan = this.planner.plan(benchmark.task);
+    const plannerScore = this.evaluatePlan(plan, benchmark);
+
+    // 2. Route
+    let routerScore = 100;
+    for (const task of plan) {
+      const route = this.router.route(task);
+      if (!route.assignedAgent) routerScore -= 20;
+    }
+    routerScore = Math.max(0, routerScore);
+
+    // 3. Execute (simulated)
+    const execResults = new Map<string, TaskResult>();
+    for (const task of plan) {
+      execResults.set(task.id, {
+        taskId: task.id,
+        agentId: "simulated",
+        success: true,
+        output: `Simulated output for: ${task.title}`,
+        durationMs: 10,
+      });
+    }
+
+    // 4. Review
+    const firstResult = Array.from(execResults.values())[0];
+    const review = await this.reviewer.review(firstResult);
+    const reviewScore = review.score;
+
+    // 5. Judge
+    const judgeResult = this.judge.judge(firstResult, review);
+    const judgeScore = judgeResult.score;
+
+    const durationMs = Date.now() - startTime;
+    const passed = judgeResult.accepted && plannerScore >= 50;
+
+    // Record metrics
+    this.metrics.record({
+      completed: true,
+      success: passed,
+      reviewCorrect: reviewScore >= benchmark.expected.minScore,
+      judgeCorrect: judgeResult.accepted === passed,
+      reviewFP: !passed && review.approved,
+      reviewFN: passed && !review.approved,
+      judgeIndependent: judgeResult.accepted !== review.approved,
+      retried: false,
+      failed: !passed,
+      executionTime: durationMs,
+    });
+
+    return {
+      taskId: benchmark.id,
+      category: benchmark.category,
+      task: benchmark.task,
+      plannerScore,
+      routerScore,
+      reviewScore,
+      judgeScore,
+      passed,
+      durationMs,
+    };
+  }
+
+  /** Run all benchmarks */
+  async runAll(): Promise<ValidationReport> {
+    const results: ValidationResult[] = [];
+    for (const benchmark of BENCHMARK_DATASET) {
+      results.push(await this.runTask(benchmark));
+    }
+
+    return {
+      timestamp: new Date().toISOString(),
+      totalTasks: results.length,
+      passed: results.filter(r => r.passed).length,
+      failed: results.filter(r => !r.passed).length,
+      metrics: this.metrics.compute(),
+      results,
+    };
+  }
+
+  private evaluatePlan(plan: Task[], benchmark: BenchmarkTask): number {
+    let score = 100;
+
+    // Over-splitting penalty
+    if (plan.length > 6) score -= (plan.length - 6) * 10;
+
+    // Under-splitting penalty
+    if (plan.length === 1 && benchmark.task.length > 50) score -= 30;
+
+    // Dependency correctness
+    const hasDeps = plan.some(t => t.dependencies.length > 0);
+    if (!hasDeps && plan.length > 1) score -= 20;
+
+    // Unique IDs
+    const ids = new Set(plan.map(t => t.id));
+    if (ids.size !== plan.length) score -= 30;
+
+    return Math.max(0, score);
+  }
+}

package/src/events/AgentEvent.ts

@@ -0,0 +1,45 @@
+/**
+ * AgentEvent — events related to agent lifecycle.
+ */
+
+export interface AgentReceivedTask {
+  type: "AgentReceivedTask";
+  id: string;
+  traceId: string;
+  timestamp: number;
+  payload: { agentId: string; taskId: string; instruction: string };
+}
+
+export interface AgentPlanned {
+  type: "AgentPlanned";
+  id: string;
+  traceId: string;
+  timestamp: number;
+  payload: { agentId: string; plan: string; taskId: string };
+}
+
+export interface AgentExecuted {
+  type: "AgentExecuted";
+  id: string;
+  traceId: string;
+  timestamp: number;
+  payload: { agentId: string; taskId: string; output: string; exitCode: number; durationMs: number };
+}
+
+export interface AgentCompleted {
+  type: "AgentCompleted";
+  id: string;
+  traceId: string;
+  timestamp: number;
+  payload: { agentId: string; taskId: string; score: number };
+}
+
+export interface AgentFailed {
+  type: "AgentFailed";
+  id: string;
+  traceId: string;
+  timestamp: number;
+  payload: { agentId: string; taskId: string; error: string };
+}
+
+export type AgentEvent = AgentReceivedTask | AgentPlanned | AgentExecuted | AgentCompleted | AgentFailed;

package/src/events/DecisionEvent.ts

@@ -0,0 +1,16 @@
+/**
+ * DecisionEvent — events related to agent decisions.
+ */
+
+export interface DecisionEvent {
+  type: "DecisionMade";
+  id: string;
+  traceId: string;
+  timestamp: number;
+  payload: {
+    agentId: string;
+    decision: string;
+    reason: string;
+    confidence: number;
+  };
+}

package/src/events/ExecutionEvent.ts

@@ -0,0 +1,38 @@
+/**
+ * ExecutionEvent — events related to command execution.
+ */
+import { CompiledCommand } from "../compiler/CommandCompiler";
+
+export interface ExecutionStarted {
+  type: "ExecutionStarted";
+  id: string;
+  traceId: string;
+  timestamp: number;
+  payload: { command: string; args: string[] };
+}
+
+export interface ExecutionCompiled {
+  type: "ExecutionCompiled";
+  id: string;
+  traceId: string;
+  timestamp: number;
+  payload: { compiled: CompiledCommand };
+}
+
+export interface ExecutionCompleted {
+  type: "ExecutionCompleted";
+  id: string;
+  traceId: string;
+  timestamp: number;
+  payload: { stdout: string; stderr: string; exitCode: number; durationMs: number };
+}
+
+export interface ExecutionFailed {
+  type: "ExecutionFailed";
+  id: string;
+  traceId: string;
+  timestamp: number;
+  payload: { error: string; exitCode: number; durationMs: number };
+}
+
+export type ExecutionEvent = ExecutionStarted | ExecutionCompiled | ExecutionCompleted | ExecutionFailed;

package/src/events/SystemEvent.ts

@@ -0,0 +1,11 @@
+/**
+ * SystemEvent — system-level events.
+ */
+
+export interface SystemEvent {
+  type: "SystemStarted" | "SystemShutdown" | "TaskSubmitted" | "TaskCompleted" | "TaskFailed";
+  id: string;
+  traceId: string;
+  timestamp: number;
+  payload: Record<string, unknown>;
+}