@toyako/hive 1.0.0

package/src/commands/test-runtime-intelligence.ts

@@ -0,0 +1,235 @@
+/**
+ * Test: Runtime Intelligence (v2.1.5)
+ */
+import { CapabilityDiscovery } from "../runtime/CapabilityDiscovery";
+import { RuntimeScoreManager } from "../runtime/RuntimeScoreManager";
+import { RuntimeSelector } from "../runtime/RuntimeSelector";
+import { Broker } from "../broker/Broker";
+import { AgentAdapter, AgentResult, ReviewResult, Task } from "../types";
+import * as fs from "fs";
+
+// ─── Mock Adapters with DISTINCT capabilities ────────
+
+class MockPlannerAdapter implements AgentAdapter {
+  name = "hermes";
+  role = "planner" as const;
+  capabilities = ["planning", "research"];
+  async detect() { return true; }
+  async health() { return true; }
+  async execute(task: Task): Promise<AgentResult> {
+    return { success: true, output: `{"capabilities": ["planning", "research"]}` };
+  }
+}
+
+class MockCodingAdapter implements AgentAdapter {
+  name = "codex";
+  role = "developer" as const;
+  capabilities = ["coding", "refactor"];
+  async detect() { return true; }
+  async health() { return true; }
+  async execute(task: Task): Promise<AgentResult> {
+    return { success: true, output: `{"capabilities": ["coding", "refactor"]}` };
+  }
+}
+
+class MockReviewAdapter implements AgentAdapter {
+  name = "claude";
+  role = "reviewer" as const;
+  capabilities = ["review", "architecture"];
+  async detect() { return true; }
+  async health() { return true; }
+  async execute(task: Task): Promise<AgentResult> {
+    return { success: true, output: `{"capabilities": ["review", "architecture"]}` };
+  }
+  async review(task: Task): Promise<ReviewResult> {
+    return { decision: "PASS", score: 90, issues: [] };
+  }
+}
+
+let passed = 0;
+let failed = 0;
+
+function assert(condition: boolean, msg: string) {
+  if (condition) {
+    console.log(`    ✓ ${msg}`);
+    passed++;
+  } else {
+    console.log(`    ✗ ${msg}`);
+    failed++;
+  }
+}
+
+async function main() {
+  // Clean persisted state
+  try { fs.rmSync(".agent-hive", { recursive: true, force: true }); } catch {}
+  fs.mkdirSync(".agent-hive", { recursive: true });
+
+  console.log("═══════════════════════════════════════════");
+  console.log("  Test Suite: Runtime Intelligence (v2.1.5)");
+  console.log("═══════════════════════════════════════════\n");
+
+  // ── Test 1: Capability Discovery ──
+  console.log("  [Test 1] Capability Discovery");
+  console.log("  ────────────────────────────────────");
+  {
+    const discovery = new CapabilityDiscovery();
+
+    const r1 = await discovery.discover(new MockPlannerAdapter());
+    assert(r1.capabilities.includes("planning"), "Planner has 'planning'");
+    assert(r1.capabilities.includes("research"), "Planner has 'research'");
+    assert(r1.confidence >= 0.8, `Confidence: ${r1.confidence}`);
+    assert(r1.source === "merged", `Source: ${r1.source}`);
+
+    const r2 = await discovery.discover(new MockCodingAdapter());
+    assert(r2.capabilities.includes("coding"), "Coder has 'coding'");
+    assert(r2.capabilities.includes("refactor"), "Coder has 'refactor'");
+
+    const r3 = await discovery.discover(new MockReviewAdapter());
+    assert(r3.capabilities.includes("review"), "Reviewer has 'review'");
+
+    const coders = discovery.findByCapability("coding");
+    assert(coders.length === 1, `Only 1 runtime with 'coding': ${coders.map(c=>c.runtimeId)}`);
+
+    const planners = discovery.findByCapability("planning");
+    assert(planners.length === 1, `Only 1 runtime with 'planning': ${planners.map(c=>c.runtimeId)}`);
+
+    console.log("  Test 1: PASS ✓\n");
+  }
+
+  // ── Test 2: Runtime Score System ──
+  console.log("  [Test 2] Runtime Score System");
+  console.log("  ────────────────────────────────────");
+  {
+    const scoreManager = new RuntimeScoreManager();
+
+    scoreManager.record({ runtimeId: "claude", role: "executor", success: true, latencyMs: 3000, reviewScore: 90, revisionCount: 0 });
+    scoreManager.record({ runtimeId: "claude", role: "executor", success: true, latencyMs: 5000, reviewScore: 85, revisionCount: 1 });
+    scoreManager.record({ runtimeId: "claude", role: "executor", success: false, latencyMs: 10000, reviewScore: 40, revisionCount: 3 });
+
+    scoreManager.record({ runtimeId: "hermes", role: "executor", success: true, latencyMs: 2000, reviewScore: 88, revisionCount: 0 });
+    scoreManager.record({ runtimeId: "hermes", role: "executor", success: true, latencyMs: 4000, reviewScore: 82, revisionCount: 0 });
+
+    const claudeStats = scoreManager.get("claude");
+    assert(claudeStats !== undefined, "Claude stats exist");
+    assert(claudeStats!.taskCount === 3, `Claude task count: ${claudeStats!.taskCount}`);
+    assert(Math.abs(claudeStats!.successRate - 2/3) < 0.01, `Claude success rate: ${claudeStats!.successRate.toFixed(2)}`);
+
+    const hermesStats = scoreManager.get("hermes");
+    assert(hermesStats!.successRate === 1.0, `Hermes success rate: ${hermesStats!.successRate}`);
+
+    const claudeScore = scoreManager.getScore("claude");
+    const hermesScore = scoreManager.getScore("hermes");
+    assert(hermesScore > claudeScore, `Hermes (${hermesScore.toFixed(2)}) > Claude (${claudeScore.toFixed(2)})`);
+
+    console.log("  Test 2: PASS ✓\n");
+  }
+
+  // ── Test 3: Auto Runtime Selection ──
+  console.log("  [Test 3] Auto Runtime Selection");
+  console.log("  ────────────────────────────────────");
+  {
+    const discovery = new CapabilityDiscovery();
+    const scoreManager = new RuntimeScoreManager();
+    const broker = new Broker();
+
+    await broker.registerAdapter(new MockCodingAdapter());
+    await broker.registerAdapter(new MockReviewAdapter());
+    await broker.registerAdapter(new MockPlannerAdapter());
+
+    await discovery.discover(new MockCodingAdapter());
+    await discovery.discover(new MockReviewAdapter());
+    await discovery.discover(new MockPlannerAdapter());
+
+    scoreManager.record({ runtimeId: "codex", role: "executor", success: true, latencyMs: 2000, reviewScore: 85 });
+    scoreManager.record({ runtimeId: "claude", role: "executor", success: true, latencyMs: 4000, reviewScore: 90 });
+    scoreManager.record({ runtimeId: "hermes", role: "executor", success: true, latencyMs: 3000, reviewScore: 80 });
+
+    const selector = new RuntimeSelector(discovery, scoreManager, broker.registry);
+
+    const codingResult = selector.select({ taskType: "coding" });
+    assert(codingResult !== null, "Coding selection returned result");
+    assert(codingResult!.runtimeId === "codex", `coding → ${codingResult!.runtimeId}`);
+
+    const reviewResult = selector.select({ taskType: "review" });
+    assert(reviewResult !== null, "Review selection returned result");
+    assert(reviewResult!.runtimeId === "claude", `review → ${reviewResult!.runtimeId}`);
+
+    const planningResult = selector.select({ taskType: "planning" });
+    assert(planningResult !== null, "Planning selection returned result");
+    assert(planningResult!.runtimeId === "hermes", `planning → ${planningResult!.runtimeId}`);
+
+    console.log("  Test 3: PASS ✓\n");
+  }
+
+  // ── Test 4: Capability Routing ──
+  console.log("  [Test 4] Capability Routing");
+  console.log("  ────────────────────────────────────");
+  {
+    const discovery = new CapabilityDiscovery();
+    const scoreManager = new RuntimeScoreManager();
+    const broker = new Broker();
+
+    await broker.registerAdapter(new MockCodingAdapter());
+    await broker.registerAdapter(new MockReviewAdapter());
+    await broker.registerAdapter(new MockPlannerAdapter());
+
+    await discovery.discover(new MockCodingAdapter());
+    await discovery.discover(new MockReviewAdapter());
+    await discovery.discover(new MockPlannerAdapter());
+
+    scoreManager.record({ runtimeId: "codex", role: "executor", success: true, latencyMs: 2000 });
+    scoreManager.record({ runtimeId: "claude", role: "executor", success: true, latencyMs: 4000 });
+    scoreManager.record({ runtimeId: "hermes", role: "executor", success: true, latencyMs: 3000 });
+
+    const selector = new RuntimeSelector(discovery, scoreManager, broker.registry);
+
+    const r1 = selector.select({ taskType: "coding" });
+    assert(r1 !== null && r1.runtimeId === "codex", `cap:coding → ${r1?.runtimeId}`);
+
+    const r2 = selector.select({ taskType: "review" });
+    assert(r2 !== null && r2.runtimeId === "claude", `cap:review → ${r2?.runtimeId}`);
+
+    const r3 = selector.select({ taskType: "planning" });
+    assert(r3 !== null && r3.runtimeId === "hermes", `cap:planning → ${r3?.runtimeId}`);
+
+    console.log("  Test 4: PASS ✓\n");
+  }
+
+  // ── Test 5: Failover ──
+  console.log("  [Test 5] Failover");
+  console.log("  ────────────────────────────────────");
+  {
+    const discovery = new CapabilityDiscovery();
+    const scoreManager = new RuntimeScoreManager();
+    const broker = new Broker();
+
+    await broker.registerAdapter(new MockCodingAdapter());
+    await broker.registerAdapter(new MockReviewAdapter());
+
+    await discovery.discover(new MockCodingAdapter());
+    await discovery.discover(new MockReviewAdapter());
+
+    scoreManager.record({ runtimeId: "codex", role: "executor", success: true, latencyMs: 2000 });
+    scoreManager.record({ runtimeId: "claude", role: "executor", success: true, latencyMs: 4000 });
+
+    const selector = new RuntimeSelector(discovery, scoreManager, broker.registry);
+
+    const normal = selector.select({ taskType: "coding" });
+    assert(normal !== null && normal.runtimeId === "codex", `Normal: coding → ${normal?.runtimeId}`);
+
+    const failover = selector.select({ taskType: "coding", exclude: ["codex"] });
+    assert(failover !== null, "Failover selection works");
+    // After excluding codex, should fall back to claude (which also has coding from probe)
+    assert(failover!.runtimeId === "claude", `Failover: coding → ${failover!.runtimeId}`);
+
+    console.log("  Test 5: PASS ✓\n");
+  }
+
+  console.log("═══════════════════════════════════════════");
+  console.log(`  Results: ${passed} passed, ${failed} failed`);
+  console.log("═══════════════════════════════════════════");
+
+  if (failed > 0) process.exit(1);
+}
+
+main().catch(err => { console.error("Fatal:", err); process.exit(1); });

package/src/commands/test-stress.ts

@@ -0,0 +1,131 @@
+/**
+ * Test A: Queue Stress Test — Detailed Evidence
+ */
+import { Broker } from "../broker/Broker";
+import { AgentAdapter, AgentResult, ReviewResult, Task } from "../types";
+
+class FastMockExecutor implements AgentAdapter {
+  name = "codex";
+  role = "developer" as const;
+  capabilities = ["coding"];
+  async detect() { return true; }
+  async health() { return true; }
+  async execute(task: Task, instruction?: string): Promise<AgentResult> {
+    await this.sleep(50);
+    return { success: true, output: `[Mock] Created file_${task.id}.js` };
+  }
+  private sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); }
+}
+
+class FastMockReviewer implements AgentAdapter {
+  name = "claude";
+  role = "reviewer" as const;
+  capabilities = ["review"];
+  async detect() { return true; }
+  async health() { return true; }
+  async execute(): Promise<AgentResult> { return { success: true, output: "" }; }
+  async review(): Promise<ReviewResult> {
+    await this.sleep(30);
+    return { decision: "PASS", score: 90, issues: [] };
+  }
+  private sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); }
+}
+
+async function main() {
+  // Clean up any leftover .agent-hive data from prior runs
+  const fs = require("fs");
+  fs.rmSync(".agent-hive", { recursive: true, force: true });
+
+  console.log("═══════════════════════════════════════════");
+  console.log("  Test A: Queue Stress Test — Evidence");
+  console.log("═══════════════════════════════════════════\n");
+
+  const broker = new Broker();
+  await broker.registerAdapter(new FastMockExecutor());
+  await broker.registerAdapter(new FastMockReviewer());
+  broker.registry.link("codex", "claude");
+
+  // ── Submit 100 tasks ──
+  console.log("  [Phase 1] Submitting 100 tasks...\n");
+  const submitStart = Date.now();
+  for (let i = 1; i <= 100; i++) {
+    broker.submit({
+      instruction: `Create file_${i}.js with content: module.exports = ${i}`,
+      executor: "codex",
+      reviewer: "claude",
+    });
+  }
+  const submitTime = Date.now() - submitStart;
+
+  // Show sample tasks
+  const allTasks = broker.listTasks();
+  console.log(`  Submitted: ${allTasks.length} tasks in ${submitTime}ms`);
+  console.log(`  Sample tasks:`);
+  for (const t of allTasks.slice(0, 5)) {
+    console.log(`    ${t.id}: "${t.instruction}" → ${t.status}`);
+  }
+  console.log(`    ... (${allTasks.length - 5} more)\n`);
+
+  // ── Process all ──
+  console.log("  [Phase 2] Processing queue...\n");
+  const runStart = Date.now();
+  await broker.run();
+  const runTime = Date.now() - runStart;
+
+  // ── Collect evidence ── (re-read from disk after processing)
+  const allTasksAfter = broker.listTasks();
+  const completed = allTasksAfter.filter(t => t.status === "COMPLETED");
+  const failed = allTasksAfter.filter(t => t.status === "FAILED");
+  const pending = allTasksAfter.filter(t => t.status === "PENDING");
+  const other = allTasksAfter.filter(t => !["COMPLETED", "FAILED", "PENDING"].includes(t.status));
+
+  // Check for retries (revisionCount > 0)
+  const retried = allTasksAfter.filter(t => t.revisionCount > 0);
+
+  // Verify each completed task has a result
+  const withResult = completed.filter(t => t.result);
+  const withoutResult = completed.filter(t => !t.result);
+
+  console.log("  [Evidence]");
+  console.log(`  ────────────────────────────────────`);
+  console.log(`  Total submitted:    ${allTasks.length}`);
+  console.log(`  COMPLETED:          ${completed.length}`);
+  console.log(`  FAILED:             ${failed.length}`);
+  console.log(`  PENDING:            ${pending.length}`);
+  console.log(`  Other:              ${other.length}`);
+  console.log(`  ────────────────────────────────────`);
+  console.log(`  With result:        ${withResult.length}`);
+  console.log(`  Without result:     ${withoutResult.length}`);
+  console.log(`  Retried (rev>0):    ${retried.length}`);
+  console.log(`  ────────────────────────────────────`);
+  console.log(`  Submit time:        ${submitTime}ms`);
+  console.log(`  Processing time:    ${runTime}ms`);
+  console.log(`  Total time:         ${submitTime + runTime}ms`);
+  console.log(`  Avg per task:       ${Math.round(runTime / allTasks.length)}ms`);
+  console.log(`  ────────────────────────────────────`);
+
+  // Show sample completed tasks
+  console.log(`\n  Sample completed tasks:`);
+  for (const t of completed.slice(0, 5)) {
+    console.log(`    ${t.id}: status=${t.status}, revCount=${t.revisionCount}, result="${(t.result || "").slice(0, 40)}"`);
+  }
+
+  // Check for state confusion (no task should be in multiple states)
+  const stateMap: Record<string, number> = {};
+  for (const t of allTasksAfter) {
+    stateMap[t.status] = (stateMap[t.status] || 0) + 1;
+  }
+  console.log(`\n  State distribution: ${JSON.stringify(stateMap)}`);
+
+  // Verify no duplicates
+  const ids = allTasksAfter.map(t => t.id);
+  const uniqueIds = new Set(ids);
+  console.log(`  Unique IDs: ${uniqueIds.size} / ${ids.length} ${uniqueIds.size === ids.length ? "✓" : "✗ DUPLICATE!"}`);
+
+  // Final verdict
+  const pass = completed.length === 100 && failed.length === 0 && pending.length === 0
+    && other.length === 0 && retried.length === 0 && uniqueIds.size === 100;
+  console.log(`\n  ═══ Test A: ${pass ? "PASS ✓" : "FAIL ✗"} ═══\n`);
+}
+
+main().catch(err => { console.error("Fatal:", err); process.exit(1); });

package/src/commands/test-timeout.ts

@@ -0,0 +1,145 @@
+/**
+ * Test C: Runtime Timeout — Detailed Evidence
+ */
+import { Broker } from "../broker/Broker";
+import { AgentAdapter, AgentResult, ReviewResult, Task } from "../types";
+
+class SlowExecutorAdapter implements AgentAdapter {
+  name = "slow-executor";
+  role = "developer" as const;
+  capabilities = ["coding"];
+  private delay: number;
+  constructor(delay: number) { this.delay = delay; }
+  async detect() { return true; }
+  async health() { return true; }
+  async execute(): Promise<AgentResult> {
+    await this.sleep(this.delay);
+    return { success: true, output: "slow result" };
+  }
+  private sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); }
+}
+
+class SlowReviewerAdapter implements AgentAdapter {
+  name = "slow-reviewer";
+  role = "reviewer" as const;
+  capabilities = ["review"];
+  private delay: number;
+  constructor(delay: number) { this.delay = delay; }
+  async detect() { return true; }
+  async health() { return true; }
+  async execute(): Promise<AgentResult> { return { success: true, output: "" }; }
+  async review(): Promise<ReviewResult> {
+    await this.sleep(this.delay);
+    return { decision: "PASS", score: 90, issues: [] };
+  }
+  private sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); }
+}
+
+async function main() {
+  console.log("═══════════════════════════════════════════");
+  console.log("  Test C: Runtime Timeout — Evidence");
+  console.log("═══════════════════════════════════════════\n");
+
+  const results: { name: string; pass: boolean; detail: string }[] = [];
+
+  // ── C1: Executor timeout ──
+  console.log("  ── C1: Executor Timeout ──");
+  console.log("  Config: executor_delay=10000ms, task_timeout=2000ms");
+  {
+    const broker = new Broker();
+    await broker.registerAdapter(new SlowExecutorAdapter(10_000));
+    await broker.registerAdapter(new SlowReviewerAdapter(0));
+    broker.registry.link("slow-executor", "slow-reviewer");
+
+    const task = broker.submit({
+      instruction: "test executor timeout",
+      executor: "slow-executor",
+      reviewer: "slow-reviewer",
+      timeout: 2000,
+    });
+    console.log(`  Task: ${task.id}, timeout=${task.timeout}ms`);
+
+    const start = Date.now();
+    await broker.run();
+    const elapsed = Date.now() - start;
+
+    const t = broker.listTasks()[0];
+    console.log(`  Elapsed: ${elapsed}ms (should be ~2000ms, not 10000ms)`);
+    console.log(`  Final status: ${t.status}`);
+    console.log(`  State: PENDING → EXECUTING → TIMEOUT → ${t.status}`);
+
+    const pass = t.status === "FAILED" && elapsed < 5000;
+    results.push({ name: "C1 Executor Timeout", pass, detail: `status=${t.status}, elapsed=${elapsed}ms` });
+    console.log(`  Result: ${pass ? "PASS ✓" : "FAIL ✗"}\n`);
+  }
+
+  // ── C2: Reviewer timeout ──
+  console.log("  ── C2: Reviewer Timeout ──");
+  console.log("  Config: reviewer_delay=10000ms, task_timeout=2000ms");
+  {
+    const broker = new Broker();
+    await broker.registerAdapter(new SlowExecutorAdapter(0));
+    await broker.registerAdapter(new SlowReviewerAdapter(10_000));
+    broker.registry.link("slow-executor", "slow-reviewer");
+
+    const task = broker.submit({
+      instruction: "test reviewer timeout",
+      executor: "slow-executor",
+      reviewer: "slow-reviewer",
+      timeout: 2000,
+    });
+    console.log(`  Task: ${task.id}, timeout=${task.timeout}ms`);
+
+    const start = Date.now();
+    await broker.run();
+    const elapsed = Date.now() - start;
+
+    const t = broker.listTasks()[0];
+    console.log(`  Elapsed: ${elapsed}ms`);
+    console.log(`  Final status: ${t.status}`);
+    console.log(`  State: PENDING → EXECUTING → REVIEWING → ERROR → ${t.status}`);
+
+    const pass = t.status === "FAILED" && elapsed < 5000;
+    results.push({ name: "C2 Reviewer Timeout", pass, detail: `status=${t.status}, elapsed=${elapsed}ms` });
+    console.log(`  Result: ${pass ? "PASS ✓" : "FAIL ✗"}\n`);
+  }
+
+  // ── C3: Both timeout ──
+  console.log("  ── C3: Both Timeout ──");
+  console.log("  Config: executor_delay=10000ms, reviewer_delay=10000ms, task_timeout=1500ms");
+  {
+    const broker = new Broker();
+    await broker.registerAdapter(new SlowExecutorAdapter(10_000));
+    await broker.registerAdapter(new SlowReviewerAdapter(10_000));
+    broker.registry.link("slow-executor", "slow-reviewer");
+
+    const task = broker.submit({
+      instruction: "test both timeout",
+      executor: "slow-executor",
+      reviewer: "slow-reviewer",
+      timeout: 1500,
+    });
+
+    const start = Date.now();
+    await broker.run();
+    const elapsed = Date.now() - start;
+
+    const t = broker.listTasks()[0];
+    console.log(`  Elapsed: ${elapsed}ms`);
+    console.log(`  Final status: ${t.status}`);
+
+    const pass = t.status === "FAILED" && elapsed < 4000;
+    results.push({ name: "C3 Both Timeout", pass, detail: `status=${t.status}, elapsed=${elapsed}ms` });
+    console.log(`  Result: ${pass ? "PASS ✓" : "FAIL ✗"}\n`);
+  }
+
+  // ── Summary ──
+  console.log("  ═══ Evidence Summary ═══");
+  for (const r of results) {
+    console.log(`    ${r.pass ? "✓" : "✗"} ${r.name}: ${r.detail}`);
+  }
+  const allPass = results.every(r => r.pass);
+  console.log(`\n  ═══ Test C: ${allPass ? "PASS ✓" : "FAIL ✗"} ═══\n`);
+}
+
+main().catch(err => { console.error("Fatal:", err); process.exit(1); });