@toyako/hive 1.0.0

package/src/benchmark/BenchmarkReport.ts

@@ -0,0 +1,147 @@
+import { BenchmarkDataset, RuntimeBenchmark } from "./BenchmarkDataset";
+import { BenchmarkResult } from "./BenchmarkSuite";
+import * as fs from "fs";
+import * as path from "path";
+
+const REPORTS_DIR = path.resolve(process.cwd(), ".agent-hive/reports");
+
+export class BenchmarkReport {
+  private dataset: BenchmarkDataset;
+
+  constructor(dataset: BenchmarkDataset) {
+    this.dataset = dataset;
+    fs.mkdirSync(REPORTS_DIR, { recursive: true });
+  }
+
+  /**
+   * Generate JSON report.
+   */
+  generateJSON(allResults: Map<string, BenchmarkResult[]>): object {
+    const rankings = this.getRankings();
+    const details: Record<string, any> = {};
+
+    for (const [runtime, results] of allResults) {
+      details[runtime] = {
+        benchmark: this.dataset.get(runtime),
+        results: results.map(r => ({
+          caseId: r.caseId,
+          category: r.category,
+          score: r.score,
+          latencyMs: r.latencyMs,
+          success: r.success,
+        })),
+      };
+    }
+
+    const report = {
+      timestamp: new Date().toISOString(),
+      rankings,
+      details,
+    };
+
+    const fp = path.join(REPORTS_DIR, "benchmark-report.json");
+    fs.writeFileSync(fp, JSON.stringify(report, null, 2));
+    return report;
+  }
+
+  /**
+   * Generate Markdown report.
+   */
+  generateMarkdown(): string {
+    const rankings = this.getRankings();
+    const lines: string[] = [];
+
+    lines.push("# Runtime Benchmark Report");
+    lines.push(`> Generated: ${new Date().toISOString()}`);
+    lines.push("");
+
+    // Overall ranking
+    lines.push("## Overall Ranking");
+    lines.push("");
+    lines.push("| Rank | Runtime | Score | Coding | Review | Planning | Reasoning | Latency |");
+    lines.push("|------|---------|-------|--------|--------|----------|-----------|---------|");
+
+    rankings.overall.forEach((r, i) => {
+      const b = this.dataset.get(r.runtimeId)!;
+      lines.push(`| ${i + 1} | ${r.runtimeId} | **${r.score}** | ${b.coding} | ${b.review} | ${b.planning} | ${b.reasoning} | ${b.latency}ms |`);
+    });
+
+    // Per-category rankings
+    for (const category of ["coding", "review", "planning", "reasoning"] as const) {
+      lines.push("");
+      lines.push(`## ${category.charAt(0).toUpperCase() + category.slice(1)} Ranking`);
+      lines.push("");
+      lines.push("| Rank | Runtime | Score |");
+      lines.push("|------|---------|-------|");
+
+      const catRankings = rankings[category];
+      catRankings.forEach((r, i) => {
+        lines.push(`| ${i + 1} | ${r.runtimeId} | ${r.score} |`);
+      });
+    }
+
+    // Best runtime per category
+    lines.push("");
+    lines.push("## Best Runtime per Category");
+    lines.push("");
+    for (const category of ["coding", "review", "planning", "reasoning"] as const) {
+      const best = rankings[category][0];
+      if (best) {
+        lines.push(`- **${category}**: ${best.runtimeId} (${best.score})`);
+      }
+    }
+
+    const md = lines.join("\n");
+    const fp = path.join(REPORTS_DIR, "benchmark-report.md");
+    fs.writeFileSync(fp, md);
+    return md;
+  }
+
+  /**
+   * Get rankings for each category and overall.
+   */
+  getRankings(): {
+    overall: { runtimeId: string; score: number }[];
+    coding: { runtimeId: string; score: number }[];
+    review: { runtimeId: string; score: number }[];
+    planning: { runtimeId: string; score: number }[];
+    reasoning: { runtimeId: string; score: number }[];
+  } {
+    const all = this.dataset.all();
+
+    const rankBy = (getter: (b: RuntimeBenchmark) => number) =>
+      all
+        .map(b => ({ runtimeId: b.runtimeId, score: Math.round(getter(b)) }))
+        .sort((a, b) => b.score - a.score);
+
+    return {
+      overall: rankBy(b => this.dataset.getOverallScore(b.runtimeId)),
+      coding: rankBy(b => b.coding),
+      review: rankBy(b => b.review),
+      planning: rankBy(b => b.planning),
+      reasoning: rankBy(b => b.reasoning),
+    };
+  }
+
+  /**
+   * Format rankings as text table.
+   */
+  formatRankings(): string {
+    const rankings = this.getRankings();
+    const lines: string[] = [];
+
+    lines.push("Runtime Benchmark Rankings:");
+    lines.push("─".repeat(65));
+    lines.push("Rank  Runtime     Overall  Coding  Review  Planning  Reasoning");
+    lines.push("─".repeat(65));
+
+    rankings.overall.forEach((r, i) => {
+      const b = this.dataset.get(r.runtimeId)!;
+      lines.push(
+        `${(i + 1).toString().padStart(4)}  ${r.runtimeId.padEnd(11)} ${r.score.toString().padStart(7)}  ${b.coding.toString().padStart(6)}  ${b.review.toString().padStart(6)}  ${b.planning.toString().padStart(8)}  ${b.reasoning.toString().padStart(9)}`
+      );
+    });
+
+    return lines.join("\n");
+  }
+}

package/src/benchmark/BenchmarkRunner.ts

@@ -0,0 +1,143 @@
+import { AgentAdapter, Task } from "../types";
+import { BenchmarkDataset, RuntimeBenchmark } from "./BenchmarkDataset";
+import { BenchmarkCase, BenchmarkResult, BENCHMARK_CASES } from "./BenchmarkSuite";
+import { TaskTimeline } from "../observability/TaskTimeline";
+import { RuntimeMetrics } from "../observability/RuntimeMetrics";
+import { DecisionRecorder } from "../observability/DecisionRecorder";
+
+export class BenchmarkRunner {
+  private dataset: BenchmarkDataset;
+  private timeline: TaskTimeline;
+  private metrics: RuntimeMetrics;
+  private decisions: DecisionRecorder;
+
+  constructor(
+    dataset: BenchmarkDataset,
+    timeline: TaskTimeline,
+    metrics: RuntimeMetrics,
+    decisions: DecisionRecorder
+  ) {
+    this.dataset = dataset;
+    this.timeline = timeline;
+    this.metrics = metrics;
+    this.decisions = decisions;
+  }
+
+  /**
+   * Run all benchmark cases against a single runtime.
+   */
+  async runRuntime(adapter: AgentAdapter, cases?: BenchmarkCase[]): Promise<BenchmarkResult[]> {
+    const testCases = cases || BENCHMARK_CASES;
+    const results: BenchmarkResult[] = [];
+
+    for (const bc of testCases) {
+      const result = await this.runCase(adapter, bc);
+      results.push(result);
+    }
+
+    return results;
+  }
+
+  /**
+   * Run all benchmark cases against all provided runtimes.
+   */
+  async runAll(adapters: AgentAdapter[]): Promise<Map<string, BenchmarkResult[]>> {
+    const allResults = new Map<string, BenchmarkResult[]>();
+
+    for (const adapter of adapters) {
+      const results = await this.runRuntime(adapter);
+      allResults.set(adapter.name, results);
+    }
+
+    return allResults;
+  }
+
+  /**
+   * Run a single benchmark case against a runtime.
+   */
+  async runCase(adapter: AgentAdapter, bc: BenchmarkCase): Promise<BenchmarkResult> {
+    const taskId = `bench-${adapter.name}-${bc.id}`;
+    this.timeline.start(taskId);
+    this.timeline.event(taskId, "benchmark-start", { runtime: adapter.name, case: bc.id });
+
+    const task: Task = {
+      id: taskId,
+      instruction: bc.instruction,
+      executor: adapter.name,
+      reviewer: "",
+      status: "EXECUTING",
+      revisionCount: 0,
+      maxRevision: 1,
+      timeout: 60_000,
+      workingDirectory: process.cwd(),
+      createdAt: Date.now(),
+      updatedAt: Date.now(),
+    };
+
+    const start = Date.now();
+    let success = false;
+    let output = "";
+    let score = 0;
+
+    try {
+      const result = await adapter.execute(task);
+      success = result.success;
+      output = result.output || "";
+
+      if (success) {
+        score = bc.evaluate(output);
+      }
+    } catch (err: any) {
+      output = err.message;
+    }
+
+    const latencyMs = Date.now() - start;
+
+    this.timeline.event(taskId, "benchmark-done", { success, score, latencyMs });
+    this.timeline.complete(taskId, success ? "COMPLETED" : "FAILED");
+
+    this.metrics.record(adapter.name, { success, latencyMs, reviewScore: score });
+
+    return {
+      caseId: bc.id,
+      category: bc.category,
+      runtimeId: adapter.name,
+      success,
+      score,
+      latencyMs,
+      output,
+    };
+  }
+
+  /**
+   * Aggregate results into a RuntimeBenchmark.
+   */
+  aggregate(runtimeId: string, results: BenchmarkResult[]): RuntimeBenchmark {
+    const byCategory = (cat: string) => results.filter(r => r.category === cat);
+
+    const avgScore = (items: BenchmarkResult[]) =>
+      items.length > 0 ? items.reduce((s, r) => s + r.score, 0) / items.length : 0;
+
+    const avgLatency = results.length > 0
+      ? results.reduce((s, r) => s + r.latencyMs, 0) / results.length
+      : 0;
+
+    const successRate = results.length > 0
+      ? results.filter(r => r.success).length / results.length
+      : 0;
+
+    const benchmark: RuntimeBenchmark = {
+      runtimeId,
+      coding: Math.round(avgScore(byCategory("coding"))),
+      review: Math.round(avgScore(byCategory("review"))),
+      planning: Math.round(avgScore(byCategory("planning"))),
+      reasoning: Math.round(avgScore(byCategory("reasoning"))),
+      latency: Math.round(avgLatency),
+      successRate: Math.round(successRate * 100) / 100,
+      updatedAt: Date.now(),
+    };
+
+    this.dataset.set(benchmark);
+    return benchmark;
+  }
+}

package/src/benchmark/BenchmarkSuite.ts

@@ -0,0 +1,193 @@
+import { AgentAdapter, Task } from "../types";
+
+export interface BenchmarkCase {
+  id: string;
+  category: "coding" | "review" | "planning" | "reasoning";
+  name: string;
+  instruction: string;
+  evaluate: (output: string) => number; // 0-100 score
+}
+
+export interface BenchmarkResult {
+  caseId: string;
+  category: string;
+  runtimeId: string;
+  success: boolean;
+  score: number;
+  latencyMs: number;
+  output: string;
+}
+
+/**
+ * Benchmark Suite — predefined test cases for each category.
+ */
+export const BENCHMARK_CASES: BenchmarkCase[] = [
+  // ── Category A: Coding ──
+  {
+    id: "code-01",
+    category: "coding",
+    name: "Hello World",
+    instruction: "Write a JavaScript function called greet that takes a name and returns 'Hello, {name}!'. Return ONLY the code.",
+    evaluate: (output) => {
+      const hasFunction = output.includes("function greet") || output.includes("greet =") || output.includes("const greet");
+      const hasReturn = output.includes("return") || output.includes("=>");
+      const hasHello = output.includes("Hello");
+      return (hasFunction ? 40 : 0) + (hasReturn ? 30 : 0) + (hasHello ? 30 : 0);
+    },
+  },
+  {
+    id: "code-02",
+    category: "coding",
+    name: "CRUD API",
+    instruction: "Design a REST API schema for a User resource with fields: id, name, email. Return as JSON.",
+    evaluate: (output) => {
+      const hasId = output.includes("id");
+      const hasName = output.includes("name");
+      const hasEmail = output.includes("email");
+      const isJson = output.includes("{") && output.includes("}");
+      return (hasId ? 25 : 0) + (hasName ? 25 : 0) + (hasEmail ? 25 : 0) + (isJson ? 25 : 0);
+    },
+  },
+  {
+    id: "code-03",
+    category: "coding",
+    name: "Refactor",
+    instruction: "Refactor this code to use async/await: function getData() { return fetch('/api').then(r => r.json()).then(d => d.items); }",
+    evaluate: (output) => {
+      const hasAsync = output.includes("async");
+      const hasAwait = output.includes("await");
+      const noThen = !output.includes(".then(");
+      return (hasAsync ? 35 : 0) + (hasAwait ? 35 : 0) + (noThen ? 30 : 0);
+    },
+  },
+  {
+    id: "code-04",
+    category: "coding",
+    name: "Bug Fix",
+    instruction: "Fix this bug: const sum = (a, b) => a + b; console.log(sum(1, '2')); What's wrong and how to fix it?",
+    evaluate: (output) => {
+      const identifiesType = output.includes("type") || output.includes("string") || output.includes("concatenat");
+      const hasFix = output.includes("Number") || output.includes("parseInt") || output.includes("parseFloat") || output.includes("+") ;
+      const hasExplanation = output.length > 50;
+      return (identifiesType ? 40 : 0) + (hasFix ? 35 : 0) + (hasExplanation ? 25 : 0);
+    },
+  },
+
+  // ── Category B: Review ──
+  {
+    id: "review-01",
+    category: "review",
+    name: "Security Review",
+    instruction: "Review this code for security issues: app.get('/user', (req, res) => { const id = req.query.id; db.query('SELECT * FROM users WHERE id=' + id); });",
+    evaluate: (output) => {
+      const identifiesSQLi = output.toLowerCase().includes("sql injection") || output.includes("SQL");
+      const identifiesSanitize = output.includes("parameterized") || output.includes("prepared") || output.includes("sanitize") || output.includes("escape");
+      const hasRecommendation = output.length > 50;
+      return (identifiesSQLi ? 45 : 0) + (identifiesSanitize ? 35 : 0) + (hasRecommendation ? 20 : 0);
+    },
+  },
+  {
+    id: "review-02",
+    category: "review",
+    name: "Architecture Review",
+    instruction: "Review this architecture: Monolith with 500 tables, single database, no caching, synchronous processing. What are the risks?",
+    evaluate: (output) => {
+      const identifiesScale = output.toLowerCase().includes("scal") || output.includes("bottleneck");
+      const identifiesSPOF = output.toLowerCase().includes("single point") || output.toLowerCase().includes("spof") || output.includes("availability");
+      const hasSuggestions = output.includes("cache") || output.includes("microservice") || output.includes("async") || output.includes("queue");
+      return (identifiesScale ? 35 : 0) + (identifiesSPOF ? 35 : 0) + (hasSuggestions ? 30 : 0);
+    },
+  },
+  {
+    id: "review-03",
+    category: "review",
+    name: "Code Review",
+    instruction: "Review this function: function divide(a, b) { return a / b; } What's missing?",
+    evaluate: (output) => {
+      const identifiesDivisionByZero = output.includes("zero") || output.includes("0") || output.includes("division");
+      const hasInputValidation = output.includes("check") || output.includes("valid") || output.includes("handle") || output.includes("throw");
+      const hasErrorHandling = output.includes("error") || output.includes("exception") || output.includes("NaN") || output.includes("Infinity");
+      return (identifiesDivisionByZero ? 40 : 0) + (hasInputValidation ? 30 : 0) + (hasErrorHandling ? 30 : 0);
+    },
+  },
+
+  // ── Category C: Planning ──
+  {
+    id: "plan-01",
+    category: "planning",
+    name: "System Design",
+    instruction: "Design a URL shortener system. List the main components and data flow.",
+    evaluate: (output) => {
+      const hasComponents = output.includes("database") || output.includes("cache") || output.includes("API");
+      const hasFlow = output.includes("redirect") || output.includes("shorten") || output.includes("generate");
+      const hasStorage = output.includes("store") || output.includes("mapping") || output.includes("hash");
+      const depth = Math.min(output.length / 20, 30);
+      return (hasComponents ? 25 : 0) + (hasFlow ? 25 : 0) + (hasStorage ? 20 : 0) + depth;
+    },
+  },
+  {
+    id: "plan-02",
+    category: "planning",
+    name: "Agent Design",
+    instruction: "Design a multi-agent system for automated code review. What agents do you need and how do they collaborate?",
+    evaluate: (output) => {
+      const hasMultipleAgents = (output.match(/agent/gi) || []).length >= 2;
+      const hasCollaboration = output.includes("collaborat") || output.includes("communicat") || output.includes("pass") || output.includes("share");
+      const hasRoles = output.includes("reviewer") || output.includes("executor") || output.includes("planner") || output.includes("tester");
+      const depth = Math.min(output.length / 20, 30);
+      return (hasMultipleAgents ? 25 : 0) + (hasCollaboration ? 25 : 0) + (hasRoles ? 20 : 0) + depth;
+    },
+  },
+  {
+    id: "plan-03",
+    category: "planning",
+    name: "Workflow Design",
+    instruction: "Design a CI/CD pipeline for a Node.js project with testing, linting, and deployment stages.",
+    evaluate: (output) => {
+      const hasTest = output.includes("test") || output.includes("jest") || output.includes("mocha");
+      const hasLint = output.includes("lint") || output.includes("eslint") || output.includes("prettier");
+      const hasDeploy = output.includes("deploy") || output.includes("production") || output.includes("release");
+      const hasStages = output.includes("stage") || output.includes("step") || output.includes("phase") || output.includes("job");
+      return (hasTest ? 25 : 0) + (hasLint ? 25 : 0) + (hasDeploy ? 25 : 0) + (hasStages ? 25 : 0);
+    },
+  },
+
+  // ── Category D: Reasoning ──
+  {
+    id: "reason-01",
+    category: "reasoning",
+    name: "Escalation Decision",
+    instruction: "A junior developer's code has been rejected 3 times. The issues are: missing error handling, no tests, inconsistent naming. What should happen next?",
+    evaluate: (output) => {
+      const hasEscalation = output.includes("escalat") || output.includes("senior") || output.includes("mentor") || output.includes("pair");
+      const hasPrioritization = output.includes("priorit") || output.includes("first") || output.includes("critical");
+      const hasAction = output.includes("review") || output.includes("guid") || output.includes("train") || output.includes("help");
+      return (hasEscalation ? 40 : 0) + (hasPrioritization ? 30 : 0) + (hasAction ? 30 : 0);
+    },
+  },
+  {
+    id: "reason-02",
+    category: "reasoning",
+    name: "Multi-step Task",
+    instruction: "You need to migrate a database from MySQL to PostgreSQL. List the steps in order.",
+    evaluate: (output) => {
+      const hasSchema = output.includes("schema") || output.includes("create table") || output.includes("DDL");
+      const hasData = output.includes("data") || output.includes("migrat") || output.includes("export") || output.includes("import");
+      const hasTest = output.includes("test") || output.includes("verify") || output.includes("validat");
+      const hasRollback = output.includes("rollback") || output.includes("backup") || output.includes("revert");
+      return (hasSchema ? 25 : 0) + (hasData ? 25 : 0) + (hasTest ? 25 : 0) + (hasRollback ? 25 : 0);
+    },
+  },
+  {
+    id: "reason-03",
+    category: "reasoning",
+    name: "Conflict Resolution",
+    instruction: "Two microservices disagree on the data format for a shared event. Service A wants XML, Service B wants JSON. How do you resolve this?",
+    evaluate: (output) => {
+      const hasAdapter = output.includes("adapter") || output.includes("transform") || output.includes("convert") || output.includes("translator");
+      const hasStandard = output.includes("standard") || output.includes("common") || output.includes("agree") || output.includes("contract");
+      const hasEventDriven = output.includes("event") || output.includes("message") || output.includes("queue") || output.includes("broker");
+      return (hasAdapter ? 35 : 0) + (hasStandard ? 35 : 0) + (hasEventDriven ? 30 : 0);
+    },
+  },
+];

package/src/benchmark/index.ts

@@ -0,0 +1,39 @@
+/**
+ * Execution Runtime Freeze + Benchmark Execution Plan v1.0
+ * 
+ * 冻结架构 + 建立系统性能基线 + 验证真实运行稳定性
+ * 
+ * 系统从工程完成 → 变成可验证产品
+ */
+
+// Phase 1: Freeze Architecture
+export {
+  FreezeConfig,
+  ArchitectureFreeze
+} from "./BenchmarkPlan";
+
+// Phase 2: Benchmark Suite
+export {
+  BenchmarkLevel,
+  BenchmarkResult,
+  BenchmarkSuite
+} from "./BenchmarkPlan";
+
+// Phase 3: Load Profiles
+export {
+  LoadProfile,
+  LoadConfig,
+  LoadGenerator
+} from "./BenchmarkPlan";
+
+// Phase 4: Metrics Definition
+export {
+  SystemMetrics,
+  MetricsCollector
+} from "./BenchmarkPlan";
+
+// Phase 5: Production Readiness Report
+export {
+  BenchmarkReport,
+  ProductionReadinessReport
+} from "./BenchmarkPlan";

package/src/broker/AgentRegistry.ts

@@ -0,0 +1,76 @@
+import { AgentAdapter, RegistryEntry } from "../types";
+import * as fs from "fs";
+import * as path from "path";
+
+const REGISTRY_FILE = path.resolve(process.cwd(), ".agent-hive/agents.json");
+
+export class AgentRegistry {
+  private adapters: Map<string, AgentAdapter> = new Map();
+  private entries: Map<string, RegistryEntry> = new Map();
+
+  constructor() {
+    this.loadPersisted();
+  }
+
+  // register: 只记录 adapter 和 installed 状态
+  // 不调用 health() — health check 由 Broker 显式触发
+  async register(adapter: AgentAdapter, installed?: boolean): Promise<RegistryEntry> {
+    const isInstalled = installed ?? await adapter.detect();
+
+    const entry: RegistryEntry = {
+      name: adapter.name,
+      role: adapter.role,
+      capabilities: adapter.capabilities,
+      installed: isInstalled,
+      healthy: isInstalled, // 默认与 installed 同，health check 会更新
+    };
+
+    this.adapters.set(adapter.name, adapter);
+    this.entries.set(adapter.name, entry);
+    this.persist();
+    return entry;
+  }
+
+  getAdapter(name: string): AgentAdapter | undefined {
+    return this.adapters.get(name);
+  }
+
+  getEntry(name: string): RegistryEntry | undefined {
+    return this.entries.get(name);
+  }
+
+  all(): RegistryEntry[] {
+    return Array.from(this.entries.values());
+  }
+
+  healthy(): RegistryEntry[] {
+    return this.all().filter(e => e.healthy);
+  }
+
+  link(from: string, to: string): void {
+    const entry = this.entries.get(from);
+    if (entry) {
+      entry.reportsTo = to;
+      this.persist();
+    }
+  }
+
+  private persist(): void {
+    const data: Record<string, RegistryEntry> = {};
+    for (const [k, v] of this.entries) data[k] = v;
+    fs.mkdirSync(path.dirname(REGISTRY_FILE), { recursive: true });
+    fs.writeFileSync(REGISTRY_FILE, JSON.stringify(data, null, 2));
+  }
+
+  private loadPersisted(): void {
+    if (!fs.existsSync(REGISTRY_FILE)) return;
+    try {
+      const data = JSON.parse(fs.readFileSync(REGISTRY_FILE, "utf-8"));
+      for (const [k, v] of Object.entries(data)) {
+        this.entries.set(k, v as RegistryEntry);
+      }
+    } catch (e) {
+      console.warn("[AgentRegistry] Failed to load persisted registry:", e instanceof Error ? e.message : e);
+    }
+  }
+}