opencode-swarm-plugin 0.20.0 → 0.21.0

package/evals/swarm-decomposition.eval.ts

@@ -0,0 +1,112 @@
+/**
+ * Swarm Decomposition Quality Eval
+ *
+ * Tests the quality of task decomposition for swarm coordination.
+ * Uses real LLM calls via AI SDK + Vercel AI Gateway.
+ *
+ * Scorers evaluate:
+ * - Subtask independence (no file conflicts)
+ * - Complexity balance (even distribution)
+ * - Coverage completeness (all required files)
+ * - Instruction clarity (actionable descriptions)
+ *
+ * Run with: pnpm evalite evals/swarm-decomposition.eval.ts
+ *
+ * Requires: ANTHROPIC_API_KEY environment variable
+ */
+import { evalite } from "evalite";
+import {
+  subtaskIndependence,
+  coverageCompleteness,
+  instructionClarity,
+} from "./scorers/index.js";
+import { decompositionCases } from "./fixtures/decomposition-cases.js";
+import {
+  generateDecomposition,
+  formatDecompositionPrompt,
+  extractJson,
+} from "./lib/llm.js";
+import {
+  loadEvalCases,
+  hasRealEvalData,
+  getEvalDataSummary,
+} from "./lib/data-loader.js";
+
+// Determine project key from current directory
+const PROJECT_KEY = "opencode-swarm-plugin";
+const PROJECT_PATH = process.cwd();
+
+// Check if we have enough real data to use instead of fixtures
+const useRealData = await hasRealEvalData(PROJECT_KEY, 5, PROJECT_PATH);
+
+// Load data based on availability
+const evalCases = useRealData
+  ? await loadEvalCases(PROJECT_KEY, { limit: 20, projectPath: PROJECT_PATH })
+  : decompositionCases.map((testCase) => ({
+      input: testCase.input,
+      expected: testCase.expected,
+    }));
+
+// Log data source for transparency
+if (useRealData) {
+  const summary = await getEvalDataSummary(PROJECT_KEY, PROJECT_PATH);
+  console.log(`[eval] Using real data from PGlite:`);
+  console.log(`  - Total records: ${summary.totalRecords}`);
+  console.log(`  - Success rate: ${(summary.successRate * 100).toFixed(1)}%`);
+  console.log(
+    `  - Strategies: ${Object.entries(summary.byStrategy)
+      .map(([s, c]) => `${s}(${c})`)
+      .join(", ")}`,
+  );
+  console.log(`  - Eval cases: ${evalCases.length}`);
+} else {
+  console.log(
+    `[eval] Using fixture data (${evalCases.length} cases) - not enough real data yet`,
+  );
+}
+
+/**
+ * Swarm Decomposition Quality Eval
+ *
+ * Tests decomposition quality with real LLM calls.
+ */
+evalite("Swarm Decomposition Quality", {
+  // Test data from PGlite or fixtures
+  data: async () => evalCases,
+
+  // Task: generate real decomposition via Claude
+  task: async (input) => {
+    const prompt = formatDecompositionPrompt(input.task, input.context);
+    const response = await generateDecomposition(prompt);
+    return extractJson(response);
+  },
+
+  // Scorers evaluate decomposition quality
+  scorers: [subtaskIndependence, coverageCompleteness, instructionClarity],
+});
+
+/**
+ * Edge Case Eval: Minimal and Complex Tasks
+ *
+ * Tests handling of edge cases in decomposition.
+ */
+evalite("Decomposition Edge Cases", {
+  data: async () => [
+    {
+      input: { task: "Fix typo in README.md" },
+      expected: { minSubtasks: 1, maxSubtasks: 2 },
+    },
+    {
+      input: { task: "Refactor entire codebase from JavaScript to TypeScript" },
+      expected: { minSubtasks: 4, maxSubtasks: 8 },
+    },
+  ],
+
+  task: async (input) => {
+    const prompt = formatDecompositionPrompt(input.task, undefined, 8);
+    const response = await generateDecomposition(prompt);
+    return extractJson(response);
+  },
+
+  scorers: [subtaskIndependence, coverageCompleteness],
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-swarm-plugin",
-  "version": "0.20.0",
+  "version": "0.21.0",
   "description": "Multi-agent swarm coordination for OpenCode with learning capabilities, beads integration, and Agent Mail",
   "type": "module",
   "main": "dist/index.js",
@@ -27,6 +27,9 @@
     "test:all": "bun run test && bun run test:swarm",
     "typecheck": "tsc --noEmit",
     "clean": "rm -rf dist",
+    "eval:dev": "evalite watch evals/",
+    "eval:run": "evalite run evals/",
+    "eval:ci": "evalite run evals/ --threshold 80",
     "release": "npm run build && npm version patch && git push && npm run publish:otp",
     "release:minor": "npm run build && npm version minor && git push && npm run publish:otp",
     "release:major": "npm run build && npm version major && git push && npm run publish:otp",
@@ -41,11 +44,15 @@
     "gray-matter": "^4.0.3",
     "ioredis": "^5.4.1",
     "minimatch": "^10.1.1",
+    "nanoid": "^5.1.6",
     "zod": "4.1.8"
   },
   "devDependencies": {
     "@types/bun": "latest",
     "@types/minimatch": "^6.0.0",
+    "ai": "6.0.0-beta.150",
+    "bun-types": "^1.3.4",
+    "evalite": "^1.0.0-beta.10",
     "typescript": "^5.7.0",
     "vitest": "^4.0.15"
   },

package/src/beads.ts

@@ -104,6 +104,8 @@ import {
   type BeadCreateArgs,
   type EpicCreateResult,
 } from "./schemas";
+import { createEvent } from "./streams/events";
+import { appendEvent } from "./streams/store";
 
 /**
  * Custom error for bead operations
@@ -321,6 +323,26 @@ export const beads_create_epic = tool({
         }),
       )
       .describe("Subtasks to create under the epic"),
+    strategy: tool.schema
+      .enum(["file-based", "feature-based", "risk-based"])
+      .optional()
+      .describe("Decomposition strategy used (default: feature-based)"),
+    task: tool.schema
+      .string()
+      .optional()
+      .describe("Original task description that was decomposed"),
+    project_key: tool.schema
+      .string()
+      .optional()
+      .describe("Project path for event emission"),
+    recovery_context: tool.schema
+      .object({
+        shared_context: tool.schema.string().optional(),
+        skills_to_load: tool.schema.array(tool.schema.string()).optional(),
+        coordinator_notes: tool.schema.string().optional(),
+      })
+      .optional()
+      .describe("Recovery context from checkpoint compaction"),
   },
   async execute(args, ctx) {
     const validated = EpicCreateArgsSchema.parse(args);
@@ -386,6 +408,33 @@ export const beads_create_epic = tool({
         subtasks: created.slice(1),
       };
 
+      // Emit DecompositionGeneratedEvent for learning system
+      if (args.project_key) {
+        try {
+          const event = createEvent("decomposition_generated", {
+            project_key: args.project_key,
+            epic_id: epic.id,
+            task: args.task || validated.epic_title,
+            context: validated.epic_description,
+            strategy: args.strategy || "feature-based",
+            epic_title: validated.epic_title,
+            subtasks: validated.subtasks.map((st) => ({
+              title: st.title,
+              files: st.files || [],
+              priority: st.priority,
+            })),
+            recovery_context: args.recovery_context,
+          });
+          await appendEvent(event, args.project_key);
+        } catch (error) {
+          // Non-fatal - log and continue
+          console.warn(
+            "[beads_create_epic] Failed to emit DecompositionGeneratedEvent:",
+            error,
+          );
+        }
+      }
+
       return JSON.stringify(result, null, 2);
     } catch (error) {
       // Partial failure - execute rollback automatically

package/src/eval-capture.ts

@@ -0,0 +1,487 @@
+/**
+ * Eval Data Capture - Captures real swarm execution data for evals
+ *
+ * Records decomposition inputs, outputs, and outcomes to JSONL files
+ * that can be used as ground truth for Evalite evals.
+ *
+ * Data flow:
+ * 1. swarm_decompose captures: task, context, generated decomposition
+ * 2. swarm_complete captures: outcome signals per subtask
+ * 3. swarm_record_outcome captures: learning signals
+ * 4. Human feedback (optional): accept/reject/modify
+ *
+ * @module eval-capture
+ */
+import { z } from "zod";
+import * as fs from "fs";
+import * as path from "path";
+
+// ============================================================================
+// Schemas
+// ============================================================================
+
+/**
+ * Subtask outcome - what actually happened
+ */
+export const SubtaskOutcomeSchema = z.object({
+  /** Subtask bead ID */
+  bead_id: z.string(),
+  /** Subtask title */
+  title: z.string(),
+  /** Planned files */
+  planned_files: z.array(z.string()),
+  /** Actual files touched */
+  actual_files: z.array(z.string()),
+  /** Duration in ms */
+  duration_ms: z.number().int().min(0),
+  /** Error count */
+  error_count: z.number().int().min(0),
+  /** Retry count */
+  retry_count: z.number().int().min(0),
+  /** Success */
+  success: z.boolean(),
+  /** Failure mode if failed */
+  failure_mode: z.string().optional(),
+});
+export type SubtaskOutcome = z.infer<typeof SubtaskOutcomeSchema>;
+
+/**
+ * Complete eval record - input, output, and outcome
+ */
+export const EvalRecordSchema = z.object({
+  /** Unique ID for this eval record */
+  id: z.string(),
+  /** Timestamp when decomposition was generated */
+  timestamp: z.string(), // ISO-8601
+  /** Project path */
+  project_path: z.string(),
+
+  // INPUT
+  /** Original task description */
+  task: z.string(),
+  /** Context provided (codebase info, CASS results, etc.) */
+  context: z.string().optional(),
+  /** Strategy used for decomposition */
+  strategy: z.enum(["file-based", "feature-based", "risk-based", "auto"]),
+  /** Max subtasks requested */
+  max_subtasks: z.number().int().min(1).max(10),
+
+  // OUTPUT (the decomposition)
+  /** Epic title */
+  epic_title: z.string(),
+  /** Epic description */
+  epic_description: z.string().optional(),
+  /** Generated subtasks */
+  subtasks: z.array(
+    z.object({
+      title: z.string(),
+      description: z.string().optional(),
+      files: z.array(z.string()),
+      dependencies: z.array(z.number()).optional(),
+      estimated_complexity: z.number().int().min(1).max(5).optional(),
+    }),
+  ),
+
+  // OUTCOME (what actually happened)
+  /** Subtask outcomes */
+  outcomes: z.array(SubtaskOutcomeSchema).optional(),
+  /** Overall success (all subtasks succeeded) */
+  overall_success: z.boolean().optional(),
+  /** Total duration (sum of all subtasks) */
+  total_duration_ms: z.number().int().min(0).optional(),
+  /** Total errors across all subtasks */
+  total_errors: z.number().int().min(0).optional(),
+
+  // HUMAN FEEDBACK (optional)
+  /** Human accepted the decomposition as-is */
+  human_accepted: z.boolean().optional(),
+  /** Human modified the decomposition */
+  human_modified: z.boolean().optional(),
+  /** Human feedback notes */
+  human_notes: z.string().optional(),
+
+  // COMPUTED METRICS
+  /** File overlap between subtasks (should be 0) */
+  file_overlap_count: z.number().int().min(0).optional(),
+  /** Scope accuracy: actual files / planned files */
+  scope_accuracy: z.number().min(0).max(2).optional(),
+  /** Time balance: max duration / min duration (lower is better) */
+  time_balance_ratio: z.number().min(1).optional(),
+});
+export type EvalRecord = z.infer<typeof EvalRecordSchema>;
+
+/**
+ * Partial record for in-progress capture
+ */
+export type PartialEvalRecord = Partial<EvalRecord> & {
+  id: string;
+  timestamp: string;
+  task: string;
+};
+
+// ============================================================================
+// Storage
+// ============================================================================
+
+/**
+ * Default path for eval data
+ */
+export const DEFAULT_EVAL_DATA_PATH = ".opencode/eval-data.jsonl";
+
+/**
+ * Get the eval data file path for a project
+ */
+export function getEvalDataPath(projectPath: string): string {
+  return path.join(projectPath, DEFAULT_EVAL_DATA_PATH);
+}
+
+/**
+ * Ensure the eval data directory exists
+ */
+export function ensureEvalDataDir(projectPath: string): void {
+  const evalPath = getEvalDataPath(projectPath);
+  const dir = path.dirname(evalPath);
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+}
+
+/**
+ * Append an eval record to the JSONL file
+ */
+export function appendEvalRecord(
+  projectPath: string,
+  record: EvalRecord | PartialEvalRecord,
+): void {
+  ensureEvalDataDir(projectPath);
+  const evalPath = getEvalDataPath(projectPath);
+  const line = JSON.stringify(record) + "\n";
+  fs.appendFileSync(evalPath, line, "utf-8");
+}
+
+/**
+ * Read all eval records from a project
+ */
+export function readEvalRecords(projectPath: string): EvalRecord[] {
+  const evalPath = getEvalDataPath(projectPath);
+  if (!fs.existsSync(evalPath)) {
+    return [];
+  }
+
+  const content = fs.readFileSync(evalPath, "utf-8");
+  const lines = content.trim().split("\n").filter(Boolean);
+
+  return lines.map((line) => {
+    const parsed = JSON.parse(line);
+    return EvalRecordSchema.parse(parsed);
+  });
+}
+
+/**
+ * Read partial records (for updating in-progress records)
+ */
+export function readPartialRecords(projectPath: string): PartialEvalRecord[] {
+  const evalPath = getEvalDataPath(projectPath);
+  if (!fs.existsSync(evalPath)) {
+    return [];
+  }
+
+  const content = fs.readFileSync(evalPath, "utf-8");
+  const lines = content.trim().split("\n").filter(Boolean);
+
+  return lines.map((line) => JSON.parse(line) as PartialEvalRecord);
+}
+
+/**
+ * Update an existing record by ID
+ */
+export function updateEvalRecord(
+  projectPath: string,
+  id: string,
+  updates: Partial<EvalRecord>,
+): boolean {
+  const records = readPartialRecords(projectPath);
+  const index = records.findIndex((r) => r.id === id);
+
+  if (index === -1) {
+    return false;
+  }
+
+  records[index] = { ...records[index], ...updates };
+
+  // Rewrite the file
+  const evalPath = getEvalDataPath(projectPath);
+  const content = records.map((r) => JSON.stringify(r)).join("\n") + "\n";
+  fs.writeFileSync(evalPath, content, "utf-8");
+
+  return true;
+}
+
+// ============================================================================
+// Capture Functions
+// ============================================================================
+
+/**
+ * In-memory store for in-progress records (keyed by epic ID)
+ */
+const inProgressRecords = new Map<string, PartialEvalRecord>();
+
+/**
+ * Start capturing a decomposition
+ *
+ * Called when swarm_decompose generates a decomposition.
+ * Creates a partial record that will be completed when outcomes arrive.
+ */
+export function captureDecomposition(params: {
+  epicId: string;
+  projectPath: string;
+  task: string;
+  context?: string;
+  strategy: "file-based" | "feature-based" | "risk-based" | "auto";
+  maxSubtasks: number;
+  epicTitle: string;
+  epicDescription?: string;
+  subtasks: Array<{
+    title: string;
+    description?: string;
+    files: string[];
+    dependencies?: number[];
+    estimated_complexity?: number;
+  }>;
+}): PartialEvalRecord {
+  const record: PartialEvalRecord = {
+    id: params.epicId,
+    timestamp: new Date().toISOString(),
+    project_path: params.projectPath,
+    task: params.task,
+    context: params.context,
+    strategy: params.strategy,
+    max_subtasks: params.maxSubtasks,
+    epic_title: params.epicTitle,
+    epic_description: params.epicDescription,
+    subtasks: params.subtasks,
+    outcomes: [],
+  };
+
+  // Store in memory for later updates
+  inProgressRecords.set(params.epicId, record);
+
+  // Also persist to disk (partial)
+  appendEvalRecord(params.projectPath, record);
+
+  return record;
+}
+
+/**
+ * Capture a subtask outcome
+ *
+ * Called when swarm_complete finishes a subtask.
+ * Updates the in-progress record with outcome data.
+ */
+export function captureSubtaskOutcome(params: {
+  epicId: string;
+  projectPath: string;
+  beadId: string;
+  title: string;
+  plannedFiles: string[];
+  actualFiles: string[];
+  durationMs: number;
+  errorCount: number;
+  retryCount: number;
+  success: boolean;
+  failureMode?: string;
+}): void {
+  const outcome: SubtaskOutcome = {
+    bead_id: params.beadId,
+    title: params.title,
+    planned_files: params.plannedFiles,
+    actual_files: params.actualFiles,
+    duration_ms: params.durationMs,
+    error_count: params.errorCount,
+    retry_count: params.retryCount,
+    success: params.success,
+    failure_mode: params.failureMode,
+  };
+
+  // Update in-memory record
+  const record = inProgressRecords.get(params.epicId);
+  if (record) {
+    record.outcomes = record.outcomes || [];
+    record.outcomes.push(outcome);
+  }
+
+  // Update on disk
+  updateEvalRecord(params.projectPath, params.epicId, {
+    outcomes: record?.outcomes,
+  });
+}
+
+/**
+ * Finalize an eval record
+ *
+ * Called when all subtasks are complete.
+ * Computes aggregate metrics and marks record as complete.
+ */
+export function finalizeEvalRecord(params: {
+  epicId: string;
+  projectPath: string;
+}): EvalRecord | null {
+  const record = inProgressRecords.get(params.epicId);
+  if (!record || !record.outcomes || record.outcomes.length === 0) {
+    return null;
+  }
+
+  // Compute aggregate metrics
+  const outcomes = record.outcomes;
+
+  const overallSuccess = outcomes.every((o) => o.success);
+  const totalDurationMs = outcomes.reduce((sum, o) => sum + o.duration_ms, 0);
+  const totalErrors = outcomes.reduce((sum, o) => sum + o.error_count, 0);
+
+  // File overlap: count files that appear in multiple subtasks
+  const allPlannedFiles = record.subtasks?.flatMap((s) => s.files) || [];
+  const fileOccurrences = new Map<string, number>();
+  for (const file of allPlannedFiles) {
+    fileOccurrences.set(file, (fileOccurrences.get(file) || 0) + 1);
+  }
+  const fileOverlapCount = Array.from(fileOccurrences.values()).filter(
+    (count) => count > 1,
+  ).length;
+
+  // Scope accuracy: actual files / planned files
+  const plannedFileSet = new Set(allPlannedFiles);
+  const actualFileSet = new Set(outcomes.flatMap((o) => o.actual_files));
+  const scopeAccuracy =
+    plannedFileSet.size > 0 ? actualFileSet.size / plannedFileSet.size : 1;
+
+  // Time balance: max duration / min duration
+  const durations = outcomes.map((o) => o.duration_ms).filter((d) => d > 0);
+  const timeBalanceRatio =
+    durations.length > 1 ? Math.max(...durations) / Math.min(...durations) : 1;
+
+  // Update record with computed metrics
+  const finalRecord: EvalRecord = {
+    ...(record as EvalRecord),
+    overall_success: overallSuccess,
+    total_duration_ms: totalDurationMs,
+    total_errors: totalErrors,
+    file_overlap_count: fileOverlapCount,
+    scope_accuracy: scopeAccuracy,
+    time_balance_ratio: timeBalanceRatio,
+  };
+
+  // Update on disk
+  updateEvalRecord(params.projectPath, params.epicId, finalRecord);
+
+  // Remove from in-progress
+  inProgressRecords.delete(params.epicId);
+
+  return finalRecord;
+}
+
+/**
+ * Capture human feedback on a decomposition
+ */
+export function captureHumanFeedback(params: {
+  epicId: string;
+  projectPath: string;
+  accepted: boolean;
+  modified: boolean;
+  notes?: string;
+}): void {
+  updateEvalRecord(params.projectPath, params.epicId, {
+    human_accepted: params.accepted,
+    human_modified: params.modified,
+    human_notes: params.notes,
+  });
+}
+
+// ============================================================================
+// Eval Data Export
+// ============================================================================
+
+/**
+ * Export eval records as Evalite-compatible test cases
+ *
+ * Filters to only complete records with outcomes.
+ */
+export function exportForEvalite(projectPath: string): Array<{
+  input: { task: string; context?: string };
+  expected: {
+    minSubtasks: number;
+    maxSubtasks: number;
+    requiredFiles?: string[];
+    overallSuccess?: boolean;
+  };
+  actual: EvalRecord;
+}> {
+  const records = readEvalRecords(projectPath);
+
+  return records
+    .filter((r) => r.outcomes && r.outcomes.length > 0)
+    .map((record) => ({
+      input: {
+        task: record.task,
+        context: record.context,
+      },
+      expected: {
+        minSubtasks: 2,
+        maxSubtasks: record.max_subtasks,
+        requiredFiles: record.subtasks.flatMap((s) => s.files),
+        overallSuccess: record.overall_success,
+      },
+      actual: record,
+    }));
+}
+
+/**
+ * Get statistics about captured eval data
+ */
+export function getEvalDataStats(projectPath: string): {
+  totalRecords: number;
+  completeRecords: number;
+  successRate: number;
+  avgSubtasks: number;
+  avgDurationMs: number;
+  avgScopeAccuracy: number;
+  avgTimeBalance: number;
+} {
+  const records = readEvalRecords(projectPath);
+  const complete = records.filter((r) => r.outcomes && r.outcomes.length > 0);
+
+  if (complete.length === 0) {
+    return {
+      totalRecords: records.length,
+      completeRecords: 0,
+      successRate: 0,
+      avgSubtasks: 0,
+      avgDurationMs: 0,
+      avgScopeAccuracy: 0,
+      avgTimeBalance: 0,
+    };
+  }
+
+  const successCount = complete.filter((r) => r.overall_success).length;
+  const avgSubtasks =
+    complete.reduce((sum, r) => sum + (r.outcomes?.length || 0), 0) /
+    complete.length;
+  const avgDurationMs =
+    complete.reduce((sum, r) => sum + (r.total_duration_ms || 0), 0) /
+    complete.length;
+  const avgScopeAccuracy =
+    complete.reduce((sum, r) => sum + (r.scope_accuracy || 1), 0) /
+    complete.length;
+  const avgTimeBalance =
+    complete.reduce((sum, r) => sum + (r.time_balance_ratio || 1), 0) /
+    complete.length;
+
+  return {
+    totalRecords: records.length,
+    completeRecords: complete.length,
+    successRate: successCount / complete.length,
+    avgSubtasks,
+    avgDurationMs,
+    avgScopeAccuracy,
+    avgTimeBalance,
+  };
+}