opencode-swarm-plugin 0.20.0 → 0.22.0

package/evals/lib/llm.ts

@@ -0,0 +1,115 @@
+/**
+ * LLM Client for Evalite Evals
+ *
+ * Uses AI SDK v6 with Vercel AI Gateway.
+ * Gateway handles provider routing - just pass "provider/model" string.
+ *
+ * @module evals/lib/llm
+ */
+import { generateText, gateway } from "ai";
+import type { GatewayModelId } from "ai";
+
+/**
+ * Default model for decomposition evals
+ * Using Claude Sonnet for good balance of quality and cost
+ */
+export const DEFAULT_MODEL: GatewayModelId = "anthropic/claude-sonnet-4-5";
+
+/**
+ * Generate a decomposition from a task description
+ *
+ * @param prompt - The full decomposition prompt
+ * @param model - Gateway model ID (e.g., "anthropic/claude-sonnet-4-5")
+ * @returns The raw text response from the LLM
+ */
+export async function generateDecomposition(
+  prompt: string,
+  model: GatewayModelId = DEFAULT_MODEL,
+): Promise<string> {
+  const { text } = await generateText({
+    model: gateway(model),
+    prompt,
+    maxOutputTokens: 4096,
+  });
+
+  return text;
+}
+
+/**
+ * Format a decomposition prompt from task and context
+ *
+ * Uses the same prompt template as swarm_plan_prompt
+ */
+export function formatDecompositionPrompt(
+  task: string,
+  context?: string,
+  maxSubtasks: number = 6,
+): string {
+  const contextSection = context ? `## Context\n${context}` : "";
+
+  return `You are decomposing a task into parallelizable subtasks for a swarm of agents.
+
+## Task
+${task}
+
+${contextSection}
+
+## Requirements
+
+1. **Break into 2-${maxSubtasks} independent subtasks** that can run in parallel
+2. **Assign files** - each subtask must specify which files it will modify
+3. **No file overlap** - files cannot appear in multiple subtasks (they get exclusive locks)
+4. **Order by dependency** - if subtask B needs subtask A's output, A must come first in the array
+5. **Estimate complexity** - 1 (trivial) to 5 (complex)
+
+## Response Format
+
+Respond with ONLY a JSON object matching this schema (no markdown, no explanation):
+
+{
+  "epic": {
+    "title": "string",
+    "description": "string"
+  },
+  "subtasks": [
+    {
+      "title": "string",
+      "description": "string",
+      "files": ["string"],
+      "dependencies": [0],
+      "estimated_complexity": 1
+    }
+  ]
+}
+
+## Guidelines
+
+- **Plan aggressively** - when in doubt, split further
+- **Prefer smaller, focused subtasks** over large complex ones
+- **Include test files** in the same subtask as the code they test
+- **Be specific about files** - use actual file paths, not placeholders
+
+Now decompose the task. Respond with JSON only:`;
+}
+
+/**
+ * Extract JSON from LLM response
+ *
+ * Handles responses that may have markdown code blocks or extra text
+ */
+export function extractJson(text: string): string {
+  // Try to find JSON in code blocks first
+  const codeBlockMatch = text.match(/```(?:json)?\s*([\s\S]*?)```/);
+  if (codeBlockMatch) {
+    return codeBlockMatch[1].trim();
+  }
+
+  // Try to find raw JSON object
+  const jsonMatch = text.match(/\{[\s\S]*\}/);
+  if (jsonMatch) {
+    return jsonMatch[0];
+  }
+
+  // Return as-is if no JSON found
+  return text;
+}

package/evals/scorers/index.ts

@@ -0,0 +1,200 @@
+import { createScorer } from "evalite";
+import type { BeadTree } from "../../src/schemas/index.js";
+
+/**
+ * Custom scorers for evaluating swarm task decomposition quality
+ */
+
+/**
+ * Checks that no files appear in multiple subtasks
+ *
+ * Independent subtasks are critical for parallel execution.
+ * File conflicts cause merge conflicts and coordination overhead.
+ *
+ * Score: 1.0 if no conflicts, 0.0 if conflicts found
+ */
+export const subtaskIndependence = createScorer({
+  name: "Subtask Independence",
+  description: "Checks that no files appear in multiple subtasks",
+  scorer: ({ output }) => {
+    try {
+      const beadTree = JSON.parse(String(output)) as BeadTree;
+      const fileMap = new Map<string, number>();
+
+      // Track which files appear in which subtasks
+      beadTree.subtasks.forEach((subtask) => {
+        subtask.files?.forEach((file) => {
+          const count = fileMap.get(file) || 0;
+          fileMap.set(file, count + 1);
+        });
+      });
+
+      // Check for conflicts
+      const conflicts = Array.from(fileMap.entries()).filter(
+        ([_, count]) => count > 1,
+      );
+
+      if (conflicts.length > 0) {
+        return {
+          score: 0,
+          message: `File conflicts found: ${conflicts.map(([f]) => f).join(", ")}`,
+        };
+      }
+
+      return {
+        score: 1,
+        message: "No file conflicts - subtasks are independent",
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse BeadTree: ${error}`,
+      };
+    }
+  },
+});
+
+// ============================================================================
+// Outcome-based scorers
+// ============================================================================
+
+export {
+  executionSuccess,
+  timeBalance,
+  scopeAccuracy,
+  scopeDrift,
+  noRework,
+} from "./outcome-scorers.js";
+
+/**
+ * Checks that subtasks cover the full task scope
+ *
+ * Incomplete coverage means:
+ * - Missing functionality
+ * - Follow-up work required
+ * - Task not actually complete
+ *
+ * Score: ratio of expected files covered (0.0 to 1.0)
+ * If no expected files specified, checks that subtasks exist
+ */
+export const coverageCompleteness = createScorer({
+  name: "Coverage Completeness",
+  description: "Checks that subtasks cover the full task scope",
+  scorer: ({ output, expected }) => {
+    try {
+      const beadTree = JSON.parse(String(output)) as BeadTree;
+
+      // If expected files specified, check coverage
+      const expectedData = expected as Record<string, unknown> | undefined;
+      if (expectedData && Array.isArray(expectedData.requiredFiles)) {
+        const allFiles = new Set(
+          beadTree.subtasks.flatMap((st) => st.files || []),
+        );
+
+        const requiredFiles = expectedData.requiredFiles as string[];
+        const coveredFiles = requiredFiles.filter((f) => allFiles.has(f));
+        const coverage = coveredFiles.length / requiredFiles.length;
+
+        return {
+          score: coverage,
+          message: `${coveredFiles.length}/${requiredFiles.length} required files covered`,
+        };
+      }
+
+      // Otherwise, check min/max subtask count
+      const minSubtasks = (expectedData?.minSubtasks as number) || 1;
+      const maxSubtasks = (expectedData?.maxSubtasks as number) || 10;
+      const count = beadTree.subtasks.length;
+
+      if (count < minSubtasks) {
+        return {
+          score: 0,
+          message: `Too few subtasks: ${count} < ${minSubtasks}`,
+        };
+      }
+
+      if (count > maxSubtasks) {
+        return {
+          score: 0.5,
+          message: `Too many subtasks: ${count} > ${maxSubtasks} (over-decomposed)`,
+        };
+      }
+
+      return {
+        score: 1,
+        message: `Good subtask count: ${count} (${minSubtasks}-${maxSubtasks})`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse BeadTree: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * Checks that each subtask has clear, actionable instructions
+ *
+ * Vague instructions lead to:
+ * - Agent confusion and blocking
+ * - Incorrect implementations
+ * - Need for coordinator intervention
+ *
+ * Score: Average of per-subtask instruction quality
+ */
+export const instructionClarity = createScorer({
+  name: "Instruction Clarity",
+  description: "Checks that subtasks have clear, actionable instructions",
+  scorer: ({ output }) => {
+    try {
+      const beadTree = JSON.parse(String(output)) as BeadTree;
+
+      if (beadTree.subtasks.length === 0) {
+        return {
+          score: 0,
+          message: "No subtasks found",
+        };
+      }
+
+      // Check each subtask for clarity signals
+      const scores = beadTree.subtasks.map((subtask) => {
+        let score = 0.5; // baseline
+
+        // Has description?
+        if (subtask.description && subtask.description.length > 20) {
+          score += 0.2;
+        }
+
+        // Has files specified?
+        if (subtask.files && subtask.files.length > 0) {
+          score += 0.2;
+        }
+
+        // Title is specific (not generic)?
+        const genericWords = ["update", "fix", "add", "change", "modify"];
+        const titleLower = subtask.title.toLowerCase();
+        const isGeneric = genericWords.some(
+          (word) => titleLower === word || titleLower.startsWith(`${word} `),
+        );
+        if (!isGeneric) {
+          score += 0.1;
+        }
+
+        return Math.min(1.0, score);
+      });
+
+      const avgScore = scores.reduce((sum, s) => sum + s, 0) / scores.length;
+
+      return {
+        score: avgScore,
+        message: `Average instruction clarity: ${(avgScore * 100).toFixed(0)}%`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse BeadTree: ${error}`,
+      };
+    }
+  },
+});

package/evals/scorers/outcome-scorers.test.ts

@@ -0,0 +1,27 @@
+/**
+ * Outcome-based Scorers Tests
+ *
+ * Tests the 5 new outcome-based scorers by verifying their exports.
+ * Full functional testing happens via Evalite integration.
+ */
+import { describe, it, expect } from "vitest";
+
+describe("Outcome Scorers", () => {
+  it("exports all 5 outcome scorers from outcome-scorers.ts", async () => {
+    const module = await import("./outcome-scorers.js");
+    expect(module.executionSuccess).toBeDefined();
+    expect(module.timeBalance).toBeDefined();
+    expect(module.scopeAccuracy).toBeDefined();
+    expect(module.scopeDrift).toBeDefined();
+    expect(module.noRework).toBeDefined();
+  });
+
+  it("re-exports all 5 outcome scorers from index.ts", async () => {
+    const indexModule = await import("./index.js");
+    expect(indexModule.executionSuccess).toBeDefined();
+    expect(indexModule.timeBalance).toBeDefined();
+    expect(indexModule.scopeAccuracy).toBeDefined();
+    expect(indexModule.scopeDrift).toBeDefined();
+    expect(indexModule.noRework).toBeDefined();
+  });
+});

package/evals/scorers/outcome-scorers.ts

@@ -0,0 +1,349 @@
+import { createScorer } from "evalite";
+import type { EvalRecord } from "../../src/eval-capture.js";
+
+/**
+ * Outcome-based scorers for evaluating decomposition quality
+ *
+ * These scorers evaluate based on ACTUAL execution outcomes,
+ * not just the structure of the decomposition.
+ *
+ * Requires EvalRecord with outcomes populated.
+ */
+
+/**
+ * Execution Success Scorer
+ *
+ * Measures whether all subtasks succeeded without errors.
+ * This is the ultimate measure - did the decomposition actually work?
+ *
+ * Score: 1.0 if all outcomes.success === true, 0.0 otherwise
+ */
+export const executionSuccess = createScorer({
+  name: "Execution Success",
+  description: "All subtasks completed successfully without errors",
+  scorer: ({ output }) => {
+    try {
+      const record = JSON.parse(String(output)) as EvalRecord;
+
+      // Check if outcomes exist
+      if (!record.outcomes || record.outcomes.length === 0) {
+        return {
+          score: 0,
+          message: "No outcome data available",
+        };
+      }
+
+      // Check if all subtasks succeeded
+      const allSucceeded = record.outcomes.every((outcome) => outcome.success);
+
+      if (allSucceeded) {
+        return {
+          score: 1,
+          message: `All ${record.outcomes.length} subtasks succeeded`,
+        };
+      }
+
+      // Report failures
+      const failures = record.outcomes.filter((o) => !o.success);
+      const failureList = failures.map((f) => f.title || f.bead_id).join(", ");
+
+      return {
+        score: 0,
+        message: `${failures.length}/${record.outcomes.length} subtasks failed: ${failureList}`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse EvalRecord: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * Time Balance Scorer
+ *
+ * Measures how evenly balanced the work was across subtasks.
+ * Unbalanced work means some agents finish early while others are bottlenecked.
+ *
+ * Score: 1.0 if max/min ratio < 2.0 (well balanced)
+ *        0.5 if ratio < 4.0 (moderately balanced)
+ *        0.0 if ratio >= 4.0 (poorly balanced)
+ */
+export const timeBalance = createScorer({
+  name: "Time Balance",
+  description: "Work is evenly distributed across subtasks (max/min duration)",
+  scorer: ({ output }) => {
+    try {
+      const record = JSON.parse(String(output)) as EvalRecord;
+
+      // Check if outcomes exist
+      if (!record.outcomes || record.outcomes.length === 0) {
+        return {
+          score: 0,
+          message: "No outcome data available",
+        };
+      }
+
+      // Need at least 2 subtasks to measure balance
+      if (record.outcomes.length < 2) {
+        return {
+          score: 1,
+          message: "Only one subtask - perfect balance",
+        };
+      }
+
+      // Get durations (filter out zeros)
+      const durations = record.outcomes
+        .map((o) => o.duration_ms)
+        .filter((d) => d > 0);
+
+      if (durations.length === 0) {
+        return {
+          score: 0,
+          message: "No duration data available",
+        };
+      }
+
+      const maxDuration = Math.max(...durations);
+      const minDuration = Math.min(...durations);
+      const ratio = maxDuration / minDuration;
+
+      // Score based on ratio
+      let score: number;
+      let assessment: string;
+
+      if (ratio < 2.0) {
+        score = 1.0;
+        assessment = "well balanced";
+      } else if (ratio < 4.0) {
+        score = 0.5;
+        assessment = "moderately balanced";
+      } else {
+        score = 0.0;
+        assessment = "poorly balanced";
+      }
+
+      const maxSeconds = Math.round(maxDuration / 1000);
+      const minSeconds = Math.round(minDuration / 1000);
+
+      return {
+        score,
+        message: `Ratio ${ratio.toFixed(1)}x (${maxSeconds}s / ${minSeconds}s) - ${assessment}`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse EvalRecord: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * Scope Accuracy Scorer
+ *
+ * Measures how accurately the decomposition predicted which files would be touched.
+ * High accuracy means the planner understood the work scope correctly.
+ *
+ * Score: intersection(actual, planned) / planned.length
+ *        1.0 = all planned files were touched, no extras
+ *        0.5 = half the planned files were touched
+ *        0.0 = none of the planned files were touched
+ */
+export const scopeAccuracy = createScorer({
+  name: "Scope Accuracy",
+  description:
+    "Planned files match actual files touched (accuracy of scope prediction)",
+  scorer: ({ output }) => {
+    try {
+      const record = JSON.parse(String(output)) as EvalRecord;
+
+      // Check if outcomes exist
+      if (!record.outcomes || record.outcomes.length === 0) {
+        return {
+          score: 0,
+          message: "No outcome data available",
+        };
+      }
+
+      // Calculate accuracy per subtask
+      let totalPlanned = 0;
+      let totalCorrect = 0;
+
+      for (const outcome of record.outcomes) {
+        const planned = new Set(outcome.planned_files);
+        const actual = new Set(outcome.actual_files);
+
+        // Count intersection (files in both planned and actual)
+        const intersection = Array.from(planned).filter((f) => actual.has(f));
+
+        totalPlanned += planned.size;
+        totalCorrect += intersection.length;
+      }
+
+      if (totalPlanned === 0) {
+        return {
+          score: 0,
+          message: "No planned files to measure against",
+        };
+      }
+
+      const accuracy = totalCorrect / totalPlanned;
+
+      return {
+        score: accuracy,
+        message: `${totalCorrect}/${totalPlanned} planned files touched (${(accuracy * 100).toFixed(0)}% accuracy)`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse EvalRecord: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * Scope Drift Scorer
+ *
+ * Penalizes when agents touch files NOT in their planned scope.
+ * Scope drift indicates poor planning or unexpected dependencies.
+ *
+ * Score: 1.0 if no drift (all actual files were planned)
+ *        Decreases linearly with drift percentage
+ *        0.0 if drift > 50%
+ */
+export const scopeDrift = createScorer({
+  name: "Scope Drift",
+  description:
+    "Agents stayed within their planned file scope (no unexpected files)",
+  scorer: ({ output }) => {
+    try {
+      const record = JSON.parse(String(output)) as EvalRecord;
+
+      // Check if outcomes exist
+      if (!record.outcomes || record.outcomes.length === 0) {
+        return {
+          score: 0,
+          message: "No outcome data available",
+        };
+      }
+
+      // Calculate drift per subtask
+      let totalActual = 0;
+      let totalDrift = 0;
+
+      for (const outcome of record.outcomes) {
+        const planned = new Set(outcome.planned_files);
+        const actual = new Set(outcome.actual_files);
+
+        // Count files in actual but NOT in planned
+        const drift = Array.from(actual).filter((f) => !planned.has(f));
+
+        totalActual += actual.size;
+        totalDrift += drift.length;
+      }
+
+      if (totalActual === 0) {
+        return {
+          score: 1,
+          message: "No files touched",
+        };
+      }
+
+      const driftRatio = totalDrift / totalActual;
+
+      // Score: 1.0 if no drift, linearly decrease to 0 at 50% drift
+      const score = Math.max(0, 1.0 - driftRatio * 2);
+
+      const driftPct = (driftRatio * 100).toFixed(0);
+
+      return {
+        score,
+        message: `${totalDrift}/${totalActual} files were unplanned (${driftPct}% drift)`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse EvalRecord: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * No Rework Scorer
+ *
+ * Checks that no subtask touched files assigned to another subtask.
+ * Rework indicates poor decomposition or missing dependencies.
+ *
+ * Score: 1.0 if no rework (no subtask touched another's planned files)
+ *        0.0 if rework detected
+ */
+export const noRework = createScorer({
+  name: "No Rework",
+  description: "No subtask touched files assigned to another subtask",
+  scorer: ({ output }) => {
+    try {
+      const record = JSON.parse(String(output)) as EvalRecord;
+
+      // Check if outcomes exist
+      if (!record.outcomes || record.outcomes.length === 0) {
+        return {
+          score: 0,
+          message: "No outcome data available",
+        };
+      }
+
+      // Build map of planned files per subtask
+      const plannedBySubtask = new Map<string, Set<string>>();
+
+      for (const outcome of record.outcomes) {
+        plannedBySubtask.set(outcome.bead_id, new Set(outcome.planned_files));
+      }
+
+      // Check each subtask for rework
+      const reworkCases: string[] = [];
+
+      for (const outcome of record.outcomes) {
+        const actualFiles = new Set(outcome.actual_files);
+
+        // Check if this subtask touched files planned for another subtask
+        for (const [otherBeadId, otherPlanned] of plannedBySubtask.entries()) {
+          if (otherBeadId === outcome.bead_id) {
+            continue; // Skip self
+          }
+
+          // Find intersection
+          const overlap = Array.from(actualFiles).filter((f) =>
+            otherPlanned.has(f),
+          );
+
+          if (overlap.length > 0) {
+            reworkCases.push(
+              `${outcome.title || outcome.bead_id} touched ${overlap.length} file(s) from ${otherBeadId}`,
+            );
+          }
+        }
+      }
+
+      if (reworkCases.length > 0) {
+        return {
+          score: 0,
+          message: `Rework detected: ${reworkCases.join("; ")}`,
+        };
+      }
+
+      return {
+        score: 1,
+        message: "No rework - all subtasks stayed in their lanes",
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse EvalRecord: ${error}`,
+      };
+    }
+  },
+});