opencode-swarm-plugin 0.44.0 → 0.44.2

package/evals/scorers/index.ts

@@ -1,328 +0,0 @@
-import { createScorer } from "evalite";
-import { generateText, gateway } from "ai";
-import type { GatewayModelId } from "ai";
-import type { CellTree } from "../../src/schemas/index.js";
-
-const JUDGE_MODEL: GatewayModelId = "anthropic/claude-haiku-4-5";
-
-/**
- * Custom scorers for evaluating swarm task decomposition quality
- */
-
-/**
- * Checks that no files appear in multiple subtasks
- *
- * Independent subtasks are critical for parallel execution.
- * File conflicts cause merge conflicts and coordination overhead.
- *
- * Score: 1.0 if no conflicts, 0.0 if conflicts found
- */
-export const subtaskIndependence = createScorer({
-  name: "Subtask Independence",
-  description: "Checks that no files appear in multiple subtasks",
-  scorer: ({ output }) => {
-    try {
-      const beadTree = JSON.parse(String(output)) as CellTree;
-      const fileMap = new Map<string, number>();
-
-      // Track which files appear in which subtasks
-      beadTree.subtasks.forEach((subtask) => {
-        subtask.files?.forEach((file) => {
-          const count = fileMap.get(file) || 0;
-          fileMap.set(file, count + 1);
-        });
-      });
-
-      // Check for conflicts
-      const conflicts = Array.from(fileMap.entries()).filter(
-        ([_, count]) => count > 1,
-      );
-
-      if (conflicts.length > 0) {
-        return {
-          score: 0,
-          message: `File conflicts found: ${conflicts.map(([f]) => f).join(", ")}`,
-        };
-      }
-
-      return {
-        score: 1,
-        message: "No file conflicts - subtasks are independent",
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse CellTree: ${error}`,
-      };
-    }
-  },
-});
-
-// ============================================================================
-// Outcome-based scorers
-// ============================================================================
-
-export {
-  executionSuccess,
-  timeBalance,
-  scopeAccuracy,
-  scopeDrift,
-  noRework,
-} from "./outcome-scorers.js";
-
-// ============================================================================
-// Compaction-specific scorers
-// ============================================================================
-
-export {
-  confidenceAccuracy,
-  contextInjectionCorrectness,
-  requiredPatternsPresent,
-  forbiddenPatternsAbsent,
-  compactionQuality,
-} from "./compaction-scorers.js";
-
-// ============================================================================
-// Coordinator discipline scorers
-// ============================================================================
-
-export {
-  violationCount,
-  spawnEfficiency,
-  reviewThoroughness,
-  timeToFirstSpawn,
-  overallDiscipline,
-} from "./coordinator-discipline.js";
-
-/**
- * Checks that subtasks cover the full task scope
- *
- * Incomplete coverage means:
- * - Missing functionality
- * - Follow-up work required
- * - Task not actually complete
- *
- * Score: ratio of expected files covered (0.0 to 1.0)
- * If no expected files specified, checks that subtasks exist
- */
-export const coverageCompleteness = createScorer({
-  name: "Coverage Completeness",
-  description: "Checks that subtasks cover the full task scope",
-  scorer: ({ output, expected }) => {
-    try {
-      const beadTree = JSON.parse(String(output)) as CellTree;
-
-      // If expected files specified, check coverage
-      const expectedData = expected as Record<string, unknown> | undefined;
-      if (expectedData && Array.isArray(expectedData.requiredFiles)) {
-        const allFiles = new Set(
-          beadTree.subtasks.flatMap((st) => st.files || []),
-        );
-
-        const requiredFiles = expectedData.requiredFiles as string[];
-        const coveredFiles = requiredFiles.filter((f) => allFiles.has(f));
-        const coverage = coveredFiles.length / requiredFiles.length;
-
-        return {
-          score: coverage,
-          message: `${coveredFiles.length}/${requiredFiles.length} required files covered`,
-        };
-      }
-
-      // Otherwise, check min/max subtask count
-      const minSubtasks = (expectedData?.minSubtasks as number) || 1;
-      const maxSubtasks = (expectedData?.maxSubtasks as number) || 10;
-      const count = beadTree.subtasks.length;
-
-      if (count < minSubtasks) {
-        return {
-          score: 0,
-          message: `Too few subtasks: ${count} < ${minSubtasks}`,
-        };
-      }
-
-      if (count > maxSubtasks) {
-        return {
-          score: 0.5,
-          message: `Too many subtasks: ${count} > ${maxSubtasks} (over-decomposed)`,
-        };
-      }
-
-      return {
-        score: 1,
-        message: `Good subtask count: ${count} (${minSubtasks}-${maxSubtasks})`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse CellTree: ${error}`,
-      };
-    }
-  },
-});
-
-/**
- * Checks that each subtask has clear, actionable instructions
- *
- * Vague instructions lead to:
- * - Agent confusion and blocking
- * - Incorrect implementations
- * - Need for coordinator intervention
- *
- * Score: Average of per-subtask instruction quality
- */
-export const instructionClarity = createScorer({
-  name: "Instruction Clarity",
-  description: "Checks that subtasks have clear, actionable instructions",
-  scorer: ({ output }) => {
-    try {
-      const beadTree = JSON.parse(String(output)) as CellTree;
-
-      if (beadTree.subtasks.length === 0) {
-        return {
-          score: 0,
-          message: "No subtasks found",
-        };
-      }
-
-      // Check each subtask for clarity signals
-      const scores = beadTree.subtasks.map((subtask) => {
-        let score = 0.5; // baseline
-
-        // Has description?
-        if (subtask.description && subtask.description.length > 20) {
-          score += 0.2;
-        }
-
-        // Has files specified?
-        if (subtask.files && subtask.files.length > 0) {
-          score += 0.2;
-        }
-
-        // Title is specific (not generic)?
-        const genericWords = ["update", "fix", "add", "change", "modify"];
-        const titleLower = subtask.title.toLowerCase();
-        const isGeneric = genericWords.some(
-          (word) => titleLower === word || titleLower.startsWith(`${word} `),
-        );
-        if (!isGeneric) {
-          score += 0.1;
-        }
-
-        return Math.min(1.0, score);
-      });
-
-      const avgScore = scores.reduce((sum, s) => sum + s, 0) / scores.length;
-
-      return {
-        score: avgScore,
-        message: `Average instruction clarity: ${(avgScore * 100).toFixed(0)}%`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse CellTree: ${error}`,
-      };
-    }
-  },
-});
-
-// ============================================================================
-// LLM-as-Judge Scorers
-// ============================================================================
-
-/**
- * LLM-as-judge scorer for decomposition coherence
- *
- * Uses Claude Haiku to evaluate whether subtasks are truly independent,
- * well-scoped, and complete. This catches nuances that heuristics miss:
- * - Semantic dependencies between subtasks
- * - Scope that's too big or too trivial
- * - Missing pieces that would block completion
- *
- * Only use for decomposition evals - this is where it matters.
- */
-export const decompositionCoherence = createScorer({
-  name: "Decomposition Coherence (LLM Judge)",
-  description:
-    "LLM evaluates whether subtasks are truly independent and well-scoped",
-  scorer: async ({ output, input }) => {
-    try {
-      const decomposition =
-        typeof output === "string" ? output : JSON.stringify(output, null, 2);
-
-      // Get original task from input if available
-      const originalTask =
-        typeof input === "object" && input !== null && "task" in input
-          ? String((input as { task: string }).task)
-          : "Unknown task";
-
-      const { text } = await generateText({
-        model: gateway(JUDGE_MODEL),
-        prompt: `You are evaluating a task decomposition for parallel agent execution.
-
-ORIGINAL TASK:
-${originalTask}
-
-DECOMPOSITION:
-${decomposition}
-
-Evaluate on these criteria (be harsh - bad decompositions waste expensive parallel work):
-
-1. INDEPENDENCE (25%): Can subtasks truly run in parallel? Look for:
-   - Shared state dependencies (one writes, another reads)
-   - Ordering requirements hidden in the task descriptions
-   - Shared files that will cause merge conflicts
-
-2. SCOPE (25%): Is each subtask right-sized?
-   - Too big: Should be split further (>2 hours of work)
-   - Too small: Trivial tasks that waste agent spawn overhead
-   - Goldilocks: 30min-2hr of focused work
-
-3. COMPLETENESS (25%): Does the sum equal the whole?
-   - Missing pieces that would leave the task incomplete
-   - Gaps between subtasks (who handles X?)
-   - Implicit work not captured in any subtask
-
-4. CLARITY (25%): Would an agent know what to do?
-   - Vague descriptions that invite interpretation
-   - Missing context needed to start work
-   - Ambiguous boundaries between subtasks
-
-Return ONLY valid JSON (no markdown, no explanation):
-{"score": <0-100>, "issues": ["issue1", "issue2"], "strengths": ["strength1"]}`,
-        maxOutputTokens: 512,
-      });
-
-      // Parse JSON response - handle potential markdown wrapping
-      let jsonText = text.trim();
-      if (jsonText.startsWith("```")) {
-        jsonText = jsonText.replace(/```json?\n?/g, "").replace(/```$/g, "");
-      }
-
-      const result = JSON.parse(jsonText) as {
-        score: number;
-        issues: string[];
-        strengths?: string[];
-      };
-
-      const issueText =
-        result.issues.length > 0 ? result.issues.join("; ") : "No issues";
-      const strengthText =
-        result.strengths && result.strengths.length > 0
-          ? ` | Strengths: ${result.strengths.join("; ")}`
-          : "";
-
-      return {
-        score: result.score / 100,
-        message: `${issueText}${strengthText}`,
-      };
-    } catch (error) {
-      // Don't fail the eval if judge fails - return neutral score
-      return {
-        score: 0.5,
-        message: `LLM judge error: ${error instanceof Error ? error.message : String(error)}`,
-      };
-    }
-  },
-});

package/evals/scorers/outcome-scorers.evalite-test.ts

@@ -1,27 +0,0 @@
-/**
- * Outcome-based Scorers Tests
- *
- * Tests the 5 new outcome-based scorers by verifying their exports.
- * Full functional testing happens via Evalite integration.
- */
-import { describe, it, expect } from "bun:test";
-
-describe("Outcome Scorers", () => {
-  it("exports all 5 outcome scorers from outcome-scorers.ts", async () => {
-    const module = await import("./outcome-scorers.js");
-    expect(module.executionSuccess).toBeDefined();
-    expect(module.timeBalance).toBeDefined();
-    expect(module.scopeAccuracy).toBeDefined();
-    expect(module.scopeDrift).toBeDefined();
-    expect(module.noRework).toBeDefined();
-  });
-
-  it("re-exports all 5 outcome scorers from index.ts", async () => {
-    const indexModule = await import("./index.js");
-    expect(indexModule.executionSuccess).toBeDefined();
-    expect(indexModule.timeBalance).toBeDefined();
-    expect(indexModule.scopeAccuracy).toBeDefined();
-    expect(indexModule.scopeDrift).toBeDefined();
-    expect(indexModule.noRework).toBeDefined();
-  });
-});

package/evals/scorers/outcome-scorers.ts

@@ -1,349 +0,0 @@
-import { createScorer } from "evalite";
-import type { EvalRecord } from "../../src/eval-capture.js";
-
-/**
- * Outcome-based scorers for evaluating decomposition quality
- *
- * These scorers evaluate based on ACTUAL execution outcomes,
- * not just the structure of the decomposition.
- *
- * Requires EvalRecord with outcomes populated.
- */
-
-/**
- * Execution Success Scorer
- *
- * Measures whether all subtasks succeeded without errors.
- * This is the ultimate measure - did the decomposition actually work?
- *
- * Score: 1.0 if all outcomes.success === true, 0.0 otherwise
- */
-export const executionSuccess = createScorer({
-  name: "Execution Success",
-  description: "All subtasks completed successfully without errors",
-  scorer: ({ output }) => {
-    try {
-      const record = JSON.parse(String(output)) as EvalRecord;
-
-      // Check if outcomes exist
-      if (!record.outcomes || record.outcomes.length === 0) {
-        return {
-          score: 0,
-          message: "No outcome data available",
-        };
-      }
-
-      // Check if all subtasks succeeded
-      const allSucceeded = record.outcomes.every((outcome) => outcome.success);
-
-      if (allSucceeded) {
-        return {
-          score: 1,
-          message: `All ${record.outcomes.length} subtasks succeeded`,
-        };
-      }
-
-      // Report failures
-      const failures = record.outcomes.filter((o) => !o.success);
-      const failureList = failures.map((f) => f.title || f.bead_id).join(", ");
-
-      return {
-        score: 0,
-        message: `${failures.length}/${record.outcomes.length} subtasks failed: ${failureList}`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse EvalRecord: ${error}`,
-      };
-    }
-  },
-});
-
-/**
- * Time Balance Scorer
- *
- * Measures how evenly balanced the work was across subtasks.
- * Unbalanced work means some agents finish early while others are bottlenecked.
- *
- * Score: 1.0 if max/min ratio < 2.0 (well balanced)
- *        0.5 if ratio < 4.0 (moderately balanced)
- *        0.0 if ratio >= 4.0 (poorly balanced)
- */
-export const timeBalance = createScorer({
-  name: "Time Balance",
-  description: "Work is evenly distributed across subtasks (max/min duration)",
-  scorer: ({ output }) => {
-    try {
-      const record = JSON.parse(String(output)) as EvalRecord;
-
-      // Check if outcomes exist
-      if (!record.outcomes || record.outcomes.length === 0) {
-        return {
-          score: 0,
-          message: "No outcome data available",
-        };
-      }
-
-      // Need at least 2 subtasks to measure balance
-      if (record.outcomes.length < 2) {
-        return {
-          score: 1,
-          message: "Only one subtask - perfect balance",
-        };
-      }
-
-      // Get durations (filter out zeros)
-      const durations = record.outcomes
-        .map((o) => o.duration_ms)
-        .filter((d) => d > 0);
-
-      if (durations.length === 0) {
-        return {
-          score: 0,
-          message: "No duration data available",
-        };
-      }
-
-      const maxDuration = Math.max(...durations);
-      const minDuration = Math.min(...durations);
-      const ratio = maxDuration / minDuration;
-
-      // Score based on ratio
-      let score: number;
-      let assessment: string;
-
-      if (ratio < 2.0) {
-        score = 1.0;
-        assessment = "well balanced";
-      } else if (ratio < 4.0) {
-        score = 0.5;
-        assessment = "moderately balanced";
-      } else {
-        score = 0.0;
-        assessment = "poorly balanced";
-      }
-
-      const maxSeconds = Math.round(maxDuration / 1000);
-      const minSeconds = Math.round(minDuration / 1000);
-
-      return {
-        score,
-        message: `Ratio ${ratio.toFixed(1)}x (${maxSeconds}s / ${minSeconds}s) - ${assessment}`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse EvalRecord: ${error}`,
-      };
-    }
-  },
-});
-
-/**
- * Scope Accuracy Scorer
- *
- * Measures how accurately the decomposition predicted which files would be touched.
- * High accuracy means the planner understood the work scope correctly.
- *
- * Score: intersection(actual, planned) / planned.length
- *        1.0 = all planned files were touched, no extras
- *        0.5 = half the planned files were touched
- *        0.0 = none of the planned files were touched
- */
-export const scopeAccuracy = createScorer({
-  name: "Scope Accuracy",
-  description:
-    "Planned files match actual files touched (accuracy of scope prediction)",
-  scorer: ({ output }) => {
-    try {
-      const record = JSON.parse(String(output)) as EvalRecord;
-
-      // Check if outcomes exist
-      if (!record.outcomes || record.outcomes.length === 0) {
-        return {
-          score: 0,
-          message: "No outcome data available",
-        };
-      }
-
-      // Calculate accuracy per subtask
-      let totalPlanned = 0;
-      let totalCorrect = 0;
-
-      for (const outcome of record.outcomes) {
-        const planned = new Set(outcome.planned_files);
-        const actual = new Set(outcome.actual_files);
-
-        // Count intersection (files in both planned and actual)
-        const intersection = Array.from(planned).filter((f) => actual.has(f));
-
-        totalPlanned += planned.size;
-        totalCorrect += intersection.length;
-      }
-
-      if (totalPlanned === 0) {
-        return {
-          score: 0,
-          message: "No planned files to measure against",
-        };
-      }
-
-      const accuracy = totalCorrect / totalPlanned;
-
-      return {
-        score: accuracy,
-        message: `${totalCorrect}/${totalPlanned} planned files touched (${(accuracy * 100).toFixed(0)}% accuracy)`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse EvalRecord: ${error}`,
-      };
-    }
-  },
-});
-
-/**
- * Scope Drift Scorer
- *
- * Penalizes when agents touch files NOT in their planned scope.
- * Scope drift indicates poor planning or unexpected dependencies.
- *
- * Score: 1.0 if no drift (all actual files were planned)
- *        Decreases linearly with drift percentage
- *        0.0 if drift > 50%
- */
-export const scopeDrift = createScorer({
-  name: "Scope Drift",
-  description:
-    "Agents stayed within their planned file scope (no unexpected files)",
-  scorer: ({ output }) => {
-    try {
-      const record = JSON.parse(String(output)) as EvalRecord;
-
-      // Check if outcomes exist
-      if (!record.outcomes || record.outcomes.length === 0) {
-        return {
-          score: 0,
-          message: "No outcome data available",
-        };
-      }
-
-      // Calculate drift per subtask
-      let totalActual = 0;
-      let totalDrift = 0;
-
-      for (const outcome of record.outcomes) {
-        const planned = new Set(outcome.planned_files);
-        const actual = new Set(outcome.actual_files);
-
-        // Count files in actual but NOT in planned
-        const drift = Array.from(actual).filter((f) => !planned.has(f));
-
-        totalActual += actual.size;
-        totalDrift += drift.length;
-      }
-
-      if (totalActual === 0) {
-        return {
-          score: 1,
-          message: "No files touched",
-        };
-      }
-
-      const driftRatio = totalDrift / totalActual;
-
-      // Score: 1.0 if no drift, linearly decrease to 0 at 50% drift
-      const score = Math.max(0, 1.0 - driftRatio * 2);
-
-      const driftPct = (driftRatio * 100).toFixed(0);
-
-      return {
-        score,
-        message: `${totalDrift}/${totalActual} files were unplanned (${driftPct}% drift)`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse EvalRecord: ${error}`,
-      };
-    }
-  },
-});
-
-/**
- * No Rework Scorer
- *
- * Checks that no subtask touched files assigned to another subtask.
- * Rework indicates poor decomposition or missing dependencies.
- *
- * Score: 1.0 if no rework (no subtask touched another's planned files)
- *        0.0 if rework detected
- */
-export const noRework = createScorer({
-  name: "No Rework",
-  description: "No subtask touched files assigned to another subtask",
-  scorer: ({ output }) => {
-    try {
-      const record = JSON.parse(String(output)) as EvalRecord;
-
-      // Check if outcomes exist
-      if (!record.outcomes || record.outcomes.length === 0) {
-        return {
-          score: 0,
-          message: "No outcome data available",
-        };
-      }
-
-      // Build map of planned files per subtask
-      const plannedBySubtask = new Map<string, Set<string>>();
-
-      for (const outcome of record.outcomes) {
-        plannedBySubtask.set(outcome.bead_id, new Set(outcome.planned_files));
-      }
-
-      // Check each subtask for rework
-      const reworkCases: string[] = [];
-
-      for (const outcome of record.outcomes) {
-        const actualFiles = new Set(outcome.actual_files);
-
-        // Check if this subtask touched files planned for another subtask
-        for (const [otherBeadId, otherPlanned] of plannedBySubtask.entries()) {
-          if (otherBeadId === outcome.bead_id) {
-            continue; // Skip self
-          }
-
-          // Find intersection
-          const overlap = Array.from(actualFiles).filter((f) =>
-            otherPlanned.has(f),
-          );
-
-          if (overlap.length > 0) {
-            reworkCases.push(
-              `${outcome.title || outcome.bead_id} touched ${overlap.length} file(s) from ${otherBeadId}`,
-            );
-          }
-        }
-      }
-
-      if (reworkCases.length > 0) {
-        return {
-          score: 0,
-          message: `Rework detected: ${reworkCases.join("; ")}`,
-        };
-      }
-
-      return {
-        score: 1,
-        message: "No rework - all subtasks stayed in their lanes",
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse EvalRecord: ${error}`,
-      };
-    }
-  },
-});