opencode-swarm-plugin 0.38.0 → 0.40.0

package/evals/scorers/coordinator-discipline.evalite-test.ts

@@ -5,6 +5,7 @@ import { describe, expect, it } from "bun:test";
 import type { CoordinatorSession } from "../../src/eval-capture.js";
 import {
 	overallDiscipline,
+	reviewEfficiency,
 	reviewThoroughness,
 	spawnEfficiency,
 	timeToFirstSpawn,
@@ -12,7 +13,7 @@ import {
 } from "./coordinator-discipline.js";
 
 describe("violationCount", () => {
-	it("scores 1.0 for zero violations", () => {
+	it("scores 1.0 for zero violations", async () => {
 		const session: CoordinatorSession = {
 			session_id: "test-session",
 			epic_id: "test-epic",
@@ -30,9 +31,10 @@ describe("violationCount", () => {
 			],
 		};
 
-		const result = violationCount.scorer({
+		const result = await violationCount({
 			output: JSON.stringify(session),
 			expected: {},
+			input: undefined,
 		});
 
 		expect(result.score).toBe(1.0);
@@ -534,3 +536,165 @@ describe("overallDiscipline", () => {
 		expect(result.message).toContain("Speed:");
 	});
 });
+
+describe("reviewEfficiency", () => {
+	it("scores 1.0 for ideal 1:1 ratio (one review per spawn)", async () => {
+		const session: CoordinatorSession = {
+			session_id: "test-session",
+			epic_id: "test-epic",
+			start_time: "2025-01-01T00:00:00Z",
+			events: [
+				{
+					session_id: "test-session",
+					epic_id: "test-epic",
+					timestamp: "2025-01-01T00:00:10Z",
+					event_type: "DECISION",
+					decision_type: "worker_spawned",
+					payload: { bead_id: "bd-1" },
+				},
+				{
+					session_id: "test-session",
+					epic_id: "test-epic",
+					timestamp: "2025-01-01T00:00:20Z",
+					event_type: "DECISION",
+					decision_type: "worker_spawned",
+					payload: { bead_id: "bd-2" },
+				},
+				{
+					session_id: "test-session",
+					epic_id: "test-epic",
+					timestamp: "2025-01-01T00:10:00Z",
+					event_type: "DECISION",
+					decision_type: "review_completed",
+					payload: { bead_id: "bd-1" },
+				},
+				{
+					session_id: "test-session",
+					epic_id: "test-epic",
+					timestamp: "2025-01-01T00:10:10Z",
+					event_type: "DECISION",
+					decision_type: "review_completed",
+					payload: { bead_id: "bd-2" },
+				},
+			],
+		};
+
+		const result = await reviewEfficiency({
+			output: JSON.stringify(session),
+			expected: {},
+			input: undefined,
+		});
+
+		expect(result.score).toBe(1.0);
+		expect(result.message).toContain("2 reviews / 2 spawns");
+	});
+
+	it("penalizes over-reviewing (>2:1 ratio)", async () => {
+		// 6 reviews for 2 spawns = 3:1 ratio (over-reviewing)
+		const session: CoordinatorSession = {
+			session_id: "test-session",
+			epic_id: "test-epic",
+			start_time: "2025-01-01T00:00:00Z",
+			events: [
+				{
+					session_id: "test-session",
+					epic_id: "test-epic",
+					timestamp: "2025-01-01T00:00:10Z",
+					event_type: "DECISION",
+					decision_type: "worker_spawned",
+					payload: { bead_id: "bd-1" },
+				},
+				{
+					session_id: "test-session",
+					epic_id: "test-epic",
+					timestamp: "2025-01-01T00:00:20Z",
+					event_type: "DECISION",
+					decision_type: "worker_spawned",
+					payload: { bead_id: "bd-2" },
+				},
+				...Array.from({ length: 6 }, (_, i) => ({
+					session_id: "test-session",
+					epic_id: "test-epic",
+					timestamp: `2025-01-01T00:10:${String(i * 10).padStart(2, "0")}Z`,
+					event_type: "DECISION" as const,
+					decision_type: "review_completed" as const,
+					payload: { bead_id: `bd-${(i % 2) + 1}` },
+				})),
+			],
+		};
+
+		const result = await reviewEfficiency({
+			output: JSON.stringify(session),
+			expected: {},
+			input: undefined,
+		});
+
+		// 3:1 ratio should be penalized (score < 0.5)
+		expect(result.score).toBeLessThan(0.5);
+		expect(result.message).toContain("6 reviews / 2 spawns");
+	});
+
+	it("handles no spawns gracefully", async () => {
+		const session: CoordinatorSession = {
+			session_id: "test-session",
+			epic_id: "test-epic",
+			start_time: "2025-01-01T00:00:00Z",
+			events: [
+				{
+					session_id: "test-session",
+					epic_id: "test-epic",
+					timestamp: "2025-01-01T00:00:00Z",
+					event_type: "DECISION",
+					decision_type: "strategy_selected",
+					payload: { strategy: "file-based" },
+				},
+			],
+		};
+
+		const result = await reviewEfficiency({
+			output: JSON.stringify(session),
+			expected: {},
+			input: undefined,
+		});
+
+		expect(result.score).toBe(1.0);
+		expect(result.message).toContain("No workers spawned");
+	});
+
+	it("handles no reviews gracefully (0:N ratio)", async () => {
+		const session: CoordinatorSession = {
+			session_id: "test-session",
+			epic_id: "test-epic",
+			start_time: "2025-01-01T00:00:00Z",
+			events: [
+				{
+					session_id: "test-session",
+					epic_id: "test-epic",
+					timestamp: "2025-01-01T00:00:10Z",
+					event_type: "DECISION",
+					decision_type: "worker_spawned",
+					payload: { bead_id: "bd-1" },
+				},
+				{
+					session_id: "test-session",
+					epic_id: "test-epic",
+					timestamp: "2025-01-01T00:00:20Z",
+					event_type: "DECISION",
+					decision_type: "worker_spawned",
+					payload: { bead_id: "bd-2" },
+				},
+			],
+		};
+
+		const result = await reviewEfficiency({
+			output: JSON.stringify(session),
+			expected: {},
+			input: undefined,
+		});
+
+		// No reviews is bad (should use reviewThoroughness for this)
+		// But this scorer focuses on over-reviewing, so no reviews = 1.0 (not over-reviewing)
+		expect(result.score).toBe(1.0);
+		expect(result.message).toContain("0 reviews / 2 spawns");
+	});
+});

package/evals/scorers/coordinator-discipline.ts

@@ -70,6 +70,9 @@ export const violationCount = createScorer({
  * Coordinators should delegate work, not do it themselves.
  *
  * Score: workers_spawned / subtasks_planned
+ *
+ * If no decomposition_complete event exists, falls back to counting spawns
+ * and returns 1.0 if any workers were spawned (better than nothing).
  */
 export const spawnEfficiency = createScorer({
   name: "Spawn Efficiency",
@@ -85,7 +88,20 @@ export const spawnEfficiency = createScorer({
           e.decision_type === "decomposition_complete"
       );
 
+      // Count worker_spawned events
+      const spawned = session.events.filter(
+        (e) =>
+          e.event_type === "DECISION" && e.decision_type === "worker_spawned"
+      ).length;
+
       if (!decomp) {
+        // Fallback: if workers were spawned but no decomp event, assume they're doing work
+        if (spawned > 0) {
+          return {
+            score: 1.0,
+            message: `${spawned} workers spawned (no decomposition event)`,
+          };
+        }
         return {
           score: 0,
           message: "No decomposition event found",
@@ -101,17 +117,81 @@ export const spawnEfficiency = createScorer({
         };
       }
 
+      const score = spawned / subtaskCount;
+
+      return {
+        score,
+        message: `${spawned}/${subtaskCount} workers spawned (${(score * 100).toFixed(0)}%)`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse CoordinatorSession: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * Review Efficiency Scorer
+ *
+ * Measures review-to-spawn ratio to detect over-reviewing.
+ * Ideal ratio is 1:1 (one review per spawned worker).
+ * Penalizes >2:1 ratio (over-reviewing wastes context).
+ *
+ * Scoring:
+ * - 0:N or 1:1 ratio = 1.0 (perfect)
+ * - 2:1 ratio = 0.5 (threshold)
+ * - >2:1 ratio = linear penalty toward 0.0
+ *
+ * Score: normalized to 0-1 (lower ratio is better)
+ */
+export const reviewEfficiency = createScorer({
+  name: "Review Efficiency",
+  description: "Review-to-spawn ratio (penalize over-reviewing >2:1)",
+  scorer: ({ output }) => {
+    try {
+      const session = JSON.parse(String(output)) as CoordinatorSession;
+
       // Count worker_spawned events
       const spawned = session.events.filter(
         (e) =>
           e.event_type === "DECISION" && e.decision_type === "worker_spawned"
       ).length;
 
-      const score = spawned / subtaskCount;
+      if (spawned === 0) {
+        return {
+          score: 1.0,
+          message: "No workers spawned",
+        };
+      }
+
+      // Count review_completed events
+      const reviewed = session.events.filter(
+        (e) =>
+          e.event_type === "DECISION" && e.decision_type === "review_completed"
+      ).length;
+
+      const ratio = reviewed / spawned;
+
+      // Scoring:
+      // - ratio <= 1.0: perfect (1.0)
+      // - ratio <= 2.0: linear decay from 1.0 to 0.5
+      // - ratio > 2.0: linear penalty from 0.5 toward 0.0
+      let score: number;
+      if (ratio <= 1.0) {
+        score = 1.0;
+      } else if (ratio <= 2.0) {
+        // Linear decay: 1.0 at ratio=1.0, 0.5 at ratio=2.0
+        score = 1.0 - (ratio - 1.0) * 0.5;
+      } else {
+        // Penalty for extreme over-reviewing: 0.5 at ratio=2.0, 0.0 at ratio=4.0
+        score = Math.max(0, 0.5 - (ratio - 2.0) * 0.25);
+      }
 
       return {
         score,
-        message: `${spawned}/${subtaskCount} workers spawned (${(score * 100).toFixed(0)}%)`,
+        message: `${reviewed} reviews / ${spawned} spawns (${ratio.toFixed(1)}:1 ratio)`,
       };
     } catch (error) {
       return {
@@ -254,6 +334,259 @@ export const timeToFirstSpawn = createScorer({
   },
 });
 
+/**
+ * Researcher Spawn Rate Scorer
+ *
+ * Measures whether coordinator spawns researchers for unfamiliar technology.
+ * Coordinators should delegate research instead of calling pdf-brain/context7 directly.
+ *
+ * Score: 1.0 if researcher_spawned events exist, 0.0 otherwise
+ */
+export const researcherSpawnRate = createScorer({
+  name: "Researcher Spawn Rate",
+  description: "Coordinator spawned researchers for unfamiliar tech",
+  scorer: ({ output }) => {
+    try {
+      const session = JSON.parse(String(output)) as CoordinatorSession;
+
+      // Count researcher_spawned events
+      const researchers = session.events.filter(
+        (e) =>
+          e.event_type === "DECISION" && e.decision_type === "researcher_spawned"
+      );
+
+      const count = researchers.length;
+
+      if (count === 0) {
+        return {
+          score: 0.0,
+          message: "No researchers spawned (may indicate coordinator queried docs directly)",
+        };
+      }
+
+      return {
+        score: 1.0,
+        message: `${count} researcher(s) spawned`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse CoordinatorSession: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * Skill Loading Rate Scorer
+ *
+ * Measures whether coordinator loads relevant skills via skills_use().
+ * Shows knowledge-seeking behavior.
+ *
+ * Score: 1.0 if skill_loaded events exist, 0.5 otherwise (not critical, but helpful)
+ */
+export const skillLoadingRate = createScorer({
+  name: "Skill Loading Rate",
+  description: "Coordinator loaded relevant skills for domain knowledge",
+  scorer: ({ output }) => {
+    try {
+      const session = JSON.parse(String(output)) as CoordinatorSession;
+
+      // Count skill_loaded events
+      const skills = session.events.filter(
+        (e) =>
+          e.event_type === "DECISION" && e.decision_type === "skill_loaded"
+      );
+
+      const count = skills.length;
+
+      if (count === 0) {
+        return {
+          score: 0.5,
+          message: "No skills loaded (not critical, but helpful)",
+        };
+      }
+
+      return {
+        score: 1.0,
+        message: `${count} skill(s) loaded`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse CoordinatorSession: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * Inbox Monitoring Rate Scorer
+ *
+ * Measures how frequently coordinator checks inbox for worker messages.
+ * Regular monitoring (every ~15min or when workers finish) shows good coordination.
+ *
+ * Score based on inbox_checked events relative to worker activity:
+ * - 0 checks = 0.0 (coordinator not monitoring)
+ * - 1+ checks = 1.0 (coordinator is responsive)
+ */
+export const inboxMonitoringRate = createScorer({
+  name: "Inbox Monitoring Rate",
+  description: "Coordinator checked inbox regularly for worker messages",
+  scorer: ({ output }) => {
+    try {
+      const session = JSON.parse(String(output)) as CoordinatorSession;
+
+      // Count inbox_checked events
+      const checks = session.events.filter(
+        (e) =>
+          e.event_type === "DECISION" && e.decision_type === "inbox_checked"
+      );
+
+      // Count worker activity (spawns + outcomes)
+      const workerActivity = session.events.filter(
+        (e) =>
+          (e.event_type === "DECISION" && e.decision_type === "worker_spawned") ||
+          (e.event_type === "OUTCOME" &&
+            ["subtask_success", "subtask_failed", "blocker_detected"].includes(
+              e.outcome_type
+            ))
+      );
+
+      const checkCount = checks.length;
+      const activityCount = workerActivity.length;
+
+      if (activityCount === 0) {
+        return {
+          score: 1.0,
+          message: "No worker activity to monitor",
+        };
+      }
+
+      if (checkCount === 0) {
+        return {
+          score: 0.0,
+          message: `${activityCount} worker events, 0 inbox checks (not monitoring)`,
+        };
+      }
+
+      return {
+        score: 1.0,
+        message: `${checkCount} inbox check(s) for ${activityCount} worker events`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse CoordinatorSession: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * Blocker Response Time Scorer
+ *
+ * Measures how quickly coordinator responds to blocked workers.
+ * Time between blocker_detected (OUTCOME) and blocker_resolved (DECISION).
+ *
+ * Normalization:
+ * - < 5min: 1.0 (excellent)
+ * - 5-15min: linear decay to 0.5
+ * - > 15min: 0.0 (too slow, worker is idle)
+ *
+ * Score: Average response time across all blockers
+ */
+export const blockerResponseTime = createScorer({
+  name: "Blocker Response Time",
+  description: "Coordinator unblocked workers quickly",
+  scorer: ({ output }) => {
+    try {
+      const session = JSON.parse(String(output)) as CoordinatorSession;
+
+      // Find blocker_detected events
+      const blockers = session.events.filter(
+        (e) =>
+          e.event_type === "OUTCOME" && e.outcome_type === "blocker_detected"
+      );
+
+      if (blockers.length === 0) {
+        return {
+          score: 1.0,
+          message: "No blockers detected",
+        };
+      }
+
+      // Find blocker_resolved events
+      const resolutions = session.events.filter(
+        (e) =>
+          e.event_type === "DECISION" && e.decision_type === "blocker_resolved"
+      );
+
+      if (resolutions.length === 0) {
+        return {
+          score: 0.0,
+          message: `${blockers.length} blocker(s) detected, 0 resolved (workers still blocked)`,
+        };
+      }
+
+      // Match blockers to resolutions by subtask_id and calculate response times
+      const responseTimes: number[] = [];
+      for (const blocker of blockers) {
+        const subtaskId = (blocker.payload as any).subtask_id;
+        const blockerTime = new Date(blocker.timestamp).getTime();
+
+        // Find resolution for this subtask
+        const resolution = resolutions.find(
+          (r) => (r.payload as any).subtask_id === subtaskId
+        );
+
+        if (resolution) {
+          const resolutionTime = new Date(resolution.timestamp).getTime();
+          const deltaMs = resolutionTime - blockerTime;
+          responseTimes.push(deltaMs);
+        }
+      }
+
+      if (responseTimes.length === 0) {
+        return {
+          score: 0.5,
+          message: `${blockers.length} blocker(s) detected, ${resolutions.length} resolution(s), but no matches by subtask_id`,
+        };
+      }
+
+      // Calculate average response time
+      const avgResponseMs =
+        responseTimes.reduce((sum, t) => sum + t, 0) / responseTimes.length;
+
+      // Normalize: < 5min = 1.0, > 15min = 0.0, linear in between
+      const EXCELLENT_MS = 5 * 60 * 1000; // 5 min
+      const POOR_MS = 15 * 60 * 1000; // 15 min
+
+      let score: number;
+      if (avgResponseMs < EXCELLENT_MS) {
+        score = 1.0;
+      } else if (avgResponseMs > POOR_MS) {
+        score = 0.0;
+      } else {
+        // Linear decay from 1.0 to 0.0
+        score = 1.0 - (avgResponseMs - EXCELLENT_MS) / (POOR_MS - EXCELLENT_MS);
+      }
+
+      const avgMinutes = Math.round(avgResponseMs / 1000 / 60);
+
+      return {
+        score,
+        message: `Avg response time: ${avgMinutes}min (${responseTimes.length}/${blockers.length} blockers resolved)`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse CoordinatorSession: ${error}`,
+      };
+    }
+  },
+});
+
 /**
  * Overall Discipline Scorer
  *
@@ -270,14 +603,14 @@ export const timeToFirstSpawn = createScorer({
 export const overallDiscipline = createScorer({
   name: "Overall Coordinator Discipline",
   description: "Composite score for coordinator protocol adherence",
-  scorer: ({ output, expected }) => {
+  scorer: async ({ output, expected, input }) => {
     try {
       // Run all scorers
       const scores = {
-        violations: violationCount.scorer({ output, expected }),
-        spawn: spawnEfficiency.scorer({ output, expected }),
-        review: reviewThoroughness.scorer({ output, expected }),
-        speed: timeToFirstSpawn.scorer({ output, expected }),
+        violations: await violationCount({ output, expected, input }),
+        spawn: await spawnEfficiency({ output, expected, input }),
+        review: await reviewThoroughness({ output, expected, input }),
+        speed: await timeToFirstSpawn({ output, expected, input }),
       };
 
       // Weighted average
@@ -289,16 +622,16 @@ export const overallDiscipline = createScorer({
       };
 
       const totalScore =
-        scores.violations.score * weights.violations +
-        scores.spawn.score * weights.spawn +
-        scores.review.score * weights.review +
-        scores.speed.score * weights.speed;
+        (scores.violations.score ?? 0) * weights.violations +
+        (scores.spawn.score ?? 0) * weights.spawn +
+        (scores.review.score ?? 0) * weights.review +
+        (scores.speed.score ?? 0) * weights.speed;
 
       const details = [
-        `Violations: ${(scores.violations.score * 100).toFixed(0)}%`,
-        `Spawn: ${(scores.spawn.score * 100).toFixed(0)}%`,
-        `Review: ${(scores.review.score * 100).toFixed(0)}%`,
-        `Speed: ${(scores.speed.score * 100).toFixed(0)}%`,
+        `Violations: ${((scores.violations.score ?? 0) * 100).toFixed(0)}%`,
+        `Spawn: ${((scores.spawn.score ?? 0) * 100).toFixed(0)}%`,
+        `Review: ${((scores.review.score ?? 0) * 100).toFixed(0)}%`,
+        `Speed: ${((scores.speed.score ?? 0) * 100).toFixed(0)}%`,
       ].join(", ");
 
       return {