opencode-swarm-plugin 0.40.0 → 0.42.0

package/evals/example.eval.ts

@@ -14,19 +14,18 @@ evalite("Example: Basic scorer test", {
   data: async () => {
     return [
       {
-        input: "Test task",
-        output: JSON.stringify({
+        input: {
           epic: { title: "Test Epic", description: "Test" },
           subtasks: [
             { title: "Subtask 1", files: ["a.ts"], estimated_complexity: 1 },
             { title: "Subtask 2", files: ["b.ts"], estimated_complexity: 1 },
           ],
-        }),
+        },
       },
     ];
   },
   task: async (input) => {
-    return input; // passthrough for testing
+    return JSON.stringify(input);
   },
   scorers: [subtaskIndependence],
 });

package/evals/fixtures/compaction-prompt-cases.ts

@@ -78,6 +78,8 @@ Coordinators do NOT edit code directly. These tools are FORBIDDEN:
 - edit
 - write
 - bash (for file modifications)
+- swarmmail_reserve (only workers reserve)
+- git commit (workers commit)
 
 Use swarm_spawn_subtask to delegate work to workers.
 
@@ -249,6 +251,8 @@ You are the COORDINATOR of epic mjkweh7q9n4.
 - edit
 - write
 - bash (for file mods)
+- swarmmail_reserve (only workers)
+- git commit (workers only)
 
 NEVER edit files yourself.
 ALWAYS delegate to workers.
@@ -289,6 +293,8 @@ You are coordinating epics:
 - edit
 - write  
 - bash
+- swarmmail_reserve
+- git commit
 
 ALWAYS check status first.
 NEVER edit files directly.

package/evals/scorers/coordinator-discipline.ts

@@ -334,259 +334,6 @@ export const timeToFirstSpawn = createScorer({
   },
 });
 
-/**
- * Researcher Spawn Rate Scorer
- *
- * Measures whether coordinator spawns researchers for unfamiliar technology.
- * Coordinators should delegate research instead of calling pdf-brain/context7 directly.
- *
- * Score: 1.0 if researcher_spawned events exist, 0.0 otherwise
- */
-export const researcherSpawnRate = createScorer({
-  name: "Researcher Spawn Rate",
-  description: "Coordinator spawned researchers for unfamiliar tech",
-  scorer: ({ output }) => {
-    try {
-      const session = JSON.parse(String(output)) as CoordinatorSession;
-
-      // Count researcher_spawned events
-      const researchers = session.events.filter(
-        (e) =>
-          e.event_type === "DECISION" && e.decision_type === "researcher_spawned"
-      );
-
-      const count = researchers.length;
-
-      if (count === 0) {
-        return {
-          score: 0.0,
-          message: "No researchers spawned (may indicate coordinator queried docs directly)",
-        };
-      }
-
-      return {
-        score: 1.0,
-        message: `${count} researcher(s) spawned`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse CoordinatorSession: ${error}`,
-      };
-    }
-  },
-});
-
-/**
- * Skill Loading Rate Scorer
- *
- * Measures whether coordinator loads relevant skills via skills_use().
- * Shows knowledge-seeking behavior.
- *
- * Score: 1.0 if skill_loaded events exist, 0.5 otherwise (not critical, but helpful)
- */
-export const skillLoadingRate = createScorer({
-  name: "Skill Loading Rate",
-  description: "Coordinator loaded relevant skills for domain knowledge",
-  scorer: ({ output }) => {
-    try {
-      const session = JSON.parse(String(output)) as CoordinatorSession;
-
-      // Count skill_loaded events
-      const skills = session.events.filter(
-        (e) =>
-          e.event_type === "DECISION" && e.decision_type === "skill_loaded"
-      );
-
-      const count = skills.length;
-
-      if (count === 0) {
-        return {
-          score: 0.5,
-          message: "No skills loaded (not critical, but helpful)",
-        };
-      }
-
-      return {
-        score: 1.0,
-        message: `${count} skill(s) loaded`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse CoordinatorSession: ${error}`,
-      };
-    }
-  },
-});
-
-/**
- * Inbox Monitoring Rate Scorer
- *
- * Measures how frequently coordinator checks inbox for worker messages.
- * Regular monitoring (every ~15min or when workers finish) shows good coordination.
- *
- * Score based on inbox_checked events relative to worker activity:
- * - 0 checks = 0.0 (coordinator not monitoring)
- * - 1+ checks = 1.0 (coordinator is responsive)
- */
-export const inboxMonitoringRate = createScorer({
-  name: "Inbox Monitoring Rate",
-  description: "Coordinator checked inbox regularly for worker messages",
-  scorer: ({ output }) => {
-    try {
-      const session = JSON.parse(String(output)) as CoordinatorSession;
-
-      // Count inbox_checked events
-      const checks = session.events.filter(
-        (e) =>
-          e.event_type === "DECISION" && e.decision_type === "inbox_checked"
-      );
-
-      // Count worker activity (spawns + outcomes)
-      const workerActivity = session.events.filter(
-        (e) =>
-          (e.event_type === "DECISION" && e.decision_type === "worker_spawned") ||
-          (e.event_type === "OUTCOME" &&
-            ["subtask_success", "subtask_failed", "blocker_detected"].includes(
-              e.outcome_type
-            ))
-      );
-
-      const checkCount = checks.length;
-      const activityCount = workerActivity.length;
-
-      if (activityCount === 0) {
-        return {
-          score: 1.0,
-          message: "No worker activity to monitor",
-        };
-      }
-
-      if (checkCount === 0) {
-        return {
-          score: 0.0,
-          message: `${activityCount} worker events, 0 inbox checks (not monitoring)`,
-        };
-      }
-
-      return {
-        score: 1.0,
-        message: `${checkCount} inbox check(s) for ${activityCount} worker events`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse CoordinatorSession: ${error}`,
-      };
-    }
-  },
-});
-
-/**
- * Blocker Response Time Scorer
- *
- * Measures how quickly coordinator responds to blocked workers.
- * Time between blocker_detected (OUTCOME) and blocker_resolved (DECISION).
- *
- * Normalization:
- * - < 5min: 1.0 (excellent)
- * - 5-15min: linear decay to 0.5
- * - > 15min: 0.0 (too slow, worker is idle)
- *
- * Score: Average response time across all blockers
- */
-export const blockerResponseTime = createScorer({
-  name: "Blocker Response Time",
-  description: "Coordinator unblocked workers quickly",
-  scorer: ({ output }) => {
-    try {
-      const session = JSON.parse(String(output)) as CoordinatorSession;
-
-      // Find blocker_detected events
-      const blockers = session.events.filter(
-        (e) =>
-          e.event_type === "OUTCOME" && e.outcome_type === "blocker_detected"
-      );
-
-      if (blockers.length === 0) {
-        return {
-          score: 1.0,
-          message: "No blockers detected",
-        };
-      }
-
-      // Find blocker_resolved events
-      const resolutions = session.events.filter(
-        (e) =>
-          e.event_type === "DECISION" && e.decision_type === "blocker_resolved"
-      );
-
-      if (resolutions.length === 0) {
-        return {
-          score: 0.0,
-          message: `${blockers.length} blocker(s) detected, 0 resolved (workers still blocked)`,
-        };
-      }
-
-      // Match blockers to resolutions by subtask_id and calculate response times
-      const responseTimes: number[] = [];
-      for (const blocker of blockers) {
-        const subtaskId = (blocker.payload as any).subtask_id;
-        const blockerTime = new Date(blocker.timestamp).getTime();
-
-        // Find resolution for this subtask
-        const resolution = resolutions.find(
-          (r) => (r.payload as any).subtask_id === subtaskId
-        );
-
-        if (resolution) {
-          const resolutionTime = new Date(resolution.timestamp).getTime();
-          const deltaMs = resolutionTime - blockerTime;
-          responseTimes.push(deltaMs);
-        }
-      }
-
-      if (responseTimes.length === 0) {
-        return {
-          score: 0.5,
-          message: `${blockers.length} blocker(s) detected, ${resolutions.length} resolution(s), but no matches by subtask_id`,
-        };
-      }
-
-      // Calculate average response time
-      const avgResponseMs =
-        responseTimes.reduce((sum, t) => sum + t, 0) / responseTimes.length;
-
-      // Normalize: < 5min = 1.0, > 15min = 0.0, linear in between
-      const EXCELLENT_MS = 5 * 60 * 1000; // 5 min
-      const POOR_MS = 15 * 60 * 1000; // 15 min
-
-      let score: number;
-      if (avgResponseMs < EXCELLENT_MS) {
-        score = 1.0;
-      } else if (avgResponseMs > POOR_MS) {
-        score = 0.0;
-      } else {
-        // Linear decay from 1.0 to 0.0
-        score = 1.0 - (avgResponseMs - EXCELLENT_MS) / (POOR_MS - EXCELLENT_MS);
-      }
-
-      const avgMinutes = Math.round(avgResponseMs / 1000 / 60);
-
-      return {
-        score,
-        message: `Avg response time: ${avgMinutes}min (${responseTimes.length}/${blockers.length} blockers resolved)`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse CoordinatorSession: ${error}`,
-      };
-    }
-  },
-});
-
 /**
  * Overall Discipline Scorer
  *

package/evals/swarm-decomposition.eval.ts

@@ -34,7 +34,9 @@ import {
 } from "./lib/data-loader.js";
 
 // Determine project key from current directory
-const PROJECT_KEY = "opencode-swarm-plugin";
+// NOTE: project_key in eval_records is the full path (from getHiveWorkingDirectory),
+// not a short name. Use process.cwd() to match.
+const PROJECT_KEY = process.cwd();
 const PROJECT_PATH = process.cwd();
 
 // Check if we have enough real data to use instead of fixtures
@@ -42,7 +44,7 @@ const useRealData = await hasRealEvalData(PROJECT_KEY, 5, PROJECT_PATH);
 
 // Load data based on availability
 const evalCases = useRealData
-  ? await loadEvalCases(PROJECT_KEY, { limit: 20, projectPath: PROJECT_PATH })
+  ? await loadEvalCases(PROJECT_KEY, { limit: 20, projectPath: PROJECT_PATH })  // PROJECT_KEY is now process.cwd()
   : decompositionCases.map((testCase) => ({
       input: testCase.input,
       expected: testCase.expected,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-swarm-plugin",
-  "version": "0.40.0",
+  "version": "0.42.0",
   "description": "Multi-agent swarm coordination for OpenCode with learning capabilities, beads integration, and Agent Mail",
   "type": "module",
   "main": "./dist/index.js",
@@ -23,7 +23,7 @@
     "registry": "https://registry.npmjs.org/"
   },
   "scripts": {
-    "build": "bun build ./src/index.ts --outdir ./dist --target node --external @electric-sql/pglite --external swarm-mail && bun build ./src/plugin.ts --outfile ./dist/plugin.js --target node --external @electric-sql/pglite --external swarm-mail && tsc",
+    "build": "bun build ./src/index.ts --outdir ./dist --target node --external @electric-sql/pglite --external swarm-mail --external vitest --external @vitest/ui --external lightningcss && bun build ./src/plugin.ts --outfile ./dist/plugin.js --target node --external @electric-sql/pglite --external swarm-mail --external vitest --external @vitest/ui --external lightningcss && tsc",
     "dev": "bun --watch src/index.ts",
     "test": "bun test --timeout 10000 src/anti-patterns.test.ts src/mandate-promotion.test.ts src/mandate-storage.test.ts src/output-guardrails.test.ts src/pattern-maturity.test.ts src/skills.test.ts src/structured.test.ts src/schemas/",
     "test:integration": "bun test --timeout 60000 src/*.integration.test.ts",
@@ -34,6 +34,7 @@
     "eval:decomposition": "bun --env-file=.env run bunx evalite run evals/swarm-decomposition.eval.ts",
     "eval:coordinator": "bun --env-file=.env run bunx evalite run evals/coordinator-session.eval.ts",
     "eval:compaction": "bun --env-file=.env run bunx evalite run evals/compaction-prompt.eval.ts",
+    "eval:gate": "bun run bin/eval-gate.ts",
     "migrate:sessions": "bun run scripts/migrate-unknown-sessions.ts",
     "postinstall": "node -e \"console.log('\\n\\x1b[33m  Run \\x1b[36mswarm setup\\x1b[33m to configure OpenCode integration\\x1b[0m\\n')\""
   },
@@ -46,7 +47,7 @@
     "minimatch": "^10.1.1",
     "pino": "^9.6.0",
     "pino-roll": "^1.3.0",
-    "swarm-mail": "1.5.2",
+    "swarm-mail": "1.5.4",
     "yaml": "^2.8.2",
     "zod": "4.1.8"
   },

package/src/compaction-prompt-scorers.test.ts

@@ -173,16 +173,17 @@ describe("forbiddenToolsPresent scorer", () => {
 - Edit (use swarm_spawn_subtask)
 - Write (use swarm_spawn_subtask)
 - swarmmail_reserve (only workers reserve)
-- bash with git commit (workers commit)`,
+- git commit (workers commit)
+- bash (for file modifications)`,
 		};
 
 		const result = scoreForbiddenToolsPresent(prompt);
 
 		expect(result.score).toBe(1.0);
-		expect(result.message).toContain("All 4 forbidden tools");
+		expect(result.message).toContain("All 5 forbidden tools");
 	});
 
-	test("scores 0.75 when 3 out of 4 tools listed", () => {
+	test("scores 0.6 when 3 out of 5 tools listed", () => {
 		const prompt: CompactionPrompt = {
 			content: `🚫 FORBIDDEN TOOLS:
 - Edit
@@ -192,19 +193,19 @@ describe("forbiddenToolsPresent scorer", () => {
 
 		const result = scoreForbiddenToolsPresent(prompt);
 
-		expect(result.score).toBe(0.75);
-		expect(result.message).toContain("3/4");
+		expect(result.score).toBe(0.6);
+		expect(result.message).toContain("3/5");
 	});
 
-	test("scores 0.5 when 2 out of 4 tools listed", () => {
+	test("scores 0.4 when 2 out of 5 tools listed", () => {
 		const prompt: CompactionPrompt = {
 			content: `Don't use Edit or Write directly.`,
 		};
 
 		const result = scoreForbiddenToolsPresent(prompt);
 
-		expect(result.score).toBe(0.5);
-		expect(result.message).toContain("2/4");
+		expect(result.score).toBe(0.4);
+		expect(result.message).toContain("2/5");
 	});
 
 	test("scores 0.0 when no forbidden tools listed", () => {
@@ -215,7 +216,7 @@ describe("forbiddenToolsPresent scorer", () => {
 		const result = scoreForbiddenToolsPresent(prompt);
 
 		expect(result.score).toBe(0.0);
-		expect(result.message).toContain("0/4");
+		expect(result.message).toContain("0/5");
 	});
 });
 

package/src/compaction-prompt-scoring.ts

@@ -203,6 +203,7 @@ export function scoreCoordinatorIdentity(
  * 2. Write
  * 3. swarmmail_reserve (only workers reserve)
  * 4. git commit (workers commit)
+ * 5. bash (for file modifications)
  *
  * @returns ratio of forbidden tools mentioned (0.0 to 1.0)
  */
@@ -211,10 +212,11 @@ export function scoreForbiddenToolsPresent(
 ): ScorerResult {
 	// Check for forbidden tool mentions
 	const forbiddenTools = [
-		/\bEdit\b/,
-		/\bWrite\b/,
+		/\bEdit\b/i,
+		/\bWrite\b/i,
 		/swarmmail_reserve/,
 		/git commit/,
+		/\bbash\b/i,
 	];
 
 	const foundTools = forbiddenTools.filter((pattern) =>
@@ -226,20 +228,20 @@ export function scoreForbiddenToolsPresent(
 	if (score === 1.0) {
 		return {
 			score: 1.0,
-			message: "All 4 forbidden tools listed",
+			message: "All 5 forbidden tools listed",
 		};
 	}
 
 	if (score === 0) {
 		return {
 			score: 0.0,
-			message: "No forbidden tools listed (0/4)",
+			message: "No forbidden tools listed (0/5)",
 		};
 	}
 
 	return {
 		score,
-		message: `${foundTools.length}/4 forbidden tools listed`,
+		message: `${foundTools.length}/5 forbidden tools listed`,
 	};
 }
 

package/src/eval-runner.test.ts

@@ -4,9 +4,11 @@
  * TDD: These tests MUST fail initially, then pass after implementation.
  */
 
-import { describe, test, expect, beforeAll } from "bun:test";
+import { describe, test, expect, beforeAll, afterEach } from "bun:test";
 import { runEvals } from "./eval-runner";
 import path from "node:path";
+import fs from "node:fs";
+import { getEvalHistoryPath } from "./eval-history";
 
 // Use project root for all tests
 const PROJECT_ROOT = path.resolve(import.meta.dir, "..");
@@ -93,4 +95,129 @@ describe("runEvals", () => {
     expect(result.totalSuites).toBe(0);
     expect(result.suites).toEqual([]);
   }, 10000);
+
+  test("records eval run to history after execution", async () => {
+    // Clean up any existing history before test
+    const historyPath = getEvalHistoryPath(PROJECT_ROOT);
+    const historyBackup = historyPath + ".backup";
+    
+    // Backup existing history
+    if (fs.existsSync(historyPath)) {
+      fs.copyFileSync(historyPath, historyBackup);
+    }
+    
+    try {
+      // Remove history file to get clean state
+      if (fs.existsSync(historyPath)) {
+        fs.unlinkSync(historyPath);
+      }
+
+      // Run evals
+      const result = await runEvals({
+        cwd: PROJECT_ROOT,
+        suiteFilter: "example",
+      });
+
+      // Should have succeeded
+      expect(result.success).toBe(true);
+      expect(result.suites.length).toBeGreaterThan(0);
+
+      // History file should have been created
+      expect(fs.existsSync(historyPath)).toBe(true);
+
+      // Read history file
+      const historyContent = fs.readFileSync(historyPath, "utf-8");
+      const lines = historyContent.trim().split("\n");
+
+      // Should have one line per suite
+      expect(lines.length).toBe(result.suites.length);
+
+      // Parse first line and verify structure
+      const firstRecord = JSON.parse(lines[0]);
+      
+      // Verify structure has all required fields
+      expect(typeof firstRecord.timestamp).toBe("string");
+      expect(typeof firstRecord.eval_name).toBe("string");
+      expect(typeof firstRecord.score).toBe("number");
+      expect(typeof firstRecord.run_count).toBe("number");
+
+      // Verify eval_name matches suite name
+      expect(firstRecord.eval_name).toBe(result.suites[0].name);
+      
+      // Verify score matches suite averageScore
+      expect(firstRecord.score).toBe(result.suites[0].averageScore);
+      
+      // First run should have run_count = 1
+      expect(firstRecord.run_count).toBe(1);
+    } finally {
+      // Restore backup
+      if (fs.existsSync(historyBackup)) {
+        fs.copyFileSync(historyBackup, historyPath);
+        fs.unlinkSync(historyBackup);
+      }
+    }
+  }, 30000);
+
+  test("checks gates for each suite after recording", async () => {
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+      suiteFilter: "example",
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.gateResults).toBeDefined();
+    expect(Array.isArray(result.gateResults)).toBe(true);
+    
+    // Should have gate result for each suite
+    expect(result.gateResults?.length).toBe(result.suites.length);
+    
+    // Each gate result should have required fields
+    if (result.gateResults && result.gateResults.length > 0) {
+      const gateResult = result.gateResults[0];
+      expect(gateResult).toHaveProperty("suite");
+      expect(gateResult).toHaveProperty("passed");
+      expect(gateResult).toHaveProperty("phase");
+      expect(gateResult).toHaveProperty("message");
+      expect(gateResult).toHaveProperty("currentScore");
+    }
+  }, 30000);
+
+  test("calls learnFromEvalFailure when gate fails", async () => {
+    // This test requires manually creating a history with regression
+    // For now, we just verify the code path exists
+    // In practice, this would be tested with mocked checkGate returning failed=true
+    
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+      suiteFilter: "example",
+    });
+
+    // Gate results should be present even if no failures
+    expect(result.gateResults).toBeDefined();
+  }, 30000);
+
+  test("does NOT call learnFromEvalFailure when gate passes", async () => {
+    // Similar to above - verifies the happy path
+    // Real test would mock checkGate and verify learnFromEvalFailure NOT called
+    
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+      suiteFilter: "example",
+    });
+
+    // Should succeed with gate results
+    expect(result.success).toBe(true);
+    expect(result.gateResults).toBeDefined();
+  }, 30000);
+
+  test("includes gateResults in return value", async () => {
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+      suiteFilter: "example",
+    });
+
+    // gateResults should be array (even if empty)
+    expect(result).toHaveProperty("gateResults");
+    expect(Array.isArray(result.gateResults)).toBe(true);
+  }, 30000);
 });

package/src/eval-runner.ts

@@ -13,6 +13,10 @@ import { createInMemoryStorage } from "evalite/in-memory-storage";
 import type { Evalite } from "evalite/types";
 import fs from "node:fs/promises";
 import path from "node:path";
+import { recordEvalRun, getScoreHistory } from "./eval-history.js";
+import { checkGate } from "./eval-gates.js";
+import { learnFromEvalFailure } from "./eval-learning.js";
+import { getMemoryAdapter } from "./memory-tools.js";
 
 /**
  * Options for running evals programmatically
@@ -97,6 +101,17 @@ export interface RunEvalsResult {
   
   /** Error message if run failed */
   error?: string;
+  
+  /** Gate check results per suite */
+  gateResults?: Array<{
+    suite: string;
+    passed: boolean;
+    phase: string;
+    message: string;
+    baseline?: number;
+    currentScore: number;
+    regressionPercent?: number;
+  }>;
 }
 
 /**
@@ -246,6 +261,36 @@ export async function runEvals(
       })),
     }));
 
+    // Record eval runs to history
+    for (const suite of suites) {
+      const history = getScoreHistory(projectRoot, suite.name);
+      recordEvalRun(projectRoot, {
+        timestamp: new Date().toISOString(),
+        eval_name: suite.name,
+        score: suite.averageScore,
+        run_count: history.length + 1,
+      });
+    }
+
+    // Check gates for each suite
+    const gateResults = [];
+    for (const suite of suites) {
+      const history = getScoreHistory(projectRoot, suite.name);
+      const gate = checkGate(projectRoot, suite.name, suite.averageScore);
+      gateResults.push({ suite: suite.name, ...gate });
+      
+      // If gate failed, trigger learning
+      if (!gate.passed) {
+        try {
+          const memoryAdapter = await getMemoryAdapter();
+          await learnFromEvalFailure(suite.name, suite.averageScore, history, memoryAdapter);
+        } catch (e) {
+          // Learning is best-effort, don't fail the eval run
+          console.warn(`Failed to store learning for ${suite.name}:`, e);
+        }
+      }
+    }
+
     // Calculate overall metrics
     const totalEvals = suites.reduce((sum, s) => sum + s.evalCount, 0);
     const averageScore =
@@ -263,6 +308,7 @@ export async function runEvals(
       totalEvals,
       averageScore,
       suites,
+      gateResults,
     };
   } catch (error) {
     // Return error result