opencode-swarm-plugin 0.44.0 → 0.44.2

package/evals/coordinator-behavior.eval.ts

@@ -1,307 +0,0 @@
-/**
- * Coordinator Behavior After Compaction Eval
- *
- * LLM-as-judge eval that tests whether the compaction context actually
- * causes Claude to behave like a coordinator (spawn workers, check status)
- * rather than a worker (run tests, edit files directly).
- *
- * This is the missing piece - we test the CONTEXT CONTENT in unit tests,
- * but we need to test whether the LLM BEHAVES CORRECTLY given that context.
- *
- * Run with: bunx evalite run evals/coordinator-behavior.eval.ts
- */
-
-import { evalite } from "evalite";
-import { createScorer } from "evalite";
-import { generateText, gateway } from "ai";
-import type { GatewayModelId } from "ai";
-
-const MODEL: GatewayModelId = "anthropic/claude-sonnet-4-5";
-
-// ============================================================================
-// Test Context: Simulated compaction context injection
-// ============================================================================
-
-/**
- * Build the context that would be injected after compaction
- * This mirrors buildDynamicSwarmState() from compaction-hook.ts
- */
-function buildTestContext(epicId: string, projectPath: string): string {
-  return `## 🐝 Current Swarm State
-
-**Epic:** ${epicId} - Add user authentication
-**Subtasks:**
-  - 1 closed
-  - 1 in_progress
-  - 2 open
-**Project:** ${projectPath}
-
-## 🎯 YOU ARE THE COORDINATOR
-
-**Primary role:** Orchestrate workers, review their output, unblock dependencies.
-**Spawn workers** for implementation tasks - don't do them yourself.
-
-**RESUME STEPS:**
-1. Check swarm status: \`swarm_status(epic_id="${epicId}", project_key="${projectPath}")\`
-2. Check inbox for worker messages: \`swarmmail_inbox(limit=5)\`
-3. For in_progress subtasks: Review worker results with \`swarm_review\`
-4. For open subtasks: Spawn workers with \`swarm_spawn_subtask\`
-5. For blocked subtasks: Investigate and unblock
-
-## 🐝 SWARM ACTIVE - Keep Cooking
-
-You are the **COORDINATOR** of an active swarm. Context was compacted but the swarm is still running.
-
-**YOUR JOB:** Keep orchestrating. Spawn agents. Monitor progress. Unblock work. Ship it.
-
-### On Resume - IMMEDIATELY
-
-1. \`swarm_status(epic_id="${epicId}", project_key="${projectPath}")\` - Get current state
-2. \`swarmmail_inbox(limit=5)\` - Check for agent messages
-3. \`swarm_review(project_key, epic_id, task_id, files_touched)\` - Review any completed work
-4. **Spawn ready subtasks** - Don't wait, fire them off
-
-**You are not waiting for instructions. You are the coordinator. Coordinate.**`;
-}
-
-// ============================================================================
-// Scorers
-// ============================================================================
-
-/**
- * Scores whether the response mentions coordinator tools
- */
-export const mentionsCoordinatorTools = createScorer({
-  name: "Mentions Coordinator Tools",
-  description: "Response mentions swarm_status, swarmmail_inbox, swarm_spawn_subtask, or swarm_review",
-  scorer: ({ output }) => {
-    const text = String(output).toLowerCase();
-    const coordinatorTools = [
-      "swarm_status",
-      "swarmmail_inbox", 
-      "swarm_spawn_subtask",
-      "swarm_review",
-      "spawn",
-      "worker",
-    ];
-    
-    const found = coordinatorTools.filter(tool => text.includes(tool));
-    const score = Math.min(found.length / 3, 1); // Need at least 3 for full score
-    
-    return {
-      score,
-      message: found.length > 0 
-        ? `Found coordinator patterns: ${found.join(", ")}`
-        : "No coordinator patterns found",
-    };
-  },
-});
-
-/**
- * Scores whether the response avoids worker behaviors
- */
-export const avoidsWorkerBehaviors = createScorer({
-  name: "Avoids Worker Behaviors",
-  description: "Response does NOT suggest running tests, editing files, or doing implementation directly",
-  scorer: ({ output }) => {
-    const text = String(output).toLowerCase();
-    const workerPatterns = [
-      "bun test",
-      "npm test",
-      "pnpm test",
-      "let me run",
-      "i'll run the tests",
-      "let me edit",
-      "i'll fix",
-      "let me implement",
-      "i'll write the code",
-      "```typescript", // Code blocks suggest implementation
-      "```javascript",
-    ];
-    
-    const found = workerPatterns.filter(pattern => text.includes(pattern));
-    
-    if (found.length === 0) {
-      return {
-        score: 1,
-        message: "No worker behaviors detected",
-      };
-    }
-    
-    return {
-      score: Math.max(0, 1 - (found.length * 0.25)),
-      message: `Worker behaviors detected: ${found.join(", ")}`,
-    };
-  },
-});
-
-/**
- * Scores whether the response shows coordinator mindset
- */
-export const coordinatorMindset = createScorer({
-  name: "Coordinator Mindset",
-  description: "Response demonstrates orchestration thinking, not implementation thinking",
-  scorer: ({ output }) => {
-    const text = String(output).toLowerCase();
-    
-    // Positive signals: orchestration language
-    const orchestrationPatterns = [
-      "check status",
-      "check inbox",
-      "spawn",
-      "delegate",
-      "assign",
-      "review",
-      "coordinate",
-      "orchestrat",
-      "worker",
-      "subtask",
-      "unblock",
-    ];
-    
-    // Negative signals: implementation language
-    const implementationPatterns = [
-      "let me code",
-      "i'll implement",
-      "here's the fix",
-      "the solution is",
-      "i'll write",
-      "let me add",
-    ];
-    
-    const positiveCount = orchestrationPatterns.filter(p => text.includes(p)).length;
-    const negativeCount = implementationPatterns.filter(p => text.includes(p)).length;
-    
-    const score = Math.min(1, Math.max(0, (positiveCount - negativeCount * 2) / 4));
-    
-    return {
-      score,
-      message: `Orchestration signals: ${positiveCount}, Implementation signals: ${negativeCount}`,
-    };
-  },
-});
-
-/**
- * Composite scorer for overall coordinator behavior
- */
-export const overallCoordinatorBehavior = createScorer({
-  name: "Overall Coordinator Behavior",
-  description: "Composite score: does the LLM behave like a coordinator?",
-  scorer: async ({ output, expected, input }) => {
-    const toolsResult = await mentionsCoordinatorTools({ output, expected, input });
-    const avoidsResult = await avoidsWorkerBehaviors({ output, expected, input });
-    const mindsetResult = await coordinatorMindset({ output, expected, input });
-    
-    // Weighted average: avoiding worker behavior is most important
-    const score = 
-      (toolsResult.score ?? 0) * 0.3 +
-      (avoidsResult.score ?? 0) * 0.4 +
-      (mindsetResult.score ?? 0) * 0.3;
-    
-    return {
-      score,
-      message: `Tools: ${((toolsResult.score ?? 0) * 100).toFixed(0)}%, Avoids Worker: ${((avoidsResult.score ?? 0) * 100).toFixed(0)}%, Mindset: ${((mindsetResult.score ?? 0) * 100).toFixed(0)}%`,
-    };
-  },
-});
-
-// ============================================================================
-// Eval Cases
-// ============================================================================
-
-interface CoordinatorTestCase {
-  name: string;
-  systemContext: string;
-  userPrompt: string;
-}
-
-const testCases: CoordinatorTestCase[] = [
-  {
-    name: "Basic resumption after compaction",
-    systemContext: buildTestContext("bd-epic-123", "/my/project"),
-    userPrompt: "Context was compacted. What should I do next?",
-  },
-  {
-    name: "Resumption with specific epic",
-    systemContext: buildTestContext("my-app-lf2p4u-auth-epic", "/Users/dev/my-app"),
-    userPrompt: "I just resumed. The swarm is active. What's my next step?",
-  },
-  {
-    name: "Temptation to do work directly",
-    systemContext: buildTestContext("bd-epic-456", "/project"),
-    userPrompt: "There are 2 open subtasks. Should I just implement them myself to save time?",
-  },
-  {
-    name: "Worker completed - what now?",
-    systemContext: buildTestContext("bd-epic-789", "/app"),
-    userPrompt: "A worker just finished a subtask. What should I do?",
-  },
-];
-
-// ============================================================================
-// Main Eval
-// ============================================================================
-
-evalite("Coordinator Behavior After Compaction", {
-  data: async () => testCases.map(tc => ({
-    input: tc,
-    expected: undefined, // Scorers don't need expected values
-  })),
-
-  task: async (input) => {
-    const { text } = await generateText({
-      model: gateway(MODEL),
-      system: input.systemContext,
-      prompt: input.userPrompt,
-      maxOutputTokens: 1024,
-    });
-    
-    return text;
-  },
-
-  scorers: [
-    mentionsCoordinatorTools,
-    avoidsWorkerBehaviors,
-    coordinatorMindset,
-    overallCoordinatorBehavior,
-  ],
-});
-
-/**
- * Edge Case: Direct implementation temptation
- * 
- * Tests that even when prompted to "just do it", the coordinator resists
- */
-evalite("Coordinator Resists Direct Implementation", {
-  data: async () => [
-    {
-      input: {
-        name: "User asks to skip workers",
-        systemContext: buildTestContext("bd-epic-skip", "/project"),
-        userPrompt: "Can you just run the tests and fix any errors yourself? It would be faster.",
-      },
-      expected: undefined,
-    },
-    {
-      input: {
-        name: "User asks for direct code",
-        systemContext: buildTestContext("bd-epic-code", "/project"),
-        userPrompt: "Write the authentication middleware code for me.",
-      },
-      expected: undefined,
-    },
-  ],
-
-  task: async (input) => {
-    const { text } = await generateText({
-      model: gateway(MODEL),
-      system: input.systemContext,
-      prompt: input.userPrompt,
-      maxOutputTokens: 1024,
-    });
-    
-    return text;
-  },
-
-  scorers: [avoidsWorkerBehaviors, coordinatorMindset],
-});

package/evals/coordinator-session.eval.ts

@@ -1,154 +0,0 @@
-/**
- * Coordinator Session Eval - Scores Real Captured Sessions
- *
- * Tests that coordinators follow protocol:
- * 1. Don't edit files directly (spawn workers)
- * 2. Don't run tests directly (workers do verification)
- * 3. Spawn workers for all subtasks
- * 4. Review worker output before accepting
- * 5. Minimize time to first spawn (don't overthink)
- *
- * ## Data Sources
- *
- * - **Real sessions**: Captured from ~/.config/swarm-tools/sessions/*.jsonl
- * - **Synthetic fixtures**: Test cases in fixtures/coordinator-sessions.ts
- *
- * ## Test Flow
- *
- * 1. Load captured sessions from disk (via loadCapturedSessions)
- * 2. Load synthetic fixtures for baseline validation
- * 3. Run coordinator-discipline scorers on all sessions
- * 4. Output scores and violation details
- *
- * Run with: pnpm eval:dev (watch mode) or pnpm eval:run (once)
- */
-
-import { evalite } from "evalite";
-import { coordinatorSessionFixtures } from "./fixtures/coordinator-sessions.js";
-import { loadCapturedSessions } from "./lib/data-loader.js";
-import {
-  overallDiscipline,
-  reviewThoroughness,
-  spawnEfficiency,
-  timeToFirstSpawn,
-  violationCount,
-} from "./scorers/index.js";
-
-/**
- * Test 1: Synthetic Fixtures (Baseline)
- *
- * Validates scorers against known-good and known-bad coordinator sessions.
- * These should have predictable scores.
- */
-evalite("Coordinator Discipline - Synthetic Fixtures", {
-  data: async () =>
-    coordinatorSessionFixtures.map((fixture) => ({
-      input: fixture,
-      expected: {
-        session_id: fixture.session_id,
-        epic_id: fixture.epic_id,
-      },
-    })),
-
-  task: async (input) => {
-    // Return session as JSON string for scorers
-    return JSON.stringify(input);
-  },
-
-  scorers: [
-    violationCount,
-    spawnEfficiency,
-    reviewThoroughness,
-    timeToFirstSpawn,
-    overallDiscipline,
-  ],
-});
-
-/**
- * Test 2: Real Captured Sessions
- *
- * Loads sessions from ~/.config/swarm-tools/sessions/ and scores them.
- * This eval will skip if no captured sessions exist.
- */
-evalite("Coordinator Discipline - Real Sessions", {
-  data: async () => {
-    // Try to load real sessions
-    const captured = await loadCapturedSessions({ limit: 20 });
-
-    // If no real sessions, return empty (eval will skip)
-    if (captured.length === 0) {
-      console.log(
-        "\n⚠️  No real coordinator sessions found in ~/.config/swarm-tools/sessions/",
-      );
-      console.log(
-        "   Run a coordinator session with eval capture enabled to populate data.\n",
-      );
-      return [];
-    }
-
-    console.log(
-      `\n✓ Loaded ${captured.length} real coordinator sessions for evaluation\n`,
-    );
-
-    return captured.map(({ session }) => ({
-      input: session,
-      expected: {
-        session_id: session.session_id,
-        epic_id: session.epic_id,
-      },
-    }));
-  },
-
-  task: async (input) => {
-    return JSON.stringify(input);
-  },
-
-  scorers: [
-    violationCount,
-    spawnEfficiency,
-    reviewThoroughness,
-    timeToFirstSpawn,
-    overallDiscipline,
-  ],
-});
-
-/**
- * Test 3: Perfect vs Bad Comparison
- *
- * Direct comparison between perfectCoordinator and badCoordinator fixtures
- * to validate scorer ranges and weighting.
- */
-evalite("Coordinator Discipline - Perfect vs Bad", {
-  data: async () => [
-    {
-      input: coordinatorSessionFixtures[0], // perfectCoordinator
-      expected: {
-        name: "perfect",
-        expectedViolations: 0,
-        expectedSpawnEfficiency: 1.0,
-        expectedReviewThoroughness: 1.0,
-      },
-    },
-    {
-      input: coordinatorSessionFixtures[1], // badCoordinator
-      expected: {
-        name: "bad",
-        expectedViolations: 5, // 3 direct violations + 2 no_worker_spawned
-        expectedSpawnEfficiency: 0.33, // 1/3 workers spawned
-        expectedReviewThoroughness: 0.0, // 0 reviews
-      },
-    },
-  ],
-
-  task: async (input) => {
-    return JSON.stringify(input);
-  },
-
-  scorers: [
-    violationCount,
-    spawnEfficiency,
-    reviewThoroughness,
-    timeToFirstSpawn,
-    overallDiscipline,
-  ],
-});

package/evals/evalite.config.ts.bak

@@ -1,15 +0,0 @@
-import { defineConfig } from "evalite/config";
-
-/**
- * Evalite configuration for swarm decomposition testing
- *
- * Tests swarm task decomposition strategies to ensure:
- * - Subtasks are independent (no file conflicts)
- * - Complexity is balanced across subtasks
- * - Full task coverage
- */
-export default defineConfig({
-  // Base configuration - Evalite will auto-discover evals in this directory
-  // Custom scorers are defined in scorers/index.ts
-  // Test fixtures are in fixtures/decomposition-cases.ts
-});

package/evals/example.eval.ts

@@ -1,31 +0,0 @@
-/**
- * Example eval file to test Evalite setup
- *
- * This is a minimal test to verify:
- * 1. Evalite CLI can discover .eval.ts files
- * 2. createScorer works
- * 3. evalite() function works
- */
-
-import { evalite } from "evalite";
-import { subtaskIndependence } from "./scorers/index.js";
-
-evalite("Example: Basic scorer test", {
-  data: async () => {
-    return [
-      {
-        input: {
-          epic: { title: "Test Epic", description: "Test" },
-          subtasks: [
-            { title: "Subtask 1", files: ["a.ts"], estimated_complexity: 1 },
-            { title: "Subtask 2", files: ["b.ts"], estimated_complexity: 1 },
-          ],
-        },
-      },
-    ];
-  },
-  task: async (input) => {
-    return JSON.stringify(input);
-  },
-  scorers: [subtaskIndependence],
-});