opencode-swarm-plugin 0.20.0 → 0.22.0

package/evals/swarm-decomposition.eval.ts

@@ -0,0 +1,112 @@
+/**
+ * Swarm Decomposition Quality Eval
+ *
+ * Tests the quality of task decomposition for swarm coordination.
+ * Uses real LLM calls via AI SDK + Vercel AI Gateway.
+ *
+ * Scorers evaluate:
+ * - Subtask independence (no file conflicts)
+ * - Complexity balance (even distribution)
+ * - Coverage completeness (all required files)
+ * - Instruction clarity (actionable descriptions)
+ *
+ * Run with: pnpm evalite evals/swarm-decomposition.eval.ts
+ *
+ * Requires: ANTHROPIC_API_KEY environment variable
+ */
+import { evalite } from "evalite";
+import {
+  subtaskIndependence,
+  coverageCompleteness,
+  instructionClarity,
+} from "./scorers/index.js";
+import { decompositionCases } from "./fixtures/decomposition-cases.js";
+import {
+  generateDecomposition,
+  formatDecompositionPrompt,
+  extractJson,
+} from "./lib/llm.js";
+import {
+  loadEvalCases,
+  hasRealEvalData,
+  getEvalDataSummary,
+} from "./lib/data-loader.js";
+
+// Determine project key from current directory
+const PROJECT_KEY = "opencode-swarm-plugin";
+const PROJECT_PATH = process.cwd();
+
+// Check if we have enough real data to use instead of fixtures
+const useRealData = await hasRealEvalData(PROJECT_KEY, 5, PROJECT_PATH);
+
+// Load data based on availability
+const evalCases = useRealData
+  ? await loadEvalCases(PROJECT_KEY, { limit: 20, projectPath: PROJECT_PATH })
+  : decompositionCases.map((testCase) => ({
+      input: testCase.input,
+      expected: testCase.expected,
+    }));
+
+// Log data source for transparency
+if (useRealData) {
+  const summary = await getEvalDataSummary(PROJECT_KEY, PROJECT_PATH);
+  console.log(`[eval] Using real data from PGlite:`);
+  console.log(`  - Total records: ${summary.totalRecords}`);
+  console.log(`  - Success rate: ${(summary.successRate * 100).toFixed(1)}%`);
+  console.log(
+    `  - Strategies: ${Object.entries(summary.byStrategy)
+      .map(([s, c]) => `${s}(${c})`)
+      .join(", ")}`,
+  );
+  console.log(`  - Eval cases: ${evalCases.length}`);
+} else {
+  console.log(
+    `[eval] Using fixture data (${evalCases.length} cases) - not enough real data yet`,
+  );
+}
+
+/**
+ * Swarm Decomposition Quality Eval
+ *
+ * Tests decomposition quality with real LLM calls.
+ */
+evalite("Swarm Decomposition Quality", {
+  // Test data from PGlite or fixtures
+  data: async () => evalCases,
+
+  // Task: generate real decomposition via Claude
+  task: async (input) => {
+    const prompt = formatDecompositionPrompt(input.task, input.context);
+    const response = await generateDecomposition(prompt);
+    return extractJson(response);
+  },
+
+  // Scorers evaluate decomposition quality
+  scorers: [subtaskIndependence, coverageCompleteness, instructionClarity],
+});
+
+/**
+ * Edge Case Eval: Minimal and Complex Tasks
+ *
+ * Tests handling of edge cases in decomposition.
+ */
+evalite("Decomposition Edge Cases", {
+  data: async () => [
+    {
+      input: { task: "Fix typo in README.md" },
+      expected: { minSubtasks: 1, maxSubtasks: 2 },
+    },
+    {
+      input: { task: "Refactor entire codebase from JavaScript to TypeScript" },
+      expected: { minSubtasks: 4, maxSubtasks: 8 },
+    },
+  ],
+
+  task: async (input) => {
+    const prompt = formatDecompositionPrompt(input.task, undefined, 8);
+    const response = await generateDecomposition(prompt);
+    return extractJson(response);
+  },
+
+  scorers: [subtaskIndependence, coverageCompleteness],
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "opencode-swarm-plugin",
-  "version": "0.20.0",
+  "version": "0.22.0",
   "description": "Multi-agent swarm coordination for OpenCode with learning capabilities, beads integration, and Agent Mail",
   "type": "module",
   "main": "dist/index.js",
@@ -27,6 +27,9 @@
     "test:all": "bun run test && bun run test:swarm",
     "typecheck": "tsc --noEmit",
     "clean": "rm -rf dist",
+    "eval:dev": "evalite watch evals/",
+    "eval:run": "evalite run evals/",
+    "eval:ci": "evalite run evals/ --threshold 80",
     "release": "npm run build && npm version patch && git push && npm run publish:otp",
     "release:minor": "npm run build && npm version minor && git push && npm run publish:otp",
     "release:major": "npm run build && npm version major && git push && npm run publish:otp",
@@ -41,11 +44,15 @@
     "gray-matter": "^4.0.3",
     "ioredis": "^5.4.1",
     "minimatch": "^10.1.1",
+    "nanoid": "^5.1.6",
     "zod": "4.1.8"
   },
   "devDependencies": {
     "@types/bun": "latest",
     "@types/minimatch": "^6.0.0",
+    "ai": "6.0.0-beta.150",
+    "bun-types": "^1.3.4",
+    "evalite": "^1.0.0-beta.10",
     "typescript": "^5.7.0",
     "vitest": "^4.0.15"
   },

package/scripts/cleanup-test-memories.ts

@@ -0,0 +1,346 @@
+#!/usr/bin/env bun
+/**
+ * Semantic Memory Test Pollution Cleanup
+ *
+ * This script audits and documents test pollution in semantic-memory storage.
+ * Test artifacts from integration tests pollute the production knowledge base,
+ * making semantic search unreliable and wasting storage.
+ *
+ * ROOT CAUSE:
+ * - Integration tests write to shared semantic-memory MCP server
+ * - No isolation between test and production collections
+ * - Tests don't clean up after themselves
+ * - No in-memory test mode available
+ *
+ * PREVENTION STRATEGY:
+ * 1. Test isolation via collection prefixes (test-*, temp-*)
+ * 2. Cleanup hooks in test teardown
+ * 3. Mock semantic-memory in unit tests
+ * 4. Document production collection names
+ *
+ * Usage:
+ *   bun scripts/cleanup-test-memories.ts [--dry-run] [--collections <prefix>]
+ *
+ * Examples:
+ *   bun scripts/cleanup-test-memories.ts --dry-run
+ *   bun scripts/cleanup-test-memories.ts --collections test-patterns,test-feedback
+ *   bun scripts/cleanup-test-memories.ts
+ */
+
+import { parseArgs } from "node:util";
+
+/** Test collection patterns to identify pollution */
+const TEST_COLLECTION_PATTERNS = [
+  "test-patterns",
+  "test-feedback",
+  /^test-.*/,
+  /^temp-.*/,
+] as const;
+
+interface Memory {
+  id: string;
+  collection: string;
+  content: string;
+  metadata?: string;
+  created_at?: string;
+}
+
+interface AuditReport {
+  total_memories: number;
+  test_artifacts: Memory[];
+  production_memories: Memory[];
+  collections: {
+    name: string;
+    count: number;
+    is_test: boolean;
+  }[];
+}
+
+/**
+ * Check if a collection name matches test patterns
+ */
+function isTestCollection(collection: string): boolean {
+  return TEST_COLLECTION_PATTERNS.some((pattern) => {
+    if (typeof pattern === "string") {
+      return collection === pattern;
+    }
+    return pattern.test(collection);
+  });
+}
+
+/**
+ * Parse semantic-memory_list output into structured data
+ *
+ * Output format is like:
+ * ```
+ * • 32577e43... (test-patterns)
+ *   {"id":"pattern-1765749526038-65vu4n","content":"Test pattern...
+ * • 825ccc37... (test-feedback)
+ *   {"id":"test-1765749524072-fs3i37vpoik","criterion":"type_safe"...
+ * ```
+ */
+function parseMemoryList(output: string): Memory[] {
+  const memories: Memory[] = [];
+  const lines = output.split("\n");
+
+  let currentMemory: Partial<Memory> | null = null;
+
+  for (const line of lines) {
+    // Match memory header: • 32577e43... (collection-name)
+    const headerMatch = line.match(/^•\s+([a-f0-9]+)\.\.\.\s+\(([^)]+)\)/);
+    if (headerMatch) {
+      if (currentMemory) {
+        memories.push(currentMemory as Memory);
+      }
+      currentMemory = {
+        id: headerMatch[1],
+        collection: headerMatch[2],
+        content: "",
+      };
+      continue;
+    }
+
+    // Match content line (indented JSON or text)
+    if (currentMemory && line.trim()) {
+      currentMemory.content = (
+        currentMemory.content +
+        " " +
+        line.trim()
+      ).trim();
+    }
+  }
+
+  if (currentMemory) {
+    memories.push(currentMemory as Memory);
+  }
+
+  return memories;
+}
+
+/**
+ * Audit semantic-memory for test pollution
+ *
+ * NOTE: This is a documentation-only script since semantic-memory MCP
+ * does not expose delete/remove APIs. The actual cleanup must be done
+ * manually via PostgreSQL.
+ */
+async function auditMemories(): Promise<AuditReport> {
+  console.log("🔍 Auditing semantic-memory for test pollution...\n");
+  console.log(
+    "⚠️  NOTE: semantic-memory_list is an MCP tool that must be called",
+  );
+  console.log("   by the AI agent, not from this script.\n");
+  console.log("Based on manual inspection, here's the pollution summary:\n");
+
+  // Simulated data based on actual semantic-memory_list output
+  const knownTestCollections = {
+    "test-patterns": 16,
+    "test-feedback": 16,
+  };
+
+  const knownProductionCollections = {
+    default: 5, // egghead-rails, POC migration, Docker, Durable Streams, one test
+  };
+
+  const totalTest = Object.values(knownTestCollections).reduce(
+    (a, b) => a + b,
+    0,
+  );
+  const totalProd = Object.values(knownProductionCollections).reduce(
+    (a, b) => a + b,
+    0,
+  );
+  const totalMemories = totalTest + totalProd;
+
+  // Build collections array
+  const collections = [
+    ...Object.entries(knownTestCollections).map(([name, count]) => ({
+      name,
+      count,
+      is_test: true,
+    })),
+    ...Object.entries(knownProductionCollections).map(([name, count]) => ({
+      name,
+      count,
+      is_test: false,
+    })),
+  ];
+
+  // Simulate test artifacts for reporting
+  const testArtifacts = Array.from({ length: totalTest }, (_, i) => ({
+    id: `test-${i}`,
+    collection: i < 16 ? "test-patterns" : "test-feedback",
+    content: "Test artifact",
+  }));
+
+  const productionMemories = Array.from({ length: totalProd }, (_, i) => ({
+    id: `prod-${i}`,
+    collection: "default",
+    content: "Production memory",
+  }));
+
+  return {
+    total_memories: totalMemories,
+    test_artifacts: testArtifacts,
+    production_memories: productionMemories,
+    collections,
+  };
+}
+
+/**
+ * Generate cleanup report
+ */
+function generateReport(report: AuditReport, dryRun: boolean): void {
+  console.log("📊 SEMANTIC MEMORY AUDIT REPORT");
+  console.log("================================\n");
+
+  console.log(`Total memories: ${report.total_memories}`);
+  console.log(
+    `Test artifacts: ${report.test_artifacts.length} (${Math.round((report.test_artifacts.length / report.total_memories) * 100)}%)`,
+  );
+  console.log(`Production memories: ${report.production_memories.length}\n`);
+
+  console.log("Collections breakdown:");
+  console.log("----------------------");
+  for (const col of report.collections) {
+    const marker = col.is_test ? "🚨 TEST" : "✅ PROD";
+    console.log(`  ${marker} ${col.name.padEnd(20)} ${col.count} memories`);
+  }
+
+  console.log("\n⚠️  CLEANUP REQUIRED\n");
+
+  if (report.test_artifacts.length > 0) {
+    console.log("Test collections to remove:");
+    const testCollections = new Set(
+      report.test_artifacts.map((m) => m.collection),
+    );
+    for (const col of testCollections) {
+      const count = report.test_artifacts.filter(
+        (m) => m.collection === col,
+      ).length;
+      console.log(`  - ${col} (${count} memories)`);
+    }
+  }
+
+  console.log("\n📝 MANUAL CLEANUP STEPS\n");
+  console.log(
+    "semantic-memory MCP server does not expose delete/remove tools.",
+  );
+  console.log("Cleanup must be done via direct database access:\n");
+  console.log("1. Stop semantic-memory MCP server");
+  console.log("2. Connect to PostgreSQL:");
+  console.log("   psql -h /Users/joel/.semantic-memory/memory");
+  console.log("3. Delete test collections:");
+  console.log(
+    "   DELETE FROM memories WHERE collection IN ('test-patterns', 'test-feedback');",
+  );
+  console.log("4. Restart semantic-memory MCP server");
+  console.log("5. Verify with semantic-memory_list\n");
+
+  console.log("🛡️  PREVENTION STRATEGY\n");
+  console.log("To prevent future pollution:");
+  console.log("1. ✅ Add test collection prefix isolation (subtask 1 - DONE)");
+  console.log("2. ✅ Add cleanup hooks in afterEach (subtask 2 - DONE)");
+  console.log("3. 📝 Document production collection names");
+  console.log("4. 📝 Add collection naming convention to CONTRIBUTING.md");
+  console.log(
+    "5. 📝 Consider requesting delete/remove API from MCP maintainers\n",
+  );
+
+  if (!dryRun) {
+    console.log(
+      "⚠️  --dry-run not specified, but no automated cleanup available.",
+    );
+    console.log("   Follow manual steps above.\n");
+  }
+}
+
+/**
+ * Store cleanup learnings in semantic-memory for future reference
+ */
+async function storeCleanupLearnings(report: AuditReport): Promise<void> {
+  console.log("💾 Storing cleanup learnings in semantic-memory...\n");
+
+  const rootCause = `
+ROOT CAUSE: Semantic Memory Test Pollution (Dec 2025)
+
+PROBLEM: Integration tests polluted production semantic-memory with ${report.test_artifacts.length} test artifacts across collections: ${Array.from(new Set(report.test_artifacts.map((m) => m.collection))).join(", ")}.
+
+WHY IT HAPPENED:
+1. Tests wrote to shared MCP server (no isolation)
+2. No collection prefix strategy for test data
+3. No cleanup hooks in test teardown
+4. MCP server has no delete/remove API
+
+IMPACT:
+- ${Math.round((report.test_artifacts.length / report.total_memories) * 100)}% of semantic search results are test noise
+- Production knowledge base unreliable
+- Wasted storage and embedding costs
+
+PREVENTION:
+1. ✅ Collection prefix isolation: test-*, temp-* reserved for tests
+2. ✅ Cleanup hooks: afterEach() deletes test collections
+3. ✅ Mock semantic-memory in unit tests (avoid MCP calls)
+4. 📝 Document production collection naming conventions
+5. 📝 Add safeguards to prevent test->prod collection writes
+
+MANUAL CLEANUP REQUIRED:
+semantic-memory MCP lacks delete API. Must use direct PostgreSQL:
+  psql -h /Users/joel/.semantic-memory/memory
+  DELETE FROM memories WHERE collection LIKE 'test-%';
+
+FUTURE: Request delete/remove API from @opencode/semantic-memory maintainers.
+`.trim();
+
+  // Note: In real implementation, this would call semantic-memory_store
+  console.log("Would store:");
+  console.log(rootCause);
+  console.log("\nCollection: default");
+  console.log("Metadata: test-pollution, cleanup, prevention\n");
+}
+
+// CLI Entry Point
+const { values } = parseArgs({
+  args: process.argv.slice(2),
+  options: {
+    "dry-run": { type: "boolean", default: true },
+    collections: { type: "string" },
+    help: { type: "boolean", short: "h", default: false },
+  },
+  allowPositionals: true,
+});
+
+if (values.help) {
+  console.log(`
+Semantic Memory Test Pollution Cleanup
+
+Audits semantic-memory for test artifacts and provides cleanup guidance.
+
+Usage:
+  bun scripts/cleanup-test-memories.ts [options]
+
+Options:
+  --dry-run           Show what would be cleaned (default: true)
+  --collections <csv> Comma-separated list of collections to audit
+  -h, --help          Show this help message
+
+Examples:
+  bun scripts/cleanup-test-memories.ts
+  bun scripts/cleanup-test-memories.ts --dry-run=false
+  bun scripts/cleanup-test-memories.ts --collections test-patterns,test-feedback
+
+Notes:
+  - semantic-memory MCP server does not expose delete/remove API
+  - Cleanup requires direct PostgreSQL access
+  - See script output for manual cleanup steps
+`);
+  process.exit(0);
+}
+
+// Run audit
+const report = await auditMemories();
+const dryRun = values["dry-run"] ?? true;
+generateReport(report, dryRun);
+await storeCleanupLearnings(report);
+
+console.log("✅ Audit complete. See manual cleanup steps above.\n");

package/src/beads.ts

@@ -104,6 +104,8 @@ import {
   type BeadCreateArgs,
   type EpicCreateResult,
 } from "./schemas";
+import { createEvent } from "./streams/events";
+import { appendEvent } from "./streams/store";
 
 /**
  * Custom error for bead operations
@@ -321,6 +323,26 @@ export const beads_create_epic = tool({
         }),
       )
       .describe("Subtasks to create under the epic"),
+    strategy: tool.schema
+      .enum(["file-based", "feature-based", "risk-based"])
+      .optional()
+      .describe("Decomposition strategy used (default: feature-based)"),
+    task: tool.schema
+      .string()
+      .optional()
+      .describe("Original task description that was decomposed"),
+    project_key: tool.schema
+      .string()
+      .optional()
+      .describe("Project path for event emission"),
+    recovery_context: tool.schema
+      .object({
+        shared_context: tool.schema.string().optional(),
+        skills_to_load: tool.schema.array(tool.schema.string()).optional(),
+        coordinator_notes: tool.schema.string().optional(),
+      })
+      .optional()
+      .describe("Recovery context from checkpoint compaction"),
   },
   async execute(args, ctx) {
     const validated = EpicCreateArgsSchema.parse(args);
@@ -386,6 +408,33 @@ export const beads_create_epic = tool({
         subtasks: created.slice(1),
       };
 
+      // Emit DecompositionGeneratedEvent for learning system
+      if (args.project_key) {
+        try {
+          const event = createEvent("decomposition_generated", {
+            project_key: args.project_key,
+            epic_id: epic.id,
+            task: args.task || validated.epic_title,
+            context: validated.epic_description,
+            strategy: args.strategy || "feature-based",
+            epic_title: validated.epic_title,
+            subtasks: validated.subtasks.map((st) => ({
+              title: st.title,
+              files: st.files || [],
+              priority: st.priority,
+            })),
+            recovery_context: args.recovery_context,
+          });
+          await appendEvent(event, args.project_key);
+        } catch (error) {
+          // Non-fatal - log and continue
+          console.warn(
+            "[beads_create_epic] Failed to emit DecompositionGeneratedEvent:",
+            error,
+          );
+        }
+      }
+
       return JSON.stringify(result, null, 2);
     } catch (error) {
       // Partial failure - execute rollback automatically