opencode-swarm-plugin 0.20.0 → 0.22.0

package/evals/README.md

@@ -0,0 +1,116 @@
+# Evalite - Swarm Decomposition Evals
+
+TypeScript-native evaluation framework for testing swarm task decomposition quality.
+
+## Quick Start
+
+```bash
+# Watch mode for development
+pnpm eval:dev
+
+# Run all evals once
+pnpm eval:run
+
+# CI mode with 80% threshold
+pnpm eval:ci
+```
+
+## Structure
+
+```
+evals/
+├── evalite.config.ts       # Evalite configuration
+├── scorers/
+│   └── index.ts           # Custom scorers (independence, balance, coverage, clarity)
+├── fixtures/
+│   └── decomposition-cases.ts  # Test cases with expected outcomes
+└── *.eval.ts              # Eval files (auto-discovered)
+```
+
+## Custom Scorers
+
+### Subtask Independence (0-1)
+
+Checks no files appear in multiple subtasks. File conflicts cause merge conflicts and coordination overhead.
+
+### Complexity Balance (0-1)
+
+Measures coefficient of variation (CV) of estimated_complexity across subtasks. CV < 0.3 scores 1.0, decreases linearly to 0 at CV = 1.0.
+
+### Coverage Completeness (0-1)
+
+If expected.requiredFiles specified: ratio of covered files.
+Otherwise: checks subtask count is within min/max range.
+
+### Instruction Clarity (0-1)
+
+Average quality score per subtask based on:
+
+- Description length > 20 chars (+0.2)
+- Files specified (+0.2)
+- Non-generic title (+0.1)
+
+## Writing Evals
+
+```typescript
+import { evalite } from "evalite";
+import { subtaskIndependence, coverageCompleteness } from "./scorers/index.js";
+
+evalite("My decomposition test", {
+  data: async () => {
+    return [
+      {
+        input: "Add OAuth authentication",
+        expected: {
+          minSubtasks: 3,
+          maxSubtasks: 6,
+          requiredFiles: ["src/auth/oauth.ts", "src/middleware/auth.ts"],
+        },
+      },
+    ];
+  },
+  task: async (input) => {
+    // Call your decomposition logic here
+    // Should return BeadTree JSON as string
+  },
+  scorers: [subtaskIndependence, coverageCompleteness],
+});
+```
+
+## BeadTree Format
+
+Scorers expect output as JSON string matching:
+
+```typescript
+{
+  epic: {
+    title: string;
+    description: string;
+  }
+  subtasks: Array<{
+    title: string;
+    description?: string;
+    files?: string[];
+    estimated_complexity?: number; // 1-3
+  }>;
+}
+```
+
+## Fixtures
+
+See `fixtures/decomposition-cases.ts` for example test cases covering:
+
+- OAuth implementation
+- Rate limiting
+- TypeScript migration
+- Admin dashboard
+- Memory leak debugging
+- Feature flag system
+
+## Notes
+
+- Evalite v1.0.0-beta.15 installed
+- Built on Vitest
+- Runs locally, no API keys required
+- Results cached in `node_modules/.evalite/`
+- Clear cache if needed: `rm -rf node_modules/.evalite`

package/evals/evalite.config.ts

@@ -0,0 +1,15 @@
+import { defineConfig } from "evalite/config";
+
+/**
+ * Evalite configuration for swarm decomposition testing
+ *
+ * Tests swarm task decomposition strategies to ensure:
+ * - Subtasks are independent (no file conflicts)
+ * - Complexity is balanced across subtasks
+ * - Full task coverage
+ */
+export default defineConfig({
+  // Base configuration - Evalite will auto-discover evals in this directory
+  // Custom scorers are defined in scorers/index.ts
+  // Test fixtures are in fixtures/decomposition-cases.ts
+});

package/evals/example.eval.ts

@@ -0,0 +1,32 @@
+/**
+ * Example eval file to test Evalite setup
+ *
+ * This is a minimal test to verify:
+ * 1. Evalite CLI can discover .eval.ts files
+ * 2. createScorer works
+ * 3. evalite() function works
+ */
+
+import { evalite } from "evalite";
+import { subtaskIndependence } from "./scorers/index.js";
+
+evalite("Example: Basic scorer test", {
+  data: async () => {
+    return [
+      {
+        input: "Test task",
+        output: JSON.stringify({
+          epic: { title: "Test Epic", description: "Test" },
+          subtasks: [
+            { title: "Subtask 1", files: ["a.ts"], estimated_complexity: 1 },
+            { title: "Subtask 2", files: ["b.ts"], estimated_complexity: 1 },
+          ],
+        }),
+      },
+    ];
+  },
+  task: async (input) => {
+    return input; // passthrough for testing
+  },
+  scorers: [subtaskIndependence],
+});

package/evals/fixtures/decomposition-cases.ts

@@ -0,0 +1,105 @@
+/**
+ * Test cases for swarm task decomposition
+ *
+ * Each case includes:
+ * - input: task description and optional context
+ * - expected: validation criteria (min/max subtasks, required files)
+ */
+
+export interface DecompositionTestCase {
+  input: {
+    task: string;
+    context?: string;
+  };
+  expected: {
+    minSubtasks: number;
+    maxSubtasks: number;
+    requiredFiles?: string[];
+  };
+}
+
+export const decompositionCases: DecompositionTestCase[] = [
+  {
+    input: {
+      task: "Add user authentication with OAuth",
+      context: "Next.js App Router application with existing user model",
+    },
+    expected: {
+      minSubtasks: 3,
+      maxSubtasks: 6,
+      requiredFiles: [
+        "src/auth/oauth.ts",
+        "src/auth/middleware.ts",
+        "app/api/auth/[...nextauth]/route.ts",
+      ],
+    },
+  },
+  {
+    input: {
+      task: "Implement rate limiting for API endpoints",
+      context: "Express.js API with Redis available",
+    },
+    expected: {
+      minSubtasks: 2,
+      maxSubtasks: 4,
+      requiredFiles: [
+        "src/middleware/rate-limit.ts",
+        "src/utils/redis-client.ts",
+      ],
+    },
+  },
+  {
+    input: {
+      task: "Add TypeScript strict mode to legacy JavaScript project",
+      context: "Large codebase with 50+ JS files, currently untyped",
+    },
+    expected: {
+      minSubtasks: 4,
+      maxSubtasks: 8,
+      requiredFiles: ["tsconfig.json"],
+    },
+  },
+  {
+    input: {
+      task: "Create admin dashboard for user management",
+      context: "React app with existing component library and API client",
+    },
+    expected: {
+      minSubtasks: 4,
+      maxSubtasks: 7,
+      requiredFiles: [
+        "src/pages/admin/Dashboard.tsx",
+        "src/components/admin/UserTable.tsx",
+        "src/api/admin.ts",
+      ],
+    },
+  },
+  {
+    input: {
+      task: "Fix memory leak in long-running background job",
+      context:
+        "Node.js worker that processes queue messages, memory grows over time",
+    },
+    expected: {
+      minSubtasks: 2,
+      maxSubtasks: 4,
+      requiredFiles: ["src/workers/queue-processor.ts"],
+    },
+  },
+  {
+    input: {
+      task: "Implement feature flag system with remote config",
+      context:
+        "Microservices architecture, need runtime toggles without deploys",
+    },
+    expected: {
+      minSubtasks: 3,
+      maxSubtasks: 6,
+      requiredFiles: [
+        "src/feature-flags/client.ts",
+        "src/feature-flags/middleware.ts",
+        "src/feature-flags/types.ts",
+      ],
+    },
+  },
+];

package/evals/lib/data-loader.test.ts

@@ -0,0 +1,288 @@
+/**
+ * Data Loader Tests
+ *
+ * Tests the PGlite-backed eval data loader functions.
+ * Uses a real in-memory PGlite database for accurate testing.
+ */
+import { describe, it, expect, beforeAll, afterAll } from "bun:test";
+import {
+  loadEvalCases,
+  hasRealEvalData,
+  getEvalDataSummary,
+} from "./data-loader.js";
+import { appendEvent } from "../../src/streams/store.js";
+import { getDatabase, closeDatabase } from "../../src/streams/index.js";
+import type {
+  DecompositionGeneratedEvent,
+  SubtaskOutcomeEvent,
+} from "../../src/streams/events.js";
+import * as fs from "node:fs";
+import * as path from "node:path";
+import * as os from "node:os";
+
+const TEST_PROJECT_KEY = "test-project-eval-loader";
+
+// Create a unique temp directory for this test run
+let testDir: string;
+
+describe("Data Loader", () => {
+  beforeAll(async () => {
+    // Create temp directory for test database
+    testDir = fs.mkdtempSync(path.join(os.tmpdir(), "eval-loader-test-"));
+
+    // Initialize database by getting it (lazy init)
+    await getDatabase(testDir);
+  });
+
+  afterAll(async () => {
+    await closeDatabase(testDir);
+    // Clean up temp directory
+    fs.rmSync(testDir, { recursive: true, force: true });
+  });
+
+  describe("loadEvalCases", () => {
+    it("transforms eval records to EvalCase format", async () => {
+      // Insert a decomposition event
+      const decompositionEvent: DecompositionGeneratedEvent = {
+        type: "decomposition_generated",
+        timestamp: Date.now(),
+        project_key: TEST_PROJECT_KEY,
+        epic_id: "epic-load-1",
+        task: "Add authentication",
+        context: "Next.js app",
+        strategy: "feature-based",
+        epic_title: "Auth Epic",
+        subtasks: [
+          { title: "OAuth setup", files: ["src/auth/oauth.ts"], priority: 1 },
+          {
+            title: "Session management",
+            files: ["src/auth/session.ts"],
+            priority: 2,
+          },
+        ],
+      };
+      await appendEvent(decompositionEvent, testDir);
+
+      // Insert outcome events for both subtasks
+      const outcome1: SubtaskOutcomeEvent = {
+        type: "subtask_outcome",
+        timestamp: Date.now(),
+        project_key: TEST_PROJECT_KEY,
+        epic_id: "epic-load-1",
+        bead_id: "epic-load-1.1",
+        planned_files: ["src/auth/oauth.ts"],
+        actual_files: ["src/auth/oauth.ts"],
+        duration_ms: 5000,
+        error_count: 0,
+        retry_count: 0,
+        success: true,
+      };
+      await appendEvent(outcome1, testDir);
+
+      const outcome2: SubtaskOutcomeEvent = {
+        type: "subtask_outcome",
+        timestamp: Date.now(),
+        project_key: TEST_PROJECT_KEY,
+        epic_id: "epic-load-1",
+        bead_id: "epic-load-1.2",
+        planned_files: ["src/auth/session.ts"],
+        actual_files: ["src/auth/session.ts"],
+        duration_ms: 3000,
+        error_count: 0,
+        retry_count: 0,
+        success: true,
+      };
+      await appendEvent(outcome2, testDir);
+
+      const cases = await loadEvalCases(TEST_PROJECT_KEY, {
+        projectPath: testDir,
+      });
+
+      expect(cases.length).toBeGreaterThanOrEqual(1);
+      const authCase = cases.find((c) => c.input.task === "Add authentication");
+      expect(authCase).toBeDefined();
+      expect(authCase!.input.context).toBe("Next.js app");
+      expect(authCase!.expected.minSubtasks).toBe(2);
+      expect(authCase!.expected.maxSubtasks).toBe(2);
+      expect(authCase!.expected.requiredFiles).toContain("src/auth/oauth.ts");
+      expect(authCase!.expected.requiredFiles).toContain("src/auth/session.ts");
+      expect(authCase!.actual).toBeDefined();
+    });
+
+    it("filters by success when successOnly is true", async () => {
+      // Insert a successful decomposition
+      const successEvent: DecompositionGeneratedEvent = {
+        type: "decomposition_generated",
+        timestamp: Date.now(),
+        project_key: TEST_PROJECT_KEY,
+        epic_id: "epic-success-filter",
+        task: "Success task for filter",
+        strategy: "feature-based",
+        epic_title: "Success Epic",
+        subtasks: [{ title: "Sub", files: ["src/success.ts"], priority: 1 }],
+      };
+      await appendEvent(successEvent, testDir);
+
+      // Mark it successful
+      const successOutcome: SubtaskOutcomeEvent = {
+        type: "subtask_outcome",
+        timestamp: Date.now(),
+        project_key: TEST_PROJECT_KEY,
+        epic_id: "epic-success-filter",
+        bead_id: "epic-success-filter.1",
+        planned_files: ["src/success.ts"],
+        actual_files: ["src/success.ts"],
+        duration_ms: 1000,
+        error_count: 0,
+        retry_count: 0,
+        success: true,
+      };
+      await appendEvent(successOutcome, testDir);
+
+      // Insert a failed decomposition
+      const failEvent: DecompositionGeneratedEvent = {
+        type: "decomposition_generated",
+        timestamp: Date.now(),
+        project_key: TEST_PROJECT_KEY,
+        epic_id: "epic-fail-filter",
+        task: "Failed task for filter",
+        strategy: "feature-based",
+        epic_title: "Failed Epic",
+        subtasks: [{ title: "Sub", files: ["src/fail.ts"], priority: 1 }],
+      };
+      await appendEvent(failEvent, testDir);
+
+      // Mark it failed
+      const failOutcome: SubtaskOutcomeEvent = {
+        type: "subtask_outcome",
+        timestamp: Date.now(),
+        project_key: TEST_PROJECT_KEY,
+        epic_id: "epic-fail-filter",
+        bead_id: "epic-fail-filter.1",
+        planned_files: ["src/fail.ts"],
+        actual_files: [],
+        duration_ms: 500,
+        error_count: 3,
+        retry_count: 2,
+        success: false,
+      };
+      await appendEvent(failOutcome, testDir);
+
+      const successCases = await loadEvalCases(TEST_PROJECT_KEY, {
+        successOnly: true,
+        projectPath: testDir,
+      });
+
+      // Should only include successful cases
+      const failedCase = successCases.find(
+        (c) => c.input.task === "Failed task for filter",
+      );
+      expect(failedCase).toBeUndefined();
+    });
+
+    it("passes strategy filter to getEvalRecords", async () => {
+      // Insert file-based decomposition
+      const fileBasedEvent: DecompositionGeneratedEvent = {
+        type: "decomposition_generated",
+        timestamp: Date.now(),
+        project_key: TEST_PROJECT_KEY,
+        epic_id: "epic-file-based",
+        task: "File-based task",
+        strategy: "file-based",
+        epic_title: "File Epic",
+        subtasks: [{ title: "Sub", files: ["src/file.ts"], priority: 1 }],
+      };
+      await appendEvent(fileBasedEvent, testDir);
+
+      const fileBasedCases = await loadEvalCases(TEST_PROJECT_KEY, {
+        strategy: "file-based",
+        projectPath: testDir,
+      });
+
+      // All returned cases should be file-based
+      for (const c of fileBasedCases) {
+        expect(c.actual?.strategy).toBe("file-based");
+      }
+    });
+
+    it("passes limit to getEvalRecords", async () => {
+      const cases = await loadEvalCases(TEST_PROJECT_KEY, {
+        limit: 2,
+        projectPath: testDir,
+      });
+
+      expect(cases.length).toBeLessThanOrEqual(2);
+    });
+
+    it("handles records with no context", async () => {
+      const noContextEvent: DecompositionGeneratedEvent = {
+        type: "decomposition_generated",
+        timestamp: Date.now(),
+        project_key: TEST_PROJECT_KEY,
+        epic_id: "epic-no-context",
+        task: "Task without context",
+        // context is undefined
+        strategy: "feature-based",
+        epic_title: "No Context Epic",
+        subtasks: [{ title: "Sub", files: [], priority: 1 }],
+      };
+      await appendEvent(noContextEvent, testDir);
+
+      const cases = await loadEvalCases(TEST_PROJECT_KEY, {
+        projectPath: testDir,
+      });
+      const noContextCase = cases.find(
+        (c) => c.input.task === "Task without context",
+      );
+
+      expect(noContextCase).toBeDefined();
+      expect(noContextCase!.input.context).toBeUndefined();
+    });
+  });
+
+  describe("hasRealEvalData", () => {
+    it("returns true when enough records exist", async () => {
+      // We've inserted several records above, should have enough
+      const hasData = await hasRealEvalData(TEST_PROJECT_KEY, 1, testDir);
+      expect(hasData).toBe(true);
+    });
+
+    it("returns false when not enough records exist", async () => {
+      // Use a project key with no data
+      const hasData = await hasRealEvalData("nonexistent-project", 5, testDir);
+      expect(hasData).toBe(false);
+    });
+
+    it("uses custom minRecords threshold", async () => {
+      // Should have at least 1 record
+      const hasData = await hasRealEvalData(TEST_PROJECT_KEY, 1, testDir);
+      expect(hasData).toBe(true);
+
+      // Should not have 1000 records
+      const hasLotsOfData = await hasRealEvalData(
+        TEST_PROJECT_KEY,
+        1000,
+        testDir,
+      );
+      expect(hasLotsOfData).toBe(false);
+    });
+  });
+
+  describe("getEvalDataSummary", () => {
+    it("returns formatted summary with hasEnoughData flag", async () => {
+      const summary = await getEvalDataSummary(TEST_PROJECT_KEY, testDir);
+
+      expect(summary.totalRecords).toBeGreaterThanOrEqual(1);
+      expect(typeof summary.successRate).toBe("number");
+      expect(typeof summary.byStrategy).toBe("object");
+      expect(typeof summary.hasEnoughData).toBe("boolean");
+    });
+
+    it("sets hasEnoughData based on record count", async () => {
+      // Empty project should not have enough data
+      const emptySummary = await getEvalDataSummary("empty-project", testDir);
+      expect(emptySummary.hasEnoughData).toBe(false);
+      expect(emptySummary.totalRecords).toBe(0);
+    });
+  });
+});

package/evals/lib/data-loader.ts

@@ -0,0 +1,111 @@
+/**
+ * PGlite-backed eval data loader
+ *
+ * Loads real decomposition outcomes from the eval_records table
+ * for use in Evalite evals.
+ */
+import {
+  getEvalRecords,
+  getEvalStats,
+  type EvalRecord,
+} from "../../src/streams/projections.js";
+
+export interface EvalCase {
+  input: { task: string; context?: string };
+  expected: {
+    minSubtasks: number;
+    maxSubtasks: number;
+    requiredFiles?: string[];
+    overallSuccess?: boolean;
+  };
+  actual?: EvalRecord;
+}
+
+/**
+ * Load eval cases from PGlite
+ *
+ * @param projectKey - Project key for filtering records
+ * @param options - Filter options
+ * @returns Array of eval cases ready for Evalite
+ */
+export async function loadEvalCases(
+  projectKey: string,
+  options?: {
+    limit?: number;
+    strategy?: "file-based" | "feature-based" | "risk-based";
+    successOnly?: boolean;
+    projectPath?: string;
+  },
+): Promise<EvalCase[]> {
+  const { limit, strategy, successOnly, projectPath } = options ?? {};
+
+  // Query eval records from PGlite
+  const records = await getEvalRecords(
+    projectKey,
+    { limit, strategy },
+    projectPath,
+  );
+
+  // Filter by success if requested
+  const filtered = successOnly
+    ? records.filter((r) => r.overall_success === true)
+    : records;
+
+  // Transform to EvalCase format
+  return filtered.map((record) => ({
+    input: {
+      task: record.task,
+      context: record.context ?? undefined,
+    },
+    expected: {
+      minSubtasks: 2,
+      maxSubtasks: record.subtasks.length,
+      requiredFiles: record.subtasks.flatMap((s) => s.files),
+      overallSuccess: record.overall_success ?? undefined,
+    },
+    actual: record,
+  }));
+}
+
+/**
+ * Check if we have enough real data to run evals
+ *
+ * @param projectKey - Project key to check
+ * @param minRecords - Minimum number of records required (default: 5)
+ * @param projectPath - Optional project path for database lookup
+ * @returns True if enough data exists
+ */
+export async function hasRealEvalData(
+  projectKey: string,
+  minRecords: number = 5,
+  projectPath?: string,
+): Promise<boolean> {
+  const stats = await getEvalStats(projectKey, projectPath);
+  return stats.totalRecords >= minRecords;
+}
+
+/**
+ * Get eval data stats for reporting
+ *
+ * @param projectKey - Project key to query
+ * @param projectPath - Optional project path for database lookup
+ * @returns Summary of available eval data
+ */
+export async function getEvalDataSummary(
+  projectKey: string,
+  projectPath?: string,
+): Promise<{
+  totalRecords: number;
+  successRate: number;
+  byStrategy: Record<string, number>;
+  hasEnoughData: boolean;
+}> {
+  const stats = await getEvalStats(projectKey, projectPath);
+
+  return {
+    totalRecords: stats.totalRecords,
+    successRate: stats.successRate,
+    byStrategy: stats.byStrategy,
+    hasEnoughData: stats.totalRecords >= 5,
+  };
+}