opencode-swarm-plugin 0.38.0 → 0.39.1

package/evals/lib/data-loader.test.ts

@@ -0,0 +1,345 @@
+/**
+ * Tests for data-loader quality filters
+ *
+ * TDD approach: RED → GREEN → REFACTOR
+ */
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import type { CoordinatorEvent } from "../../src/eval-capture.js";
+import { loadCapturedSessions } from "./data-loader.js";
+
+// Test helper: create a temp session directory
+let tempSessionDir: string;
+
+beforeEach(() => {
+  tempSessionDir = fs.mkdtempSync(path.join(os.tmpdir(), "test-sessions-"));
+});
+
+afterEach(() => {
+  if (fs.existsSync(tempSessionDir)) {
+    fs.rmSync(tempSessionDir, { recursive: true });
+  }
+});
+
+/**
+ * Helper: create a session JSONL file with events
+ */
+function createSessionFile(
+  sessionId: string,
+  events: CoordinatorEvent[],
+): void {
+  const filePath = path.join(tempSessionDir, `${sessionId}.jsonl`);
+  const lines = events.map((e) => JSON.stringify(e)).join("\n") + "\n";
+  fs.writeFileSync(filePath, lines, "utf-8");
+}
+
+/**
+ * Helper: create minimal events
+ */
+function createEvent(
+  sessionId: string,
+  epicId: string,
+  type: "DECISION" | "VIOLATION" | "OUTCOME",
+  subtype: string,
+): CoordinatorEvent {
+  const base = {
+    session_id: sessionId,
+    epic_id: epicId,
+    timestamp: new Date().toISOString(),
+    payload: {},
+  };
+
+  if (type === "DECISION") {
+    return {
+      ...base,
+      event_type: "DECISION" as const,
+      decision_type: subtype as any,
+    };
+  } else if (type === "VIOLATION") {
+    return {
+      ...base,
+      event_type: "VIOLATION" as const,
+      violation_type: subtype as any,
+    };
+  } else {
+    return {
+      ...base,
+      event_type: "OUTCOME" as const,
+      outcome_type: subtype as any,
+    };
+  }
+}
+
+describe("loadCapturedSessions - quality filters", () => {
+  test("filters out sessions with fewer than minEvents (default: 3)", async () => {
+    // Create sessions with different event counts
+    createSessionFile("session-2-events", [
+      createEvent("session-2-events", "epic-1", "DECISION", "worker_spawned"),
+      createEvent("session-2-events", "epic-1", "OUTCOME", "subtask_success"),
+    ]);
+
+    createSessionFile("session-3-events", [
+      createEvent("session-3-events", "epic-2", "DECISION", "worker_spawned"),
+      createEvent("session-3-events", "epic-2", "DECISION", "review_completed"),
+      createEvent("session-3-events", "epic-2", "OUTCOME", "subtask_success"),
+    ]);
+
+    createSessionFile("session-5-events", [
+      createEvent("session-5-events", "epic-3", "DECISION", "worker_spawned"),
+      createEvent("session-5-events", "epic-3", "DECISION", "review_completed"),
+      createEvent("session-5-events", "epic-3", "OUTCOME", "subtask_success"),
+      createEvent("session-5-events", "epic-3", "OUTCOME", "subtask_success"),
+      createEvent("session-5-events", "epic-3", "OUTCOME", "epic_complete"),
+    ]);
+
+    const sessions = await loadCapturedSessions({
+      minEvents: 3,
+      sessionDir: tempSessionDir,
+    });
+
+    // Should only get sessions with >= 3 events
+    expect(sessions.length).toBe(2);
+    expect(
+      sessions.some((s) => s.session.session_id === "session-3-events"),
+    ).toBe(true);
+    expect(
+      sessions.some((s) => s.session.session_id === "session-5-events"),
+    ).toBe(true);
+    expect(
+      sessions.some((s) => s.session.session_id === "session-2-events"),
+    ).toBe(false);
+  });
+
+  test("filters out sessions without worker_spawned event when requireWorkerSpawn=true", async () => {
+    // Session WITH worker_spawned
+    createSessionFile("session-with-spawn", [
+      createEvent("session-with-spawn", "epic-1", "DECISION", "worker_spawned"),
+      createEvent(
+        "session-with-spawn",
+        "epic-1",
+        "DECISION",
+        "review_completed",
+      ),
+      createEvent("session-with-spawn", "epic-1", "OUTCOME", "subtask_success"),
+    ]);
+
+    // Session WITHOUT worker_spawned
+    createSessionFile("session-no-spawn", [
+      createEvent(
+        "session-no-spawn",
+        "epic-2",
+        "DECISION",
+        "strategy_selected",
+      ),
+      createEvent(
+        "session-no-spawn",
+        "epic-2",
+        "DECISION",
+        "decomposition_complete",
+      ),
+      createEvent("session-no-spawn", "epic-2", "OUTCOME", "epic_complete"),
+    ]);
+
+    const sessions = await loadCapturedSessions({
+      requireWorkerSpawn: true,
+      sessionDir: tempSessionDir,
+    });
+
+    expect(sessions.length).toBe(1);
+    expect(sessions[0]?.session.session_id).toBe("session-with-spawn");
+  });
+
+  test("filters out sessions without review_completed event when requireReview=true", async () => {
+    // Session WITH review
+    createSessionFile("session-with-review", [
+      createEvent(
+        "session-with-review",
+        "epic-1",
+        "DECISION",
+        "worker_spawned",
+      ),
+      createEvent(
+        "session-with-review",
+        "epic-1",
+        "DECISION",
+        "review_completed",
+      ),
+      createEvent("session-with-review", "epic-1", "OUTCOME", "subtask_success"),
+    ]);
+
+    // Session WITHOUT review
+    createSessionFile("session-no-review", [
+      createEvent("session-no-review", "epic-2", "DECISION", "worker_spawned"),
+      createEvent("session-no-review", "epic-2", "OUTCOME", "subtask_success"),
+      createEvent("session-no-review", "epic-2", "OUTCOME", "epic_complete"),
+    ]);
+
+    const sessions = await loadCapturedSessions({
+      requireReview: true,
+      sessionDir: tempSessionDir,
+    });
+
+    expect(sessions.length).toBe(1);
+    expect(sessions[0]?.session.session_id).toBe("session-with-review");
+  });
+
+  test("allows disabling filters individually", async () => {
+    // Session with only 2 events, no worker_spawned, no review
+    createSessionFile("session-low-quality", [
+      createEvent(
+        "session-low-quality",
+        "epic-1",
+        "DECISION",
+        "strategy_selected",
+      ),
+      createEvent("session-low-quality", "epic-1", "OUTCOME", "epic_complete"),
+    ]);
+
+    // Disable all filters
+    const sessions = await loadCapturedSessions({
+      minEvents: 0,
+      requireWorkerSpawn: false,
+      requireReview: false,
+      sessionDir: tempSessionDir,
+    });
+
+    expect(sessions.length).toBe(1);
+    expect(sessions[0]?.session.session_id).toBe("session-low-quality");
+  });
+
+  test("applies limit AFTER filtering", async () => {
+    // Create 5 high-quality sessions
+    for (let i = 1; i <= 5; i++) {
+      createSessionFile(`session-${i}`, [
+        createEvent(`session-${i}`, `epic-${i}`, "DECISION", "worker_spawned"),
+        createEvent(
+          `session-${i}`,
+          `epic-${i}`,
+          "DECISION",
+          "review_completed",
+        ),
+        createEvent(`session-${i}`, `epic-${i}`, "OUTCOME", "subtask_success"),
+      ]);
+    }
+
+    // Create 3 low-quality sessions (will be filtered out)
+    for (let i = 6; i <= 8; i++) {
+      createSessionFile(`session-${i}`, [
+        createEvent(`session-${i}`, `epic-${i}`, "DECISION", "strategy_selected"),
+      ]);
+    }
+
+    // Filter first (remove 3 low-quality), then limit to 2
+    const sessions = await loadCapturedSessions({
+      minEvents: 3,
+      requireWorkerSpawn: true,
+      requireReview: true,
+      limit: 2,
+      sessionDir: tempSessionDir,
+    });
+
+    // Should get 2 sessions from the 5 high-quality ones
+    expect(sessions.length).toBe(2);
+    expect(sessions.every((s) => s.session.events.length >= 3)).toBe(true);
+  });
+
+  test("combines all filters correctly", async () => {
+    // High-quality session (passes all filters)
+    createSessionFile("session-high-quality", [
+      createEvent(
+        "session-high-quality",
+        "epic-1",
+        "DECISION",
+        "worker_spawned",
+      ),
+      createEvent(
+        "session-high-quality",
+        "epic-1",
+        "DECISION",
+        "review_completed",
+      ),
+      createEvent("session-high-quality", "epic-1", "OUTCOME", "subtask_success"),
+      createEvent("session-high-quality", "epic-1", "OUTCOME", "epic_complete"),
+    ]);
+
+    // Missing worker_spawned
+    createSessionFile("session-no-spawn", [
+      createEvent(
+        "session-no-spawn",
+        "epic-2",
+        "DECISION",
+        "review_completed",
+      ),
+      createEvent("session-no-spawn", "epic-2", "OUTCOME", "subtask_success"),
+      createEvent("session-no-spawn", "epic-2", "OUTCOME", "epic_complete"),
+    ]);
+
+    // Missing review_completed
+    createSessionFile("session-no-review", [
+      createEvent("session-no-review", "epic-3", "DECISION", "worker_spawned"),
+      createEvent("session-no-review", "epic-3", "OUTCOME", "subtask_success"),
+      createEvent("session-no-review", "epic-3", "OUTCOME", "epic_complete"),
+    ]);
+
+    // Too few events
+    createSessionFile("session-too-few", [
+      createEvent("session-too-few", "epic-4", "DECISION", "worker_spawned"),
+      createEvent("session-too-few", "epic-4", "DECISION", "review_completed"),
+    ]);
+
+    const sessions = await loadCapturedSessions({
+      minEvents: 3,
+      requireWorkerSpawn: true,
+      requireReview: true,
+      sessionDir: tempSessionDir,
+    });
+
+    // Only high-quality session should pass
+    expect(sessions.length).toBe(1);
+    expect(sessions[0]?.session.session_id).toBe("session-high-quality");
+  });
+
+  test("defaults are: minEvents=3, requireWorkerSpawn=true, requireReview=true", async () => {
+    // Create one session that meets defaults
+    createSessionFile("session-meets-defaults", [
+      createEvent(
+        "session-meets-defaults",
+        "epic-1",
+        "DECISION",
+        "worker_spawned",
+      ),
+      createEvent(
+        "session-meets-defaults",
+        "epic-1",
+        "DECISION",
+        "review_completed",
+      ),
+      createEvent(
+        "session-meets-defaults",
+        "epic-1",
+        "OUTCOME",
+        "subtask_success",
+      ),
+    ]);
+
+    // Create one that doesn't
+    createSessionFile("session-fails-defaults", [
+      createEvent(
+        "session-fails-defaults",
+        "epic-2",
+        "DECISION",
+        "strategy_selected",
+      ),
+    ]);
+
+    // Call with NO options except sessionDir - should use defaults
+    const sessions = await loadCapturedSessions({
+      sessionDir: tempSessionDir,
+    });
+
+    expect(sessions.length).toBe(1);
+    expect(sessions[0]?.session.session_id).toBe("session-meets-defaults");
+  });
+});

package/evals/lib/data-loader.ts

@@ -111,24 +111,86 @@ export async function getEvalDataSummary(
   };
 }
 
+/**
+ * Check if a session meets quality criteria
+ */
+function meetsQualityCriteria(
+  session: import("../../src/eval-capture.js").CoordinatorSession,
+  criteria: {
+    minEvents: number;
+    requireWorkerSpawn: boolean;
+    requireReview: boolean;
+  },
+): boolean {
+  // Filter 1: minEvents
+  if (session.events.length < criteria.minEvents) {
+    return false;
+  }
+
+  // Filter 2: requireWorkerSpawn
+  if (
+    criteria.requireWorkerSpawn &&
+    !session.events.some(
+      (e) => e.event_type === "DECISION" && e.decision_type === "worker_spawned",
+    )
+  ) {
+    return false;
+  }
+
+  // Filter 3: requireReview
+  if (
+    criteria.requireReview &&
+    !session.events.some(
+      (e) =>
+        e.event_type === "DECISION" && e.decision_type === "review_completed",
+    )
+  ) {
+    return false;
+  }
+
+  return true;
+}
+
 /**
  * Load captured coordinator sessions from ~/.config/swarm-tools/sessions/
  *
  * Reads all JSONL session files and returns CoordinatorSession objects.
  *
+ * Quality filters are applied to focus on high-signal coordinator sessions:
+ * - minEvents: Filter out incomplete/aborted sessions (default: 3)
+ * - requireWorkerSpawn: Ensure session delegated to workers (default: true)
+ * - requireReview: Ensure coordinator reviewed work (default: true)
+ *
+ * Filters are applied BEFORE the limit for accurate sampling.
+ *
  * @param options - Filter options
- * @returns Array of coordinator sessions
+ * @returns Array of coordinator sessions that meet quality criteria
  */
 export async function loadCapturedSessions(options?: {
   sessionIds?: string[];
   limit?: number;
+  /** Minimum number of events required (default: 3) */
+  minEvents?: number;
+  /** Require at least one worker_spawned event (default: true) */
+  requireWorkerSpawn?: boolean;
+  /** Require at least one review_completed event (default: true) */
+  requireReview?: boolean;
+  /** Override session directory for testing */
+  sessionDir?: string;
 }): Promise<
   Array<{ session: import("../../src/eval-capture.js").CoordinatorSession }>
 > {
   const { getSessionDir, readSessionEvents, saveSession } = await import(
     "../../src/eval-capture.js"
   );
-  const sessionDir = getSessionDir();
+  const sessionDir = options?.sessionDir ?? getSessionDir();
+
+  // Default quality filters
+  const qualityCriteria = {
+    minEvents: options?.minEvents ?? 3,
+    requireWorkerSpawn: options?.requireWorkerSpawn ?? true,
+    requireReview: options?.requireReview ?? true,
+  };
 
   // If session dir doesn't exist, return empty
   if (!fs.existsSync(sessionDir)) {
@@ -149,32 +211,71 @@ export async function loadCapturedSessions(options?: {
   const sessions: Array<{
     session: import("../../src/eval-capture.js").CoordinatorSession;
   }> = [];
+  let filteredOutCount = 0;
 
   for (const file of targetFiles) {
     const sessionId = file.replace(".jsonl", "");
 
     try {
-      const events = readSessionEvents(sessionId);
+      let events: import("../../src/eval-capture.js").CoordinatorEvent[];
+
+      // If custom sessionDir, read directly; otherwise use eval-capture functions
+      if (options?.sessionDir) {
+        const sessionPath = `${sessionDir}/${sessionId}.jsonl`;
+        if (!fs.existsSync(sessionPath)) continue;
+
+        const content = fs.readFileSync(sessionPath, "utf-8");
+        const lines = content.trim().split("\n").filter(Boolean);
+        const { CoordinatorEventSchema } = await import(
+          "../../src/eval-capture.js"
+        );
+        events = lines.map((line) => {
+          const parsed = JSON.parse(line);
+          return CoordinatorEventSchema.parse(parsed);
+        });
+      } else {
+        events = readSessionEvents(sessionId);
+      }
+
       if (events.length === 0) continue;
 
       // Find epic_id from first event
       const epicId = events[0]?.epic_id;
       if (!epicId) continue;
 
-      const session = saveSession({ session_id: sessionId, epic_id: epicId });
-      if (session) {
+      // Build session object
+      const session: import("../../src/eval-capture.js").CoordinatorSession = {
+        session_id: sessionId,
+        epic_id: epicId,
+        start_time: events[0]?.timestamp ?? new Date().toISOString(),
+        end_time: events[events.length - 1]?.timestamp,
+        events,
+      };
+      if (!session) continue;
+
+      // Apply quality filters BEFORE limit
+      if (meetsQualityCriteria(session, qualityCriteria)) {
         sessions.push({ session });
+      } else {
+        filteredOutCount++;
       }
     } catch (error) {
       // Skip invalid sessions
       console.warn(`Failed to load session ${sessionId}:`, error);
     }
 
-    // Apply limit if specified
+    // Apply limit AFTER filtering
     if (options?.limit && sessions.length >= options.limit) {
       break;
     }
   }
 
+  // Log filtering stats for visibility
+  if (filteredOutCount > 0) {
+    console.log(
+      `Filtered out ${filteredOutCount} sessions (minEvents=${qualityCriteria.minEvents}, requireWorkerSpawn=${qualityCriteria.requireWorkerSpawn}, requireReview=${qualityCriteria.requireReview})`,
+    );
+  }
+
   return sessions;
 }

package/evals/scorers/compaction-prompt-scorers.ts

@@ -0,0 +1,145 @@
+/**
+ * Compaction Prompt Quality Scorers - Evalite Wrappers
+ *
+ * These wrap the pure scoring functions from src/compaction-prompt-scoring.ts
+ * for use with evalite's test runner.
+ *
+ * Weighted scoring:
+ * - epicIdSpecificity (0.20) - real IDs not placeholders
+ * - actionability (0.20) - swarm_status/inbox with real values
+ * - coordinatorIdentity (0.25) - ASCII header + strong mandates
+ * - forbiddenToolsPresent (0.15) - lists forbidden tools by name
+ * - postCompactionDiscipline (0.20) - first tool correct, no edit/write
+ */
+
+import { createScorer } from "evalite";
+import type { CompactionPrompt } from "../../src/compaction-prompt-scoring.js";
+import {
+	scoreActionability,
+	scoreCoordinatorIdentity,
+	scoreEpicIdSpecificity,
+	scoreForbiddenToolsPresent,
+	scorePostCompactionDiscipline,
+} from "../../src/compaction-prompt-scoring.js";
+
+// Re-export types for convenience
+export type { CompactionPrompt, ScorerResult } from "../../src/compaction-prompt-scoring.js";
+
+// Re-export pure functions for direct use
+export {
+	scoreActionability,
+	scoreCoordinatorIdentity,
+	scoreEpicIdSpecificity,
+	scoreForbiddenToolsPresent,
+	scorePostCompactionDiscipline,
+} from "../../src/compaction-prompt-scoring.js";
+
+/**
+ * Epic ID Specificity Scorer
+ *
+ * Validates that epic IDs are REAL, not placeholders.
+ * Score: 1.0 if real IDs, 0.0 if placeholders found
+ */
+export const epicIdSpecificity = createScorer({
+	name: "Epic ID Specificity",
+	description: "Prompt uses real epic IDs, not placeholders",
+	scorer: ({ output }) => {
+		try {
+			const prompt = JSON.parse(String(output)) as CompactionPrompt;
+			return scoreEpicIdSpecificity(prompt);
+		} catch (error) {
+			return {
+				score: 0,
+				message: `Failed to parse prompt: ${error}`,
+			};
+		}
+	},
+});
+
+/**
+ * Actionability Scorer
+ *
+ * Validates that the prompt includes SPECIFIC actionable tool calls.
+ * Score: 1.0 if actionable tool calls with real values, 0.0 otherwise
+ */
+export const actionability = createScorer({
+	name: "Actionability",
+	description: "Prompt includes specific tool calls with real values",
+	scorer: ({ output }) => {
+		try {
+			const prompt = JSON.parse(String(output)) as CompactionPrompt;
+			return scoreActionability(prompt);
+		} catch (error) {
+			return {
+				score: 0,
+				message: `Failed to parse prompt: ${error}`,
+			};
+		}
+	},
+});
+
+/**
+ * Coordinator Identity Scorer
+ *
+ * Validates that the prompt has STRONG coordinator identity reinforcement.
+ * Score: 1.0 for ASCII header + strong mandates, 0.5 for header only, 0.0 otherwise
+ */
+export const coordinatorIdentity = createScorer({
+	name: "Coordinator Identity",
+	description: "Prompt has ASCII header and strong mandates",
+	scorer: ({ output }) => {
+		try {
+			const prompt = JSON.parse(String(output)) as CompactionPrompt;
+			return scoreCoordinatorIdentity(prompt);
+		} catch (error) {
+			return {
+				score: 0,
+				message: `Failed to parse prompt: ${error}`,
+			};
+		}
+	},
+});
+
+/**
+ * Forbidden Tools Present Scorer
+ *
+ * Validates that the prompt LISTS forbidden tools by name.
+ * Score: ratio of forbidden tools mentioned (0.0 to 1.0)
+ */
+export const forbiddenToolsPresent = createScorer({
+	name: "Forbidden Tools Present",
+	description: "Prompt lists forbidden tools by name",
+	scorer: ({ output }) => {
+		try {
+			const prompt = JSON.parse(String(output)) as CompactionPrompt;
+			return scoreForbiddenToolsPresent(prompt);
+		} catch (error) {
+			return {
+				score: 0,
+				message: `Failed to parse prompt: ${error}`,
+			};
+		}
+	},
+});
+
+/**
+ * Post-Compaction Discipline Scorer
+ *
+ * Validates that the FIRST suggested tool is correct.
+ * Score: 1.0 if first tool is swarm_status or inbox, 0.0 otherwise
+ */
+export const postCompactionDiscipline = createScorer({
+	name: "Post-Compaction Discipline",
+	description: "First suggested tool is swarm_status or inbox",
+	scorer: ({ output }) => {
+		try {
+			const prompt = JSON.parse(String(output)) as CompactionPrompt;
+			return scorePostCompactionDiscipline(prompt);
+		} catch (error) {
+			return {
+				score: 0,
+				message: `Failed to parse prompt: ${error}`,
+			};
+		}
+	},
+});

package/evals/scorers/compaction-scorers.ts

@@ -260,14 +260,14 @@ export const forbiddenPatternsAbsent = createScorer({
 export const compactionQuality = createScorer({
   name: "Overall Compaction Quality",
   description: "Composite score for compaction hook correctness",
-  scorer: ({ output, expected }) => {
+  scorer: async ({ output, expected, input }) => {
     try {
       // Run all scorers
       const scores = {
-        confidence: confidenceAccuracy.scorer({ output, expected }),
-        injection: contextInjectionCorrectness.scorer({ output, expected }),
-        required: requiredPatternsPresent.scorer({ output, expected }),
-        forbidden: forbiddenPatternsAbsent.scorer({ output, expected }),
+        confidence: await confidenceAccuracy({ output, expected, input }),
+        injection: await contextInjectionCorrectness({ output, expected, input }),
+        required: await requiredPatternsPresent({ output, expected, input }),
+        forbidden: await forbiddenPatternsAbsent({ output, expected, input }),
       };
 
       // Weighted average
@@ -279,16 +279,16 @@ export const compactionQuality = createScorer({
       };
 
       const totalScore =
-        scores.confidence.score * weights.confidence +
-        scores.injection.score * weights.injection +
-        scores.required.score * weights.required +
-        scores.forbidden.score * weights.forbidden;
+        (scores.confidence.score ?? 0) * weights.confidence +
+        (scores.injection.score ?? 0) * weights.injection +
+        (scores.required.score ?? 0) * weights.required +
+        (scores.forbidden.score ?? 0) * weights.forbidden;
 
       const details = [
-        `Confidence: ${(scores.confidence.score * 100).toFixed(0)}%`,
-        `Injection: ${(scores.injection.score * 100).toFixed(0)}%`,
-        `Required: ${(scores.required.score * 100).toFixed(0)}%`,
-        `Forbidden: ${(scores.forbidden.score * 100).toFixed(0)}%`,
+        `Confidence: ${((scores.confidence.score ?? 0) * 100).toFixed(0)}%`,
+        `Injection: ${((scores.injection.score ?? 0) * 100).toFixed(0)}%`,
+        `Required: ${((scores.required.score ?? 0) * 100).toFixed(0)}%`,
+        `Forbidden: ${((scores.forbidden.score ?? 0) * 100).toFixed(0)}%`,
       ].join(", ");
 
       return {