opencode-swarm-plugin 0.38.0 → 0.40.0

package/src/compaction-prompt-scorers.test.ts

@@ -0,0 +1,299 @@
+/**
+ * Tests for compaction prompt quality scorers
+ *
+ * TDD approach - tests written FIRST to define scorer behavior
+ * Tests the PURE scoring functions (not evalite wrappers)
+ */
+
+import { describe, expect, test } from "bun:test";
+import type { CompactionPrompt } from "./compaction-prompt-scoring.js";
+import {
+	scoreActionability,
+	scoreCoordinatorIdentity,
+	scoreEpicIdSpecificity,
+	scoreForbiddenToolsPresent,
+	scorePostCompactionDiscipline,
+} from "./compaction-prompt-scoring.js";
+
+describe("epicIdSpecificity scorer", () => {
+	test("scores 1.0 for real epic IDs", () => {
+		const prompt: CompactionPrompt = {
+			content: "Continue coordinating epic mjkw81rkq4c",
+		};
+
+		const result = scoreEpicIdSpecificity(prompt);
+
+		expect(result.score).toBe(1.0);
+		expect(result.message).toContain("real epic ID");
+	});
+
+	test("scores 0.0 for placeholder IDs like <epic-id>", () => {
+		const prompt: CompactionPrompt = {
+			content: "Continue coordinating epic <epic-id>",
+		};
+
+		const result = scoreEpicIdSpecificity(prompt);
+
+		expect(result.score).toBe(0.0);
+		expect(result.message).toContain("placeholder");
+	});
+
+	test("scores 0.0 for bd-xxx placeholders", () => {
+		const prompt: CompactionPrompt = {
+			content: "Check status of bd-xxx",
+		};
+
+		const result = scoreEpicIdSpecificity(prompt);
+
+		expect(result.score).toBe(0.0);
+		expect(result.message).toContain("placeholder");
+	});
+
+	test("scores 0.0 for generic <path> placeholders", () => {
+		const prompt: CompactionPrompt = {
+			content: "Project at <path>",
+		};
+
+		const result = scoreEpicIdSpecificity(prompt);
+
+		expect(result.score).toBe(0.0);
+	});
+
+	test("scores 0.0 when no epic ID found", () => {
+		const prompt: CompactionPrompt = {
+			content: "Continue working on the task",
+		};
+
+		const result = scoreEpicIdSpecificity(prompt);
+
+		expect(result.score).toBe(0.0);
+		expect(result.message).toContain("No epic ID");
+	});
+});
+
+describe("actionability scorer", () => {
+	test("scores 1.0 when swarm_status has real epic ID", () => {
+		const prompt: CompactionPrompt = {
+			content: `First action:
+swarm_status(epic_id='mjkw81rkq4c', project_key='/path/to/project')`,
+		};
+
+		const result = scoreActionability(prompt);
+
+		expect(result.score).toBe(1.0);
+		expect(result.message).toContain("actionable tool call");
+	});
+
+	test("scores 1.0 when swarmmail_inbox is present", () => {
+		const prompt: CompactionPrompt = {
+			content: `Check messages:
+swarmmail_inbox()`,
+		};
+
+		const result = scoreActionability(prompt);
+
+		expect(result.score).toBe(1.0);
+		expect(result.message).toContain("actionable tool call");
+	});
+
+	test("scores 0.0 for generic instructions without tool calls", () => {
+		const prompt: CompactionPrompt = {
+			content: "Check the status of workers and review progress",
+		};
+
+		const result = scoreActionability(prompt);
+
+		expect(result.score).toBe(0.0);
+		expect(result.message).toContain("No actionable");
+	});
+
+	test("scores 0.0 for swarm_status with placeholders", () => {
+		const prompt: CompactionPrompt = {
+			content: `swarm_status(epic_id='<epic-id>', project_key='<path>')`,
+		};
+
+		const result = scoreActionability(prompt);
+
+		expect(result.score).toBe(0.0);
+		expect(result.message).toContain("placeholder");
+	});
+});
+
+describe("coordinatorIdentity scorer", () => {
+	test("scores 1.0 with ASCII header and strong mandates", () => {
+		const prompt: CompactionPrompt = {
+			content: `┌─────────────────────────────────────────┐
+│     YOU ARE THE COORDINATOR             │
+│                                         │
+│  NEVER spawn workers yourself           │
+│  ALWAYS review worker output            │
+└─────────────────────────────────────────┘
+
+Continue coordinating the swarm.`,
+		};
+
+		const result = scoreCoordinatorIdentity(prompt);
+
+		expect(result.score).toBe(1.0);
+		expect(result.message).toContain("ASCII header");
+		expect(result.message).toContain("strong mandates");
+	});
+
+	test("scores 0.5 with ASCII header but weak language", () => {
+		const prompt: CompactionPrompt = {
+			content: `┌─────────────────────────────────────────┐
+│     COORDINATOR MODE                    │
+└─────────────────────────────────────────┘
+
+You should consider delegating work.`,
+		};
+
+		const result = scoreCoordinatorIdentity(prompt);
+
+		expect(result.score).toBe(0.5);
+		expect(result.message).toContain("weak language");
+	});
+
+	test("scores 0.0 without ASCII header", () => {
+		const prompt: CompactionPrompt = {
+			content: `You are the coordinator. NEVER do work directly. ALWAYS delegate.`,
+		};
+
+		const result = scoreCoordinatorIdentity(prompt);
+
+		expect(result.score).toBe(0.0);
+		expect(result.message).toContain("No ASCII header");
+	});
+});
+
+describe("forbiddenToolsPresent scorer", () => {
+	test("scores 1.0 when all forbidden tools listed", () => {
+		const prompt: CompactionPrompt = {
+			content: `🚫 FORBIDDEN TOOLS - NEVER call these:
+- Edit (use swarm_spawn_subtask)
+- Write (use swarm_spawn_subtask)
+- swarmmail_reserve (only workers reserve)
+- bash with git commit (workers commit)`,
+		};
+
+		const result = scoreForbiddenToolsPresent(prompt);
+
+		expect(result.score).toBe(1.0);
+		expect(result.message).toContain("All 4 forbidden tools");
+	});
+
+	test("scores 0.75 when 3 out of 4 tools listed", () => {
+		const prompt: CompactionPrompt = {
+			content: `🚫 FORBIDDEN TOOLS:
+- Edit
+- Write
+- swarmmail_reserve`,
+		};
+
+		const result = scoreForbiddenToolsPresent(prompt);
+
+		expect(result.score).toBe(0.75);
+		expect(result.message).toContain("3/4");
+	});
+
+	test("scores 0.5 when 2 out of 4 tools listed", () => {
+		const prompt: CompactionPrompt = {
+			content: `Don't use Edit or Write directly.`,
+		};
+
+		const result = scoreForbiddenToolsPresent(prompt);
+
+		expect(result.score).toBe(0.5);
+		expect(result.message).toContain("2/4");
+	});
+
+	test("scores 0.0 when no forbidden tools listed", () => {
+		const prompt: CompactionPrompt = {
+			content: "Continue coordinating the epic",
+		};
+
+		const result = scoreForbiddenToolsPresent(prompt);
+
+		expect(result.score).toBe(0.0);
+		expect(result.message).toContain("0/4");
+	});
+});
+
+describe("postCompactionDiscipline scorer", () => {
+	test("scores 1.0 when first tool is swarm_status", () => {
+		const prompt: CompactionPrompt = {
+			content: `Resume coordination:
+
+1. swarm_status(epic_id='mjkw81rkq4c')
+2. Check inbox
+3. Review progress`,
+		};
+
+		const result = scorePostCompactionDiscipline(prompt);
+
+		expect(result.score).toBe(1.0);
+		expect(result.message).toContain("swarm_status");
+		expect(result.message).toContain("correct");
+	});
+
+	test("scores 1.0 when first tool is swarmmail_inbox", () => {
+		const prompt: CompactionPrompt = {
+			content: `Next steps:
+1. swarmmail_inbox()
+2. Review messages`,
+		};
+
+		const result = scorePostCompactionDiscipline(prompt);
+
+		expect(result.score).toBe(1.0);
+		expect(result.message).toContain("inbox");
+		expect(result.message).toContain("correct");
+	});
+
+	test("scores 0.0 when first tool is Edit", () => {
+		const prompt: CompactionPrompt = {
+			content: `Resume:
+1. Edit(file='src/auth.ts', ...)
+2. Check status`,
+		};
+
+		const result = scorePostCompactionDiscipline(prompt);
+
+		expect(result.score).toBe(0.0);
+		expect(result.message).toContain("Edit");
+	});
+
+	test("scores 0.0 when first tool is Write", () => {
+		const prompt: CompactionPrompt = {
+			content: `1. Write(file='README.md', ...)`,
+		};
+
+		const result = scorePostCompactionDiscipline(prompt);
+
+		expect(result.score).toBe(0.0);
+		expect(result.message).toContain("Write");
+	});
+
+	test("scores 0.0 when first tool is Read", () => {
+		const prompt: CompactionPrompt = {
+			content: `1. Read(file='src/index.ts')
+2. swarm_status()`,
+		};
+
+		const result = scorePostCompactionDiscipline(prompt);
+
+		expect(result.score).toBe(0.0);
+		expect(result.message).toContain("Read");
+	});
+
+	test("scores 0.0 when no tool calls mentioned", () => {
+		const prompt: CompactionPrompt = {
+			content: "Continue coordinating the epic",
+		};
+
+		const result = scorePostCompactionDiscipline(prompt);
+
+		expect(result.score).toBe(0.0);
+		expect(result.message).toContain("No tool");
+	});
+});

package/src/compaction-prompt-scoring.ts

@@ -0,0 +1,298 @@
+/**
+ * Compaction Prompt Quality Scoring - Pure Functions
+ *
+ * Evaluates the quality of continuation prompts generated after context compaction.
+ * **Problem**: Post-compaction coordinators often "wake up" confused, forget their role,
+ * and start editing files instead of checking worker status.
+ *
+ * **Solution**: Score prompts on 5 dimensions that predict coordinator success:
+ *
+ * 1. **Epic ID Specificity (0.20)**: Real IDs (`mjkw...`) not placeholders (`<epic-id>`, `bd-xxx`)
+ *    - Placeholders = coordinator can't check actual swarm status
+ *
+ * 2. **Actionability (0.20)**: Tool calls with real values (e.g., `swarm_status(epic_id='mjkw81rkq4c')`)
+ *    - Generic instructions like "check status" don't work
+ *
+ * 3. **Coordinator Identity (0.25)**: ASCII header + strong mandates (NEVER/ALWAYS)
+ *    - Visual + semantic cues reinforce role post-compaction
+ *
+ * 4. **Forbidden Tools Listed (0.15)**: Explicitly lists Edit, Write, swarmmail_reserve, git commit
+ *    - Naming forbidden tools reduces violations
+ *
+ * 5. **Post-Compaction Discipline (0.20)**: First suggested tool is swarm_status or inbox (not Edit)
+ *    - First tool sets the pattern - "check status" vs "dive into code"
+ *
+ * **Pure functions**: These can be tested without evalite. The evalite wrappers are in
+ * `evals/scorers/compaction-prompt-scorers.ts`.
+ *
+ * **Data source**: Captured from `captureCompactionEvent()` with `compaction_type: "prompt_generated"`.
+ * The payload includes the FULL prompt content (not truncated) for scoring.
+ *
+ * **Integration**: `compaction-prompt.eval.ts` uses these scorers to track prompt quality over time.
+ * Progressive gates enforce quality: bootstrap → stabilization → production.
+ *
+ * @module compaction-prompt-scoring
+ */
+
+/**
+ * Compaction prompt structure (from LLM generation)
+ */
+export interface CompactionPrompt {
+	content: string;
+}
+
+/**
+ * Scorer result type
+ */
+export interface ScorerResult {
+	score: number;
+	message: string;
+}
+
+// ====== Shared Regex Patterns ======
+
+/** Matches real epic/cell IDs (mjkw prefix + 7+ base36 chars) */
+export const REAL_EPIC_ID = /mjkw[a-z0-9]{7,}/;
+
+/** Matches common placeholder patterns */
+export const PLACEHOLDERS = [
+	/<epic-id>/i,
+	/bd-xxx/,
+	/<path>/i,
+	/<project>/i,
+];
+
+/** Matches ASCII box-drawing characters (for headers) */
+export const ASCII_BOX = /[┌┐└┘─│]{3,}/;
+
+/** Matches strong mandate language */
+export const STRONG_LANGUAGE = [/\bNEVER\b/, /\bALWAYS\b/, /\bNON-NEGOTIABLE\b/];
+
+// ====== Pure Scoring Functions ======
+
+/**
+ * Score epic ID specificity
+ *
+ * Validates that epic IDs are REAL, not placeholders.
+ * Placeholders like <epic-id>, bd-xxx, <path> indicate
+ * the prompt generator failed to inject actual values.
+ *
+ * @returns 1.0 if real IDs, 0.0 if placeholders found
+ */
+export function scoreEpicIdSpecificity(prompt: CompactionPrompt): ScorerResult {
+	// Check for placeholder patterns
+	for (const pattern of PLACEHOLDERS) {
+		if (pattern.test(prompt.content)) {
+			return {
+				score: 0.0,
+				message: `Found placeholder: ${pattern.source}`,
+			};
+		}
+	}
+
+	// Check for real epic ID pattern
+	if (REAL_EPIC_ID.test(prompt.content)) {
+		return {
+			score: 1.0,
+			message: "Contains real epic ID",
+		};
+	}
+
+	return {
+		score: 0.0,
+		message: "No epic ID found",
+	};
+}
+
+/**
+ * Score actionability of tool calls
+ *
+ * Validates that the prompt includes SPECIFIC actionable tool calls.
+ * Generic instructions like "check status" are useless.
+ * Good: swarm_status(epic_id='mjkw81rkq4c', project_key='/path')
+ * Bad: "Check the status of workers"
+ *
+ * @returns 1.0 if actionable tool calls with real values, 0.0 otherwise
+ */
+export function scoreActionability(prompt: CompactionPrompt): ScorerResult {
+	// Check for actionable tool patterns
+	const actionableTools = [
+		/swarm_status\([^)]*epic_id\s*=\s*['"]mjkw[a-z0-9]{7,}['"]/,
+		/swarmmail_inbox\(\)/,
+	];
+
+	for (const pattern of actionableTools) {
+		if (pattern.test(prompt.content)) {
+			return {
+				score: 1.0,
+				message: "Contains actionable tool call with real values",
+			};
+		}
+	}
+
+	// Check if tool is mentioned but with placeholders
+	if (
+		/swarm_status\([^)]*<epic-id>/.test(prompt.content) ||
+		/swarm_status\([^)]*<path>/.test(prompt.content)
+	) {
+		return {
+			score: 0.0,
+			message: "Tool call has placeholders",
+		};
+	}
+
+	return {
+		score: 0.0,
+		message: "No actionable tool calls found",
+	};
+}
+
+/**
+ * Score coordinator identity reinforcement
+ *
+ * Validates that the prompt has STRONG coordinator identity reinforcement.
+ * Post-compaction coordinators lose their identity without visual+semantic cues.
+ *
+ * Checks:
+ * 1. ASCII box header (visual anchor)
+ * 2. Strong language (NEVER/ALWAYS, not "should"/"consider")
+ *
+ * @returns 1.0 for ASCII header + strong mandates, 0.5 for header only, 0.0 otherwise
+ */
+export function scoreCoordinatorIdentity(
+	prompt: CompactionPrompt,
+): ScorerResult {
+	// Check for ASCII box header (uses box-drawing characters)
+	const hasAsciiHeader =
+		ASCII_BOX.test(prompt.content) &&
+		/(YOU ARE THE COORDINATOR|COORDINATOR MODE)/i.test(prompt.content);
+
+	if (!hasAsciiHeader) {
+		return {
+			score: 0.0,
+			message: "No ASCII header found",
+		};
+	}
+
+	// Check for strong mandate language
+	const hasStrongLanguage = STRONG_LANGUAGE.some((pattern) =>
+		pattern.test(prompt.content),
+	);
+
+	if (!hasStrongLanguage) {
+		return {
+			score: 0.5,
+			message: "ASCII header present but weak language",
+		};
+	}
+
+	return {
+		score: 1.0,
+		message: "ASCII header + strong mandates present",
+	};
+}
+
+/**
+ * Score forbidden tools listing
+ *
+ * Validates that the prompt LISTS forbidden tools by name.
+ * Coordinators must know exactly which tools to avoid.
+ *
+ * Required forbidden tools:
+ * 1. Edit
+ * 2. Write
+ * 3. swarmmail_reserve (only workers reserve)
+ * 4. git commit (workers commit)
+ *
+ * @returns ratio of forbidden tools mentioned (0.0 to 1.0)
+ */
+export function scoreForbiddenToolsPresent(
+	prompt: CompactionPrompt,
+): ScorerResult {
+	// Check for forbidden tool mentions
+	const forbiddenTools = [
+		/\bEdit\b/,
+		/\bWrite\b/,
+		/swarmmail_reserve/,
+		/git commit/,
+	];
+
+	const foundTools = forbiddenTools.filter((pattern) =>
+		pattern.test(prompt.content),
+	);
+
+	const score = foundTools.length / forbiddenTools.length;
+
+	if (score === 1.0) {
+		return {
+			score: 1.0,
+			message: "All 4 forbidden tools listed",
+		};
+	}
+
+	if (score === 0) {
+		return {
+			score: 0.0,
+			message: "No forbidden tools listed (0/4)",
+		};
+	}
+
+	return {
+		score,
+		message: `${foundTools.length}/4 forbidden tools listed`,
+	};
+}
+
+/**
+ * Score post-compaction discipline (first tool correctness)
+ *
+ * Validates that the FIRST suggested tool is correct.
+ * Coordinators should check status FIRST, not edit files.
+ *
+ * Good first tools:
+ * - swarm_status
+ * - swarmmail_inbox
+ *
+ * Bad first tools:
+ * - Edit
+ * - Write
+ * - Read (should check status first)
+ *
+ * @returns 1.0 if first tool is swarm_status or inbox, 0.0 otherwise
+ */
+export function scorePostCompactionDiscipline(
+	prompt: CompactionPrompt,
+): ScorerResult {
+	// Extract first tool call (look for function-like patterns)
+	const toolCallPattern =
+		/\b(swarm_status|swarmmail_inbox|Edit|Write|Read)\b/i;
+	const match = prompt.content.match(toolCallPattern);
+
+	if (!match) {
+		return {
+			score: 0.0,
+			message: "No tool calls found",
+		};
+	}
+
+	const firstTool = match[1].toLowerCase();
+
+	if (firstTool === "swarm_status") {
+		return {
+			score: 1.0,
+			message: "First tool is swarm_status (correct)",
+		};
+	}
+
+	if (firstTool === "swarmmail_inbox") {
+		return {
+			score: 1.0,
+			message: "First tool is inbox (correct)",
+		};
+	}
+
+	return {
+		score: 0.0,
+		message: `First tool is ${match[1]} (should be swarm_status or inbox)`,
+	};
+}