@swarmtools/evals 0.2.0

package/README.md

@@ -0,0 +1,53 @@
+# @swarmtools/evals
+
+```
+    🐝   EVAL SUITE   🐝
+━━━━━━━━━━━━━━━━━━━━━━━━━━
+  Swarm Intelligence QA
+```
+
+Evaluation suite for swarm-tools multi-agent coordination. Uses [Evalite](https://evalite.dev) to measure coordinator behavior, decomposition quality, and compaction correctness.
+
+## Purpose
+
+This package contains the evaluation framework for the swarm-tools ecosystem. Extracting evals into a separate package ensures:
+
+1. **Clean Dependencies** - Main plugin doesn't need evalite/vitest in production
+2. **Faster Installs** - Eval deps only needed for development/CI
+3. **Isolated Testing** - Eval suite can evolve independently from plugin
+
+## What Gets Evaluated
+
+- **Coordinator Protocol** - Does the coordinator spawn workers vs doing work itself?
+- **Task Decomposition** - Quality of task splitting, file conflict detection
+- **Compaction** - Context compression correctness
+- **Review Thoroughness** - Does coordinator review worker output properly?
+
+## Usage
+
+```bash
+# Run all evals
+bun run test
+
+# Build for publishing
+bun run build
+
+# Type check
+bun run typecheck
+```
+
+## Package Structure
+
+This package is part of the swarm-tools monorepo:
+
+- `opencode-swarm-plugin` - Main plugin (peer dependency)
+- `swarm-mail` - Event sourcing primitives (peer dependency)
+- `@swarmtools/evals` - This package
+
+## Development
+
+Evals use real coordinator sessions captured to `~/.config/swarm-tools/sessions/*.jsonl`. See the main plugin's `evals/README.md` for details on session capture.
+
+## License
+
+MIT

package/dist/compaction-prompt.eval.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Compaction Prompt Quality Evaluation
+ *
+ * Tests that continuation prompts generated after context compaction meet
+ * quality criteria for coordinator resumption:
+ *
+ * 1. Epic ID Specificity (20%) - Real IDs not placeholders
+ * 2. Actionability (20%) - Specific tool calls with real values
+ * 3. Coordinator Identity (25%) - ASCII header + strong mandates
+ * 4. Forbidden Tools (15%) - Lists forbidden tools by name
+ * 5. Post-Compaction Discipline (20%) - First tool is correct
+ *
+ * ## Why This Matters
+ *
+ * After compaction, coordinators lose context. The continuation prompt is
+ * their ONLY guide to resume. Bad prompts cause:
+ * - Coordinators editing files (should delegate to workers)
+ * - Generic "check status" instead of actual tool calls
+ * - Lost epic IDs (can't resume coordination)
+ *
+ * ## Test Strategy
+ *
+ * - 6 synthetic fixtures covering perfect/bad prompts
+ * - Each fixture tests specific failure modes
+ * - Composite scorer validates overall quality
+ *
+ * Run with: bun run eval:compaction
+ */
+export {};
+//# sourceMappingURL=compaction-prompt.eval.d.ts.map

package/dist/compaction-prompt.eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compaction-prompt.eval.d.ts","sourceRoot":"","sources":["../src/compaction-prompt.eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG"}

package/dist/compaction-resumption.eval.d.ts

@@ -0,0 +1,23 @@
+/**
+ * Compaction Hook Coordinator Resumption Eval
+ *
+ * Tests that the compaction hook correctly detects swarm state and injects
+ * appropriate context for coordinator resumption.
+ *
+ * ## Bug Being Tested
+ *
+ * Root cause: The compaction hook injects generic "you are a coordinator"
+ * context but doesn't include the SPECIFIC epic ID, subtask status, or
+ * project path. This causes coordinators to lose identity after compaction.
+ *
+ * ## Test Cases
+ *
+ * 1. Active swarm with in_progress epic - should inject full context with epic ID
+ * 2. Multiple epics - should identify the in_progress one
+ * 3. No active swarm - should not inject coordinator context
+ * 4. Blocked epic - should still detect as active swarm
+ *
+ * Run with: pnpm eval:dev (watch mode) or pnpm eval:run (once)
+ */
+export {};
+//# sourceMappingURL=compaction-resumption.eval.d.ts.map

package/dist/compaction-resumption.eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compaction-resumption.eval.d.ts","sourceRoot":"","sources":["../src/compaction-resumption.eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG"}

package/dist/coordinator-behavior.eval.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Coordinator Behavior After Compaction Eval
+ *
+ * LLM-as-judge eval that tests whether the compaction context actually
+ * causes Claude to behave like a coordinator (spawn workers, check status)
+ * rather than a worker (run tests, edit files directly).
+ *
+ * This is the missing piece - we test the CONTEXT CONTENT in unit tests,
+ * but we need to test whether the LLM BEHAVES CORRECTLY given that context.
+ *
+ * Run with: bunx evalite run evals/coordinator-behavior.eval.ts
+ */
+/**
+ * Scores whether the response mentions coordinator tools
+ */
+export declare const mentionsCoordinatorTools: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Scores whether the response avoids worker behaviors
+ */
+export declare const avoidsWorkerBehaviors: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Scores whether the response shows coordinator mindset
+ */
+export declare const coordinatorMindset: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Composite scorer for overall coordinator behavior
+ */
+export declare const overallCoordinatorBehavior: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+//# sourceMappingURL=coordinator-behavior.eval.d.ts.map

package/dist/coordinator-behavior.eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"coordinator-behavior.eval.d.ts","sourceRoot":"","sources":["../src/coordinator-behavior.eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AA2DH;;GAEG;AACH,eAAO,MAAM,wBAAwB,6DAwBnC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,qBAAqB,6DAiChC,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,kBAAkB,6DAyC7B,CAAC;AAEH;;GAEG;AACH,eAAO,MAAM,0BAA0B,6DAmBrC,CAAC"}

package/dist/coordinator-session.eval.d.ts

@@ -0,0 +1,26 @@
+/**
+ * Coordinator Session Eval - Scores Real Captured Sessions
+ *
+ * Tests that coordinators follow protocol:
+ * 1. Don't edit files directly (spawn workers)
+ * 2. Don't run tests directly (workers do verification)
+ * 3. Spawn workers for all subtasks
+ * 4. Review worker output before accepting
+ * 5. Minimize time to first spawn (don't overthink)
+ *
+ * ## Data Sources
+ *
+ * - **Real sessions**: Captured from ~/.config/swarm-tools/sessions/*.jsonl
+ * - **Synthetic fixtures**: Test cases in fixtures/coordinator-sessions.ts
+ *
+ * ## Test Flow
+ *
+ * 1. Load captured sessions from disk (via loadCapturedSessions)
+ * 2. Load synthetic fixtures for baseline validation
+ * 3. Run coordinator-discipline scorers on all sessions
+ * 4. Output scores and violation details
+ *
+ * Run with: pnpm eval:dev (watch mode) or pnpm eval:run (once)
+ */
+export {};
+//# sourceMappingURL=coordinator-session.eval.d.ts.map

package/dist/coordinator-session.eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"coordinator-session.eval.d.ts","sourceRoot":"","sources":["../src/coordinator-session.eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG"}

package/dist/example.eval.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Example eval file to test Evalite setup
+ *
+ * This is a minimal test to verify:
+ * 1. Evalite CLI can discover .eval.ts files
+ * 2. createScorer works
+ * 3. evalite() function works
+ */
+export {};
+//# sourceMappingURL=example.eval.d.ts.map

package/dist/example.eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"example.eval.d.ts","sourceRoot":"","sources":["../src/example.eval.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG"}

package/dist/fixtures/cass-baseline.d.ts

@@ -0,0 +1,147 @@
+/**
+ * CASS Baseline Response Fixtures
+ *
+ * These fixtures capture the ACTUAL behavior of the CASS binary tools.
+ * DO NOT modify to match desired behavior - these document what the binary DOES.
+ *
+ * Purpose: Characterization tests for ADR-010 (CASS inhousing).
+ * These ensure our inhouse implementation matches the binary's behavior.
+ */
+/**
+ * cass stats --json
+ * Captured: 2025-12-25
+ */
+export declare const cassStatsBaseline: {
+    readonly by_agent: readonly [{
+        readonly agent: "claude_code";
+        readonly count: 137;
+    }, {
+        readonly agent: "cursor";
+        readonly count: 23;
+    }, {
+        readonly agent: "codex";
+        readonly count: 2;
+    }];
+    readonly conversations: 162;
+    readonly date_range: {
+        readonly newest: "2025-12-08T04:20:36.526+00:00";
+        readonly oldest: "2025-07-14T01:14:44.997+00:00";
+    };
+    readonly db_path: "/Users/joel/Library/Application Support/com.coding-agent-search.coding-agent-search/agent_search.db";
+    readonly messages: 4213;
+    readonly top_workspaces: readonly [{
+        readonly count: 28;
+        readonly workspace: "/Users/joel/Code/vercel/academy-vectr-workflow-course-content/external/workflow-builder-starter";
+    }, {
+        readonly count: 22;
+        readonly workspace: "/Users/joel/Code/vercel/slack-agents-course";
+    }];
+};
+/**
+ * cass search "swarm" --limit 2 --json
+ * Captured: 2025-12-25
+ */
+export declare const cassSearchBaseline: {
+    readonly count: 2;
+    readonly cursor: null;
+    readonly hits: readonly [{
+        readonly agent: "claude_code";
+        readonly content: "Fixed. The `plugins` key is invalid - OpenCode auto-loads plugins from directories instead.\n\n**Changes:**\n1. ✅ Removed invalid `plugins` array from `opencode.jsonc`\n2. ✅ Created `~/.config/opencode/plugin/` directory\n3. ✅ Symlinked your swarm plugin → `~/.config/opencode/plugin/swarm.js`\n\nThe plugin will now auto-load on startup. Restart OpenCode to pick it up.\n\nSources:\n- [OpenCode Plugins Documentation](https://opencode.ai/docs/plugins/)\n- [OpenCode Config Documentation](https://opencode.ai/docs/config/)";
+        readonly created_at: 1765161767083;
+        readonly line_number: 9;
+        readonly match_type: "exact";
+        readonly score: 15.536974906921387;
+        readonly snippet: "Symlinked your swarm plugin → `~/.config/opencode/plugin/swarm.js`\n\nThe plugin will now auto-load on startup. Restart OpenCode to pick it up.\n\nSources:\n- [OpenC…";
+        readonly source_path: "/Users/joel/.claude/projects/-Users-joel--config-opencode/ccd64ac6-bca7-40e5-9150-cea58c3788ae.jsonl";
+        readonly title: "@opencode.jsonc has an invalid plugins key https://opencode.ai/docs/plugins/ https://opencode.ai/doc";
+        readonly workspace: "/Users/joel/.config/opencode";
+    }, {
+        readonly agent: "claude_code";
+        readonly content: "I'm ready to help you explore the codebase and design implementation plans. I'm in **READ-ONLY mode** - I can explore files, understand architecture, and create detailed plans, but I cannot and will not modify any files.\n\nI have access to the beads issue tracker (`bd` commands) and can see your current working directory is `/Users/joel/.config/opencode`.\n\n**Current git status shows:**\n- Modified: `.beads/issues.jsonl`, `AGENTS.md`, `command/swarm.md`, `opencode.jsonc`\n- Untracked: `command/swarm-collect.md`, `command/swarm-status.md`, `plugin/`\n\n**What would you like me to explore and plan?**\n\nCommon scenarios I can help with:\n- Designing new command implementations\n- Planning plugin architecture\n- Exploring existing patterns for feature additions\n- Creating implementation strategies for beads issues\n\nLet me know what you need, and I'll dive into the codebase, understand the current architecture, and provide a detailed implementation plan.";
+        readonly created_at: 1765161814722;
+        readonly line_number: 1;
+        readonly match_type: "exact";
+        readonly score: 14.522254943847656;
+        readonly snippet: ".md`, `command/swarm.md`, `opencode.jsonc`\n- Untracked: `command/swarm-collect.md`, `command/swarm-status.md`, `plugin/`\n\n**What would you like me to explore an…";
+        readonly source_path: "/Users/joel/.claude/projects/-Users-joel--config-opencode/agent-ee2a73ee.jsonl";
+        readonly title: "opencode";
+        readonly workspace: "/Users/joel/.config/opencode";
+    }];
+    readonly hits_clamped: false;
+    readonly limit: 2;
+    readonly max_tokens: null;
+    readonly offset: 0;
+    readonly query: "swarm";
+    readonly request_id: null;
+    readonly total_matches: 2;
+};
+/**
+ * cass health (human-readable output)
+ * Captured: 2025-12-25
+ */
+export declare const cassHealthHumanBaseline = "\u2713 Healthy (3ms)\n  Note: index stale (older than 300s)";
+/**
+ * cass stats (human-readable output)
+ * Captured: 2025-12-25
+ */
+export declare const cassStatsHumanBaseline = "CASS Index Statistics\n=====================\nDatabase: /Users/joel/Library/Application Support/com.coding-agent-search.coding-agent-search/agent_search.db\n\nTotals:\n  Conversations: 162\n  Messages: 4213\n\nBy Agent:\n  claude_code: 137\n  cursor: 23\n  codex: 2\n\nTop Workspaces:\n  /Users/joel/Code/vercel/academy-vectr-workflow-course-content/external/workflow-builder-starter: 28\n  /Users/joel/Code/vercel/slack-agents-course: 22\n  /Users/joel/Code/vercel/academy-vectr-workflow-course-content: 22\n  /Users/joel/Code/vercel/academy-content: 13\n  /Users/joel/Code/joelhooks/trt-buddy: 13\n  /Users/joel/Code/vercel/front: 11\n  /Users/joel/.config/opencode: 9\n  /Users/joel: 6\n  /Users/joel/Code/badass-courses/course-builder/apps/ai-hero: 5\n  /Users/joel/Code/vercel/front/apps/vercel-academy: 4\n\nDate Range: 2025-07-14 to 2025-12-08";
+/**
+ * cass view <file> -n <line>
+ * Captured: 2025-12-25
+ *
+ * Format: File path header, line indicator with context window, separator, content with line numbers
+ */
+export declare const cassViewBaseline = "File: /Users/joel/.config/swarm-tools/sessions/ses_19yz2iaMpHxY1ddvVq2voC.jsonl\nLine: 1 (context: 5)\n----------------------------------------\n>    1 | {\"session_id\":\"ses_19yz2iaMpHxY1ddvVq2voC\",\"epic_id\":\"cell-f2p61v-mjko4d89zdt\",\"timestamp\":\"2025-12-24T23:51:52.896Z\",\"event_type\":\"OUTCOME\",\"outcome_type\":\"subtask_success\",\"payload\":{\"bead_id\":\"cell-f2p61v-mjko4d89zdt\",\"duration_ms\":0,\"files_touched\":[],\"verification_passed\":false,\"verification_skipped\":true}}\n----------------------------------------";
+/**
+ * Error responses (captured from actual failures)
+ */
+export declare const cassErrorBaseline: {
+    readonly fileNotFound: {
+        readonly error: {
+            readonly code: 3;
+            readonly hint: null;
+            readonly kind: "file-not-found";
+            readonly message: "File not found: /Users/joel/.config/swarm-tools/sessions/ses_fRrFb7WrNr9K89JBCKd6GV.jsonl";
+            readonly retryable: false;
+        };
+    };
+    readonly invalidArgument: {
+        readonly error: {
+            readonly code: 2;
+            readonly hint: {
+                readonly common_mistakes: readonly [{
+                    readonly correct: "cass robot-docs";
+                    readonly wrong: "cass --robot-docs";
+                }, {
+                    readonly correct: "cass robot-docs commands";
+                    readonly wrong: "cass --robot-docs=commands";
+                }, {
+                    readonly correct: "cass robot-docs";
+                    readonly wrong: "cass robot-docs --robot";
+                }];
+                readonly error: "error: unexpected argument '--robot' found\\n\\nUsage: cass stats [OPTIONS]\\n\\nFor more information, try '--help'.\\n";
+                readonly examples: readonly ["cass robot-docs commands", "cass robot-docs schemas", "cass robot-docs examples", "cass --robot-help"];
+                readonly flag_syntax: {
+                    readonly correct: readonly ["--limit 5", "--robot", "--json"];
+                    readonly incorrect: readonly ["-limit 5", "limit=5", "--Limit"];
+                };
+                readonly hints: readonly ["For get robot-mode documentation, try: cass --robot-help"];
+                readonly kind: "argument_parsing";
+                readonly status: "error";
+            };
+            readonly kind: "usage";
+            readonly message: "Could not parse arguments";
+            readonly retryable: false;
+        };
+    };
+};
+/**
+ * Schema definitions extracted from actual responses
+ */
+export type CassStatsResponse = typeof cassStatsBaseline;
+export type CassSearchResponse = typeof cassSearchBaseline;
+export type CassSearchHit = CassSearchResponse["hits"][number];
+export type CassAgentStats = CassStatsResponse["by_agent"][number];
+export type CassWorkspaceStats = CassStatsResponse["top_workspaces"][number];
+export type CassError = typeof cassErrorBaseline.fileNotFound | typeof cassErrorBaseline.invalidArgument;
+//# sourceMappingURL=cass-baseline.d.ts.map

package/dist/fixtures/cass-baseline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cass-baseline.d.ts","sourceRoot":"","sources":["../../src/fixtures/cass-baseline.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH;;;GAGG;AACH,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;CAkCpB,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,kBAAkB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2CrB,CAAC;AAEX;;;GAGG;AACH,eAAO,MAAM,uBAAuB,gEACE,CAAC;AAEvC;;;GAGG;AACH,eAAO,MAAM,sBAAsB,u1BAyBE,CAAC;AAEtC;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,oiBAIY,CAAC;AAE1C;;GAEG;AACH,eAAO,MAAM,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoDpB,CAAC;AAEX;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,OAAO,iBAAiB,CAAC;AACzD,MAAM,MAAM,kBAAkB,GAAG,OAAO,kBAAkB,CAAC;AAC3D,MAAM,MAAM,aAAa,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC;AAC/D,MAAM,MAAM,cAAc,GAAG,iBAAiB,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC;AACnE,MAAM,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC;AAC7E,MAAM,MAAM,SAAS,GACjB,OAAO,iBAAiB,CAAC,YAAY,GACrC,OAAO,iBAAiB,CAAC,eAAe,CAAC"}

package/dist/fixtures/compaction-cases.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Test cases for compaction hook coordinator resumption
+ *
+ * Each case simulates a different swarm state and verifies that
+ * the compaction hook injects the correct context for resumption.
+ */
+import type { Cell } from "swarm-mail";
+/**
+ * Compaction test case structure
+ */
+export interface CompactionTestCase {
+    name: string;
+    description: string;
+    /**
+     * Simulated hive state (cells to create)
+     */
+    hiveCells: Array<Omit<Cell, "created_at" | "updated_at" | "closed_at">>;
+    /**
+     * Simulated swarm-mail state
+     */
+    swarmMailState: {
+        agents: number;
+        reservations: number;
+        messages: number;
+    };
+    /**
+     * Expected detection confidence
+     */
+    expected: {
+        confidence: "high" | "medium" | "low" | "none";
+        contextInjected: boolean;
+        contextType: "full" | "fallback" | "none";
+        /**
+         * Patterns that MUST appear in injected context (if injected)
+         */
+        mustContain?: string[];
+        /**
+         * Patterns that MUST NOT appear
+         */
+        mustNotContain?: string[];
+    };
+}
+export declare const compactionCases: CompactionTestCase[];
+//# sourceMappingURL=compaction-cases.d.ts.map

package/dist/fixtures/compaction-cases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compaction-cases.d.ts","sourceRoot":"","sources":["../../src/fixtures/compaction-cases.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,YAAY,GAAG,YAAY,GAAG,WAAW,CAAC,CAAC,CAAC;IACxE;;OAEG;IACH,cAAc,EAAE;QACd,MAAM,EAAE,MAAM,CAAC;QACf,YAAY,EAAE,MAAM,CAAC;QACrB,QAAQ,EAAE,MAAM,CAAC;KAClB,CAAC;IACF;;OAEG;IACH,QAAQ,EAAE;QACR,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;QAC/C,eAAe,EAAE,OAAO,CAAC;QACzB,WAAW,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAAC;QAC1C;;WAEG;QACH,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;QACvB;;WAEG;QACH,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;KAC3B,CAAC;CACH;AAED,eAAO,MAAM,eAAe,EAAE,kBAAkB,EAgT/C,CAAC"}

package/dist/fixtures/compaction-prompt-cases.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Test cases for compaction prompt quality evaluation
+ *
+ * Each case represents a continuation prompt that should be generated
+ * after context compaction. Tests validate that prompts have:
+ * - Real epic IDs (not placeholders)
+ * - Actionable tool calls with specific values
+ * - Strong coordinator identity
+ * - Explicit forbidden tools list
+ * - Correct first tool suggestion
+ */
+import type { CompactionPrompt } from "opencode-swarm-plugin/compaction-prompt-scoring";
+/**
+ * Compaction prompt test case structure
+ */
+export interface CompactionPromptTestCase {
+    name: string;
+    description: string;
+    /**
+     * The generated continuation prompt
+     */
+    prompt: CompactionPrompt;
+    /**
+     * Expected scoring outcomes
+     */
+    expected: {
+        /**
+         * Should have real epic IDs (not placeholders)
+         */
+        hasRealEpicId: boolean;
+        /**
+         * Should have actionable tool calls
+         */
+        isActionable: boolean;
+        /**
+         * Should have strong coordinator identity
+         */
+        hasCoordinatorIdentity: boolean;
+        /**
+         * Should list forbidden tools by name
+         */
+        listsForbiddenTools: boolean;
+        /**
+         * First suggested tool should be correct
+         */
+        hasCorrectFirstTool: boolean;
+    };
+}
+export declare const compactionPromptCases: CompactionPromptTestCase[];
+//# sourceMappingURL=compaction-prompt-cases.d.ts.map

package/dist/fixtures/compaction-prompt-cases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compaction-prompt-cases.d.ts","sourceRoot":"","sources":["../../src/fixtures/compaction-prompt-cases.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iDAAiD,CAAC;AAExF;;GAEG;AACH,MAAM,WAAW,wBAAwB;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB;;OAEG;IACH,MAAM,EAAE,gBAAgB,CAAC;IACzB;;OAEG;IACH,QAAQ,EAAE;QACT;;WAEG;QACH,aAAa,EAAE,OAAO,CAAC;QACvB;;WAEG;QACH,YAAY,EAAE,OAAO,CAAC;QACtB;;WAEG;QACH,sBAAsB,EAAE,OAAO,CAAC;QAChC;;WAEG;QACH,mBAAmB,EAAE,OAAO,CAAC;QAC7B;;WAEG;QACH,mBAAmB,EAAE,OAAO,CAAC;KAC7B,CAAC;CACF;AAED,eAAO,MAAM,qBAAqB,EAAE,wBAAwB,EAmQ3D,CAAC"}

package/dist/fixtures/coordinator-sessions.d.ts

@@ -0,0 +1,73 @@
+/**
+ * Coordinator Session Test Fixtures
+ *
+ * Synthetic coordinator sessions for testing coordinator-discipline scorers.
+ * Each fixture demonstrates good or bad coordinator behavior.
+ */
+import type { CoordinatorSession } from "opencode-swarm-plugin/eval-capture";
+/**
+ * PERFECT COORDINATOR
+ *
+ * - No violations (no direct edits, tests, or reservations)
+ * - 100% spawn efficiency (3/3 workers spawned)
+ * - 100% review thoroughness (all workers reviewed)
+ * - Fast time to first spawn (30s)
+ */
+export declare const perfectCoordinator: CoordinatorSession;
+/**
+ * BAD COORDINATOR - Multiple Violations
+ *
+ * - 3 violations (edited file, ran tests, reserved files)
+ * - 33% spawn efficiency (only 1/3 workers spawned)
+ * - 0% review thoroughness (no reviews)
+ * - Slow time to first spawn (10 minutes)
+ */
+export declare const badCoordinator: CoordinatorSession;
+/**
+ * DECENT COORDINATOR - Some Issues
+ *
+ * - 1 violation (ran tests once)
+ * - 100% spawn efficiency (2/2 workers spawned)
+ * - 50% review thoroughness (reviewed only 1/2)
+ * - Good time to first spawn (45s)
+ */
+export declare const decentCoordinator: CoordinatorSession;
+/**
+ * All test fixtures
+ */
+export declare const coordinatorSessionFixtures: {
+    session_id: string;
+    epic_id: string;
+    start_time: string;
+    events: ({
+        session_id: string;
+        epic_id: string;
+        timestamp: string;
+        event_type: "DECISION";
+        decision_type: "strategy_selected" | "worker_spawned" | "review_completed" | "decomposition_complete" | "researcher_spawned" | "skill_loaded" | "inbox_checked" | "blocker_resolved" | "scope_change_approved" | "scope_change_rejected";
+        payload: any;
+    } | {
+        session_id: string;
+        epic_id: string;
+        timestamp: string;
+        event_type: "VIOLATION";
+        violation_type: "coordinator_edited_file" | "coordinator_ran_tests" | "coordinator_reserved_files" | "no_worker_spawned";
+        payload: any;
+    } | {
+        session_id: string;
+        epic_id: string;
+        timestamp: string;
+        event_type: "OUTCOME";
+        outcome_type: "subtask_success" | "subtask_retry" | "subtask_failed" | "epic_complete" | "blocker_detected";
+        payload: any;
+    } | {
+        session_id: string;
+        epic_id: string;
+        timestamp: string;
+        event_type: "COMPACTION";
+        compaction_type: "detection_complete" | "prompt_generated" | "context_injected" | "resumption_started" | "tool_call_tracked";
+        payload: any;
+    })[];
+    end_time?: string | undefined;
+}[];
+//# sourceMappingURL=coordinator-sessions.d.ts.map

package/dist/fixtures/coordinator-sessions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"coordinator-sessions.d.ts","sourceRoot":"","sources":["../../src/fixtures/coordinator-sessions.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,oCAAoC,CAAC;AAE7E;;;;;;;GAOG;AACH,eAAO,MAAM,kBAAkB,EAAE,kBAsHhC,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,cAAc,EAAE,kBA+E5B,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,iBAAiB,EAAE,kBAoF/B,CAAC;AAEF;;GAEG;AACH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAItC,CAAC"}

package/dist/fixtures/decomposition-cases.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Test cases for swarm task decomposition
+ *
+ * Each case includes:
+ * - input: task description and optional context
+ * - expected: validation criteria (min/max subtasks, required files)
+ */
+export interface DecompositionTestCase {
+    input: {
+        task: string;
+        context?: string;
+    };
+    expected: {
+        minSubtasks: number;
+        maxSubtasks: number;
+        requiredFiles?: string[];
+    };
+}
+export declare const decompositionCases: DecompositionTestCase[];
+//# sourceMappingURL=decomposition-cases.d.ts.map

package/dist/fixtures/decomposition-cases.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"decomposition-cases.d.ts","sourceRoot":"","sources":["../../src/fixtures/decomposition-cases.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,MAAM,WAAW,qBAAqB;IACpC,KAAK,EAAE;QACL,IAAI,EAAE,MAAM,CAAC;QACb,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;IACF,QAAQ,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;KAC1B,CAAC;CACH;AAED,eAAO,MAAM,kBAAkB,EAAE,qBAAqB,EAoFrD,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,8 @@
+/**
+ * @swarmtools/evals - Evaluation suite for swarm-tools
+ *
+ * Placeholder entry point. Actual eval implementations will be added when
+ * evals are migrated from opencode-swarm-plugin/evals/
+ */
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,CAAC"}

package/dist/lib/compaction-loader.d.ts

@@ -0,0 +1,103 @@
+import type { CoordinatorEvent } from "opencode-swarm-plugin/eval-capture";
+/**
+ * Compaction event - subset of CoordinatorEvent with event_type === "COMPACTION"
+ */
+export type CompactionEvent = Extract<CoordinatorEvent, {
+    event_type: "COMPACTION";
+}>;
+/**
+ * Compaction session - session with only COMPACTION events
+ */
+export interface CompactionSession {
+    session_id: string;
+    epic_id: string;
+    start_time: string;
+    end_time: string;
+    events: CompactionEvent[];
+}
+/**
+ * Load options
+ */
+export interface LoadOptions {
+    /** Filter by compaction_type */
+    compaction_type?: "detection_complete" | "prompt_generated" | "context_injected" | "resumption_started" | "tool_call_tracked";
+    /** Filter by session IDs */
+    sessionIds?: string[];
+    /** Limit number of results */
+    limit?: number;
+}
+/**
+ * Load COMPACTION events from session JSONL files
+ *
+ * Reads all .jsonl files in the session directory, parses events,
+ * and returns only COMPACTION events matching the filters.
+ *
+ * @param sessionDir - Path to session directory (default: ~/.config/swarm-tools/sessions)
+ * @param options - Filter options
+ * @returns Array of compaction events
+ *
+ * @example
+ * // Load all COMPACTION events
+ * const events = await loadCompactionEvents("/path/to/sessions");
+ *
+ * @example
+ * // Load only detection_complete events
+ * const events = await loadCompactionEvents("/path/to/sessions", {
+ *   compaction_type: "detection_complete",
+ * });
+ *
+ * @example
+ * // Load events from specific sessions
+ * const events = await loadCompactionEvents("/path/to/sessions", {
+ *   sessionIds: ["session-1", "session-2"],
+ *   limit: 10,
+ * });
+ */
+export declare function loadCompactionEvents(sessionDir: string, options?: LoadOptions): Promise<CompactionEvent[]>;
+/**
+ * Load COMPACTION sessions grouped by session_id
+ *
+ * Groups COMPACTION events by session_id and returns session metadata.
+ *
+ * @param sessionDir - Path to session directory
+ * @param options - Filter options
+ * @returns Array of compaction sessions
+ *
+ * @example
+ * // Load all sessions with COMPACTION events
+ * const sessions = await loadCompactionSessions("/path/to/sessions");
+ *
+ * @example
+ * // Load sessions with specific compaction_type
+ * const sessions = await loadCompactionSessions("/path/to/sessions", {
+ *   compaction_type: "prompt_generated",
+ * });
+ */
+export declare function loadCompactionSessions(sessionDir: string, options?: LoadOptions): Promise<CompactionSession[]>;
+/**
+ * Load COMPACTION events from default session directory
+ *
+ * Convenience wrapper that uses the default ~/.config/swarm-tools/sessions directory.
+ *
+ * @param options - Filter options
+ * @returns Array of compaction events
+ *
+ * @example
+ * // Load recent compaction events
+ * const events = await loadDefaultCompactionEvents({ limit: 10 });
+ */
+export declare function loadDefaultCompactionEvents(options?: LoadOptions): Promise<CompactionEvent[]>;
+/**
+ * Load COMPACTION sessions from default session directory
+ *
+ * Convenience wrapper that uses the default ~/.config/swarm-tools/sessions directory.
+ *
+ * @param options - Filter options
+ * @returns Array of compaction sessions
+ *
+ * @example
+ * // Load all compaction sessions
+ * const sessions = await loadDefaultCompactionSessions();
+ */
+export declare function loadDefaultCompactionSessions(options?: LoadOptions): Promise<CompactionSession[]>;
+//# sourceMappingURL=compaction-loader.d.ts.map

package/dist/lib/compaction-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compaction-loader.d.ts","sourceRoot":"","sources":["../../src/lib/compaction-loader.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAG3E;;GAEG;AACH,MAAM,MAAM,eAAe,GAAG,OAAO,CACnC,gBAAgB,EAChB;IAAE,UAAU,EAAE,YAAY,CAAA;CAAE,CAC7B,CAAC;AAEF;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,MAAM,CAAC;IACnB,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,eAAe,EAAE,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,gCAAgC;IAChC,eAAe,CAAC,EACZ,oBAAoB,GACpB,kBAAkB,GAClB,kBAAkB,GAClB,oBAAoB,GACpB,mBAAmB,CAAC;IACxB,4BAA4B;IAC5B,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,8BAA8B;IAC9B,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAsB,oBAAoB,CACxC,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,eAAe,EAAE,CAAC,CAuE5B;AAyDD;;;;;;;;;;;;;;;;;;GAkBG;AACH,wBAAsB,sBAAsB,CAC1C,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,iBAAiB,EAAE,CAAC,CA8C9B;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,2BAA2B,CAC/C,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,eAAe,EAAE,CAAC,CAG5B;AAED;;;;;;;;;;;GAWG;AACH,wBAAsB,6BAA6B,CACjD,OAAO,CAAC,EAAE,WAAW,GACpB,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAG9B"}

package/dist/lib/data-loader.d.ts

@@ -0,0 +1,79 @@
+import { type EvalRecord } from "swarm-mail";
+export interface EvalCase {
+    input: {
+        task: string;
+        context?: string;
+    };
+    expected: {
+        minSubtasks: number;
+        maxSubtasks: number;
+        requiredFiles?: string[];
+        overallSuccess?: boolean;
+    };
+    actual?: EvalRecord;
+}
+/**
+ * Load eval cases from PGlite
+ *
+ * @param projectKey - Project key for filtering records
+ * @param options - Filter options
+ * @returns Array of eval cases ready for Evalite
+ */
+export declare function loadEvalCases(projectKey: string, options?: {
+    limit?: number;
+    strategy?: "file-based" | "feature-based" | "risk-based";
+    successOnly?: boolean;
+    projectPath?: string;
+}): Promise<EvalCase[]>;
+/**
+ * Check if we have enough real data to run evals
+ *
+ * @param projectKey - Project key to check
+ * @param minRecords - Minimum number of records required (default: 5)
+ * @param projectPath - Optional project path for database lookup
+ * @returns True if enough data exists
+ */
+export declare function hasRealEvalData(projectKey: string, minRecords?: number, projectPath?: string): Promise<boolean>;
+/**
+ * Get eval data stats for reporting
+ *
+ * @param projectKey - Project key to query
+ * @param projectPath - Optional project path for database lookup
+ * @returns Summary of available eval data
+ */
+export declare function getEvalDataSummary(projectKey: string, projectPath?: string): Promise<{
+    totalRecords: number;
+    successRate: number;
+    byStrategy: Record<string, number>;
+    hasEnoughData: boolean;
+}>;
+/**
+ * Load captured coordinator sessions from ~/.config/swarm-tools/sessions/
+ *
+ * Reads all JSONL session files and returns CoordinatorSession objects.
+ *
+ * Quality filters are applied to focus on high-signal coordinator sessions:
+ * - minEvents: Filter out incomplete/aborted sessions (default: 3)
+ * - requireWorkerSpawn: Ensure session delegated to workers (default: true)
+ * - requireReview: Ensure coordinator reviewed work (default: true)
+ *
+ * Filters are applied BEFORE the limit for accurate sampling.
+ *
+ * @param options - Filter options
+ * @returns Array of coordinator sessions that meet quality criteria
+ */
+export declare function loadCapturedSessions(options?: {
+    sessionIds?: string[];
+    limit?: number;
+    /** Minimum number of events required (default: 3) */
+    minEvents?: number;
+    /** Require at least one worker_spawned event (default: true) */
+    requireWorkerSpawn?: boolean;
+    /** Require at least one review_completed event (default: true) */
+    requireReview?: boolean;
+    /** Override session directory for testing */
+    sessionDir?: string;
+}): Promise<Array<{
+    session: import("opencode-swarm-plugin/eval-capture").CoordinatorSession;
+}>>;
+//# sourceMappingURL=data-loader.d.ts.map

package/dist/lib/data-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data-loader.d.ts","sourceRoot":"","sources":["../../src/lib/data-loader.ts"],"names":[],"mappings":"AAOA,OAAO,EAGL,KAAK,UAAU,EAChB,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,QAAQ;IACvB,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1C,QAAQ,EAAE;QACR,WAAW,EAAE,MAAM,CAAC;QACpB,WAAW,EAAE,MAAM,CAAC;QACpB,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;QACzB,cAAc,CAAC,EAAE,OAAO,CAAC;KAC1B,CAAC;IACF,MAAM,CAAC,EAAE,UAAU,CAAC;CACrB;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CACjC,UAAU,EAAE,MAAM,EAClB,OAAO,CAAC,EAAE;IACR,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,YAAY,GAAG,eAAe,GAAG,YAAY,CAAC;IACzD,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GACA,OAAO,CAAC,QAAQ,EAAE,CAAC,CA6BrB;AAED;;;;;;;GAOG;AACH,wBAAsB,eAAe,CACnC,UAAU,EAAE,MAAM,EAClB,UAAU,GAAE,MAAU,EACtB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC,OAAO,CAAC,CAGlB;AAED;;;;;;GAMG;AACH,wBAAsB,kBAAkB,CACtC,UAAU,EAAE,MAAM,EAClB,WAAW,CAAC,EAAE,MAAM,GACnB,OAAO,CAAC;IACT,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,aAAa,EAAE,OAAO,CAAC;CACxB,CAAC,CASD;AA0CD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,oBAAoB,CAAC,OAAO,CAAC,EAAE;IACnD,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;IACtB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,qDAAqD;IACrD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gEAAgE;IAChE,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kEAAkE;IAClE,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,6CAA6C;IAC7C,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,GAAG,OAAO,CACT,KAAK,CAAC;IAAE,OAAO,EAAE,OAAO,oCAAoC,EAAE,kBAAkB,CAAA;CAAE,CAAC,CACpF,CAmGA"}

package/dist/lib/llm.d.ts

@@ -0,0 +1,27 @@
+import type { GatewayModelId } from "ai";
+/**
+ * Default model for decomposition evals
+ * Using Claude Sonnet for good balance of quality and cost
+ */
+export declare const DEFAULT_MODEL: GatewayModelId;
+/**
+ * Generate a decomposition from a task description
+ *
+ * @param prompt - The full decomposition prompt
+ * @param model - Gateway model ID (e.g., "anthropic/claude-sonnet-4-5")
+ * @returns The raw text response from the LLM
+ */
+export declare function generateDecomposition(prompt: string, model?: GatewayModelId): Promise<string>;
+/**
+ * Format a decomposition prompt from task and context
+ *
+ * Uses the same prompt template as swarm_plan_prompt
+ */
+export declare function formatDecompositionPrompt(task: string, context?: string, maxSubtasks?: number): string;
+/**
+ * Extract JSON from LLM response
+ *
+ * Handles responses that may have markdown code blocks or extra text
+ */
+export declare function extractJson(text: string): string;
+//# sourceMappingURL=llm.d.ts.map

package/dist/lib/llm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/lib/llm.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,IAAI,CAAC;AAEzC;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,cAA8C,CAAC;AAE3E;;;;;;GAMG;AACH,wBAAsB,qBAAqB,CACzC,MAAM,EAAE,MAAM,EACd,KAAK,GAAE,cAA8B,GACpC,OAAO,CAAC,MAAM,CAAC,CAQjB;AAED;;;;GAIG;AACH,wBAAgB,yBAAyB,CACvC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,EAChB,WAAW,GAAE,MAAU,GACtB,MAAM,CA8CR;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAehD"}

package/dist/scorers/compaction-prompt-scorers.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Compaction Prompt Quality Scorers - Evalite Wrappers
+ *
+ * These wrap the pure scoring functions from src/compaction-prompt-scoring.ts
+ * for use with evalite's test runner.
+ *
+ * Weighted scoring:
+ * - epicIdSpecificity (0.20) - real IDs not placeholders
+ * - actionability (0.20) - swarm_status/inbox with real values
+ * - coordinatorIdentity (0.25) - ASCII header + strong mandates
+ * - forbiddenToolsPresent (0.15) - lists forbidden tools by name
+ * - postCompactionDiscipline (0.20) - first tool correct, no edit/write
+ */
+export type { CompactionPrompt, ScorerResult } from "opencode-swarm-plugin/compaction-prompt-scoring";
+export { scoreActionability, scoreCoordinatorIdentity, scoreEpicIdSpecificity, scoreForbiddenToolsPresent, scorePostCompactionDiscipline, } from "opencode-swarm-plugin/compaction-prompt-scoring";
+/**
+ * Epic ID Specificity Scorer
+ *
+ * Validates that epic IDs are REAL, not placeholders.
+ * Score: 1.0 if real IDs, 0.0 if placeholders found
+ */
+export declare const epicIdSpecificity: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Actionability Scorer
+ *
+ * Validates that the prompt includes SPECIFIC actionable tool calls.
+ * Score: 1.0 if actionable tool calls with real values, 0.0 otherwise
+ */
+export declare const actionability: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Coordinator Identity Scorer
+ *
+ * Validates that the prompt has STRONG coordinator identity reinforcement.
+ * Score: 1.0 for ASCII header + strong mandates, 0.5 for header only, 0.0 otherwise
+ */
+export declare const coordinatorIdentity: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Forbidden Tools Present Scorer
+ *
+ * Validates that the prompt LISTS forbidden tools by name.
+ * Score: ratio of forbidden tools mentioned (0.0 to 1.0)
+ */
+export declare const forbiddenToolsPresent: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Post-Compaction Discipline Scorer
+ *
+ * Validates that the FIRST suggested tool is correct.
+ * Score: 1.0 if first tool is swarm_status or inbox, 0.0 otherwise
+ */
+export declare const postCompactionDiscipline: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+//# sourceMappingURL=compaction-prompt-scorers.d.ts.map

package/dist/scorers/compaction-prompt-scorers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compaction-prompt-scorers.d.ts","sourceRoot":"","sources":["../../src/scorers/compaction-prompt-scorers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAaH,YAAY,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,iDAAiD,CAAC;AAGtG,OAAO,EACN,kBAAkB,EAClB,wBAAwB,EACxB,sBAAsB,EACtB,0BAA0B,EAC1B,6BAA6B,GAC7B,MAAM,iDAAiD,CAAC;AAEzD;;;;;GAKG;AACH,eAAO,MAAM,iBAAiB,6DAc5B,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,aAAa,6DAcxB,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB,6DAc9B,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,qBAAqB,6DAchC,CAAC;AAEH;;;;;GAKG;AACH,eAAO,MAAM,wBAAwB,6DAcnC,CAAC"}

package/dist/scorers/compaction-scorers.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Custom scorers for compaction hook evaluation
+ *
+ * These scorers validate that the compaction hook correctly:
+ * 1. Detects swarm state (confidence level)
+ * 2. Injects appropriate context (full/fallback/none)
+ * 3. Includes required patterns in context
+ * 4. Excludes placeholder/generic content
+ */
+/**
+ * Expected output from compaction hook tests
+ */
+export interface CompactionResult {
+    detected: boolean;
+    confidence: "high" | "medium" | "low" | "none";
+    contextInjected: boolean;
+    contextType: "full" | "fallback" | "none";
+    injectedContext: string;
+}
+/**
+ * Expected criteria from test case
+ */
+export interface CompactionExpected {
+    confidence: "high" | "medium" | "low" | "none";
+    contextInjected: boolean;
+    contextType: "full" | "fallback" | "none";
+    mustContain?: string[];
+    mustNotContain?: string[];
+}
+/**
+ * Validates that detection confidence matches expected level
+ *
+ * Confidence determines what gets injected:
+ * - HIGH/MEDIUM: Full coordinator context
+ * - LOW: Fallback detection prompt
+ * - NONE: No injection
+ *
+ * Score: 1.0 if confidence matches, 0.0 otherwise
+ */
+export declare const confidenceAccuracy: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Validates that context injection matches expected behavior
+ *
+ * Checks:
+ * - Whether context was injected (boolean)
+ * - What type of context (full/fallback/none)
+ *
+ * Score: 1.0 if both match, 0.5 if only injection status matches, 0.0 otherwise
+ */
+export declare const contextInjectionCorrectness: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Validates that injected context contains required patterns
+ *
+ * For coordinator resumption, context MUST include:
+ * - Swarm continuation instructions
+ * - Tool names (swarm_status, swarmmail_inbox)
+ * - Actionable language ("COORDINATOR", "Keep Cooking")
+ *
+ * Score: ratio of required patterns found (0.0 to 1.0)
+ */
+export declare const requiredPatternsPresent: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Validates that injected context excludes forbidden patterns
+ *
+ * Context should NOT contain:
+ * - Placeholder IDs ("bd-xxx")
+ * - Generic/template language
+ * - Wrong context type markers
+ *
+ * Score: 1.0 if no forbidden patterns found, 0.0 if any found
+ */
+export declare const forbiddenPatternsAbsent: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Composite scorer: Overall compaction quality
+ *
+ * Combines all compaction-specific checks into single score.
+ * Weighted average:
+ * - Confidence accuracy: 25%
+ * - Context injection: 25%
+ * - Required patterns: 30%
+ * - Forbidden patterns: 20%
+ *
+ * Score: 0.0 to 1.0
+ */
+export declare const compactionQuality: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+//# sourceMappingURL=compaction-scorers.d.ts.map

package/dist/scorers/compaction-scorers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compaction-scorers.d.ts","sourceRoot":"","sources":["../../src/scorers/compaction-scorers.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,OAAO,CAAC;IAClB,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;IAC/C,eAAe,EAAE,OAAO,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAAC;IAC1C,eAAe,EAAE,MAAM,CAAC;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,UAAU,EAAE,MAAM,GAAG,QAAQ,GAAG,KAAK,GAAG,MAAM,CAAC;IAC/C,eAAe,EAAE,OAAO,CAAC;IACzB,WAAW,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAAC;IAC1C,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED;;;;;;;;;GASG;AACH,eAAO,MAAM,kBAAkB,6DA0B7B,CAAC;AAEH;;;;;;;;GAQG;AACH,eAAO,MAAM,2BAA2B,6DAoCtC,CAAC;AAEH;;;;;;;;;GASG;AACH,eAAO,MAAM,uBAAuB,6DA0DlC,CAAC;AAEH;;;;;;;;;GASG;AACH,eAAO,MAAM,uBAAuB,6DA8ClC,CAAC;AAEH;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,iBAAiB,6DA6C5B,CAAC"}

package/dist/scorers/coordinator-discipline.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Coordinator Discipline Scorers - Evaluate coordinator behavior
+ *
+ * These scorers measure whether a coordinator follows the protocol:
+ * 1. Don't edit files directly (spawn workers)
+ * 2. Don't run tests directly (workers do verification)
+ * 3. Spawn workers for all subtasks
+ * 4. Review worker output before accepting
+ * 5. Minimize time to first spawn (don't overthink)
+ *
+ * Inputs: CoordinatorSession from eval-capture
+ */
+/**
+ * Violation Count Scorer
+ *
+ * Counts VIOLATION events in the session.
+ * Each violation reduces score by 0.2.
+ *
+ * Violations tracked:
+ * - coordinator_edited_file (should spawn worker instead)
+ * - coordinator_ran_tests (workers do verification)
+ * - coordinator_reserved_files (only workers reserve)
+ * - no_worker_spawned (subtask exists but no worker)
+ *
+ * Score: 1.0 - (0.2 * violation_count), floored at 0.0
+ */
+export declare const violationCount: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Spawn Efficiency Scorer
+ *
+ * Measures whether workers were spawned for all subtasks.
+ * Coordinators should delegate work, not do it themselves.
+ *
+ * Score: workers_spawned / subtasks_planned
+ *
+ * If no decomposition_complete event exists, falls back to counting spawns
+ * and returns 1.0 if any workers were spawned (better than nothing).
+ */
+export declare const spawnEfficiency: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Review Thoroughness Scorer
+ *
+ * Measures whether coordinator reviewed worker output.
+ * Should have review_completed events for all finished subtasks.
+ *
+ * Score: reviews_completed / workers_finished
+ */
+export declare const reviewThoroughness: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Time to First Spawn Scorer
+ *
+ * Measures how fast the coordinator spawned the first worker.
+ * Overthinking and perfectionism delays workers and blocks progress.
+ *
+ * Normalization:
+ * - < 60s: 1.0 (excellent)
+ * - 60-300s: linear decay to 0.5
+ * - > 300s: 0.0 (way too slow)
+ *
+ * Score: normalized to 0-1 (faster is better)
+ */
+export declare const timeToFirstSpawn: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Overall Discipline Scorer
+ *
+ * Weighted composite of all coordinator discipline metrics.
+ *
+ * Weights:
+ * - Violations: 30% (most critical - breaking protocol)
+ * - Spawn efficiency: 25% (delegation is key)
+ * - Review thoroughness: 25% (quality gate)
+ * - Time to first spawn: 20% (bias toward action)
+ *
+ * Score: 0.0 to 1.0
+ */
+export declare const overallDiscipline: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+//# sourceMappingURL=coordinator-discipline.d.ts.map

package/dist/scorers/coordinator-discipline.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"coordinator-discipline.d.ts","sourceRoot":"","sources":["../../src/scorers/coordinator-discipline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAKH;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,cAAc,6DAiCzB,CAAC;AAEH;;;;;;;;;;GAUG;AACH,eAAO,MAAM,eAAe,6DAwD1B,CAAC;AAEH;;;;;;;GAOG;AACH,eAAO,MAAM,kBAAkB,6DAyC7B,CAAC;AAEH;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,gBAAgB,6DAkE3B,CAAC;AAEH;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,iBAAiB,6DA6C5B,CAAC"}

package/dist/scorers/index.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Custom scorers for evaluating swarm task decomposition quality
+ */
+/**
+ * Checks that no files appear in multiple subtasks
+ *
+ * Independent subtasks are critical for parallel execution.
+ * File conflicts cause merge conflicts and coordination overhead.
+ *
+ * Score: 1.0 if no conflicts, 0.0 if conflicts found
+ */
+export declare const subtaskIndependence: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+export { executionSuccess, timeBalance, scopeAccuracy, scopeDrift, noRework, } from "./outcome-scorers.js";
+export { confidenceAccuracy, contextInjectionCorrectness, requiredPatternsPresent, forbiddenPatternsAbsent, compactionQuality, } from "./compaction-scorers.js";
+export { violationCount, spawnEfficiency, reviewThoroughness, timeToFirstSpawn, overallDiscipline, } from "./coordinator-discipline.js";
+/**
+ * Checks that subtasks cover the full task scope
+ *
+ * Incomplete coverage means:
+ * - Missing functionality
+ * - Follow-up work required
+ * - Task not actually complete
+ *
+ * Score: ratio of expected files covered (0.0 to 1.0)
+ * If no expected files specified, checks that subtasks exist
+ */
+export declare const coverageCompleteness: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Checks that each subtask has clear, actionable instructions
+ *
+ * Vague instructions lead to:
+ * - Agent confusion and blocking
+ * - Incorrect implementations
+ * - Need for coordinator intervention
+ *
+ * Score: Average of per-subtask instruction quality
+ */
+export declare const instructionClarity: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * LLM-as-judge scorer for decomposition coherence
+ *
+ * Uses Claude Haiku to evaluate whether subtasks are truly independent,
+ * well-scoped, and complete. This catches nuances that heuristics miss:
+ * - Semantic dependencies between subtasks
+ * - Scope that's too big or too trivial
+ * - Missing pieces that would block completion
+ *
+ * Only use for decomposition evals - this is where it matters.
+ */
+export declare const decompositionCoherence: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+//# sourceMappingURL=index.d.ts.map

package/dist/scorers/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/scorers/index.ts"],"names":[],"mappings":"AAOA;;GAEG;AAEH;;;;;;;GAOG;AACH,eAAO,MAAM,mBAAmB,6DAuC9B,CAAC;AAMH,OAAO,EACL,gBAAgB,EAChB,WAAW,EACX,aAAa,EACb,UAAU,EACV,QAAQ,GACT,MAAM,sBAAsB,CAAC;AAM9B,OAAO,EACL,kBAAkB,EAClB,2BAA2B,EAC3B,uBAAuB,EACvB,uBAAuB,EACvB,iBAAiB,GAClB,MAAM,yBAAyB,CAAC;AAMjC,OAAO,EACL,cAAc,EACd,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,6BAA6B,CAAC;AAErC;;;;;;;;;;GAUG;AACH,eAAO,MAAM,oBAAoB,6DAsD/B,CAAC;AAEH;;;;;;;;;GASG;AACH,eAAO,MAAM,kBAAkB,6DAsD7B,CAAC;AAMH;;;;;;;;;;GAUG;AACH,eAAO,MAAM,sBAAsB,6DAmFjC,CAAC"}

package/dist/scorers/outcome-scorers.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Outcome-based scorers for evaluating decomposition quality
+ *
+ * These scorers evaluate based on ACTUAL execution outcomes,
+ * not just the structure of the decomposition.
+ *
+ * Requires EvalRecord with outcomes populated.
+ */
+/**
+ * Execution Success Scorer
+ *
+ * Measures whether all subtasks succeeded without errors.
+ * This is the ultimate measure - did the decomposition actually work?
+ *
+ * Score: 1.0 if all outcomes.success === true, 0.0 otherwise
+ */
+export declare const executionSuccess: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Time Balance Scorer
+ *
+ * Measures how evenly balanced the work was across subtasks.
+ * Unbalanced work means some agents finish early while others are bottlenecked.
+ *
+ * Score: 1.0 if max/min ratio < 2.0 (well balanced)
+ *        0.5 if ratio < 4.0 (moderately balanced)
+ *        0.0 if ratio >= 4.0 (poorly balanced)
+ */
+export declare const timeBalance: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Scope Accuracy Scorer
+ *
+ * Measures how accurately the decomposition predicted which files would be touched.
+ * High accuracy means the planner understood the work scope correctly.
+ *
+ * Score: intersection(actual, planned) / planned.length
+ *        1.0 = all planned files were touched, no extras
+ *        0.5 = half the planned files were touched
+ *        0.0 = none of the planned files were touched
+ */
+export declare const scopeAccuracy: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * Scope Drift Scorer
+ *
+ * Penalizes when agents touch files NOT in their planned scope.
+ * Scope drift indicates poor planning or unexpected dependencies.
+ *
+ * Score: 1.0 if no drift (all actual files were planned)
+ *        Decreases linearly with drift percentage
+ *        0.0 if drift > 50%
+ */
+export declare const scopeDrift: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+/**
+ * No Rework Scorer
+ *
+ * Checks that no subtask touched files assigned to another subtask.
+ * Rework indicates poor decomposition or missing dependencies.
+ *
+ * Score: 1.0 if no rework (no subtask touched another's planned files)
+ *        0.0 if rework detected
+ */
+export declare const noRework: import("evalite").Evalite.Scorer<unknown, unknown, unknown>;
+//# sourceMappingURL=outcome-scorers.d.ts.map

package/dist/scorers/outcome-scorers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"outcome-scorers.d.ts","sourceRoot":"","sources":["../../src/scorers/outcome-scorers.ts"],"names":[],"mappings":"AAGA;;;;;;;GAOG;AAEH;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,6DAwC3B,CAAC;AAEH;;;;;;;;;GASG;AACH,eAAO,MAAM,WAAW,6DAoEtB,CAAC;AAEH;;;;;;;;;;GAUG;AACH,eAAO,MAAM,aAAa,6DAmDxB,CAAC;AAEH;;;;;;;;;GASG;AACH,eAAO,MAAM,UAAU,6DAwDrB,CAAC;AAEH;;;;;;;;GAQG;AACH,eAAO,MAAM,QAAQ,6DAiEnB,CAAC"}

package/dist/swarm-decomposition.eval.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=swarm-decomposition.eval.d.ts.map

package/dist/swarm-decomposition.eval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"swarm-decomposition.eval.d.ts","sourceRoot":"","sources":["../src/swarm-decomposition.eval.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "@swarmtools/evals",
+  "version": "0.2.0",
+  "description": "Evaluation suite for swarm-tools multi-agent coordination",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/joelhooks/opencode-swarm-plugin"
+  },
+  "author": "Joel Hooks",
+  "license": "MIT",
+  "scripts": {
+    "build": "bun build ./src/index.ts --outdir ./dist --target node && tsc",
+    "test": "bun test src/**/*.test.ts",
+    "typecheck": "tsc --noEmit"
+  },
+  "dependencies": {
+    "evalite": "^1.0.0-beta.10",
+    "ai": "6.0.0-beta.150",
+    "opencode-swarm-plugin": "0.44.1",
+    "swarm-mail": "1.6.0"
+  },
+  "devDependencies": {
+    "@types/node": "^22.19.3",
+    "bun-types": "^1.3.4",
+    "typescript": "^5.7.2",
+    "vitest": "^4.0.15"
+  }
+}