opencode-swarm-plugin 0.32.0 → 0.34.0

package/evals/fixtures/compaction-cases.ts

@@ -0,0 +1,350 @@
+/**
+ * Test cases for compaction hook coordinator resumption
+ *
+ * Each case simulates a different swarm state and verifies that
+ * the compaction hook injects the correct context for resumption.
+ */
+
+import type { Cell } from "swarm-mail";
+
+/**
+ * Compaction test case structure
+ */
+export interface CompactionTestCase {
+  name: string;
+  description: string;
+  /**
+   * Simulated hive state (cells to create)
+   */
+  hiveCells: Array<Omit<Cell, "created_at" | "updated_at" | "closed_at">>;
+  /**
+   * Simulated swarm-mail state
+   */
+  swarmMailState: {
+    agents: number;
+    reservations: number;
+    messages: number;
+  };
+  /**
+   * Expected detection confidence
+   */
+  expected: {
+    confidence: "high" | "medium" | "low" | "none";
+    contextInjected: boolean;
+    contextType: "full" | "fallback" | "none";
+    /**
+     * Patterns that MUST appear in injected context (if injected)
+     */
+    mustContain?: string[];
+    /**
+     * Patterns that MUST NOT appear
+     */
+    mustNotContain?: string[];
+  };
+}
+
+export const compactionCases: CompactionTestCase[] = [
+  // ============================================================================
+  // HIGH CONFIDENCE: Active swarm with in_progress epic
+  // ============================================================================
+  {
+    name: "Active swarm with in_progress epic",
+    description:
+      "Compaction happens mid-swarm with an active epic and subtasks. Should inject full context with specific epic ID.",
+    hiveCells: [
+      {
+        id: "test-project-lf2p4u-epic123",
+        project_key: "/test/project",
+        type: "epic",
+        status: "in_progress",
+        title: "Add user authentication",
+        description: "Implement OAuth with NextAuth.js",
+        priority: 2,
+        parent_id: null,
+        assignee: "coordinator",
+        closed_reason: null,
+        deleted_at: null,
+        deleted_by: null,
+        delete_reason: null,
+        created_by: "coordinator",
+      },
+      {
+        id: "test-project-lf2p4u-epic123.1",
+        project_key: "/test/project",
+        type: "task",
+        status: "closed",
+        title: "OAuth provider config",
+        description: "Configure GitHub OAuth provider",
+        priority: 2,
+        parent_id: "test-project-lf2p4u-epic123",
+        assignee: "BlueLake",
+        closed_reason: "Done: configured GitHub provider",
+        deleted_at: null,
+        deleted_by: null,
+        delete_reason: null,
+        created_by: "coordinator",
+      },
+      {
+        id: "test-project-lf2p4u-epic123.2",
+        project_key: "/test/project",
+        type: "task",
+        status: "in_progress",
+        title: "Auth middleware",
+        description: "Create middleware for protecting routes",
+        priority: 2,
+        parent_id: "test-project-lf2p4u-epic123",
+        assignee: "RedMountain",
+        closed_reason: null,
+        deleted_at: null,
+        deleted_by: null,
+        delete_reason: null,
+        created_by: "coordinator",
+      },
+      {
+        id: "test-project-lf2p4u-epic123.3",
+        project_key: "/test/project",
+        type: "task",
+        status: "open",
+        title: "Session management",
+        description: "Implement session persistence with Redis",
+        priority: 2,
+        parent_id: "test-project-lf2p4u-epic123",
+        assignee: null,
+        closed_reason: null,
+        deleted_at: null,
+        deleted_by: null,
+        delete_reason: null,
+        created_by: "coordinator",
+      },
+    ],
+    swarmMailState: {
+      agents: 2,
+      reservations: 1,
+      messages: 5,
+    },
+    expected: {
+      confidence: "high",
+      contextInjected: true,
+      contextType: "full",
+      mustContain: [
+        "SWARM ACTIVE",
+        "COORDINATOR",
+        "swarm_status",
+        "swarmmail_inbox",
+        "Keep Cooking",
+      ],
+      mustNotContain: [
+        "bd-xxx", // Should NOT contain placeholder IDs
+        "Check Your Context", // Should NOT be fallback detection
+      ],
+    },
+  },
+
+  // ============================================================================
+  // MEDIUM CONFIDENCE: Multiple epics, need to identify active one
+  // ============================================================================
+  {
+    name: "Multiple epics with one in_progress",
+    description:
+      "Multiple epics exist, but only one is in_progress. Should detect and inject context for the active one.",
+    hiveCells: [
+      {
+        id: "test-project-lf2p4u-epic100",
+        project_key: "/test/project",
+        type: "epic",
+        status: "closed",
+        title: "Refactor auth system",
+        description: "Old completed epic",
+        priority: 2,
+        parent_id: null,
+        assignee: null,
+        closed_reason: "Done",
+        deleted_at: null,
+        deleted_by: null,
+        delete_reason: null,
+        created_by: "coordinator",
+      },
+      {
+        id: "test-project-lf2p4u-epic200",
+        project_key: "/test/project",
+        type: "epic",
+        status: "in_progress",
+        title: "Add rate limiting",
+        description: "Implement Redis-based rate limiting",
+        priority: 2,
+        parent_id: null,
+        assignee: "coordinator",
+        closed_reason: null,
+        deleted_at: null,
+        deleted_by: null,
+        delete_reason: null,
+        created_by: "coordinator",
+      },
+      {
+        id: "test-project-lf2p4u-epic200.1",
+        project_key: "/test/project",
+        type: "task",
+        status: "open",
+        title: "Rate limit middleware",
+        description: "Create Express middleware",
+        priority: 2,
+        parent_id: "test-project-lf2p4u-epic200",
+        assignee: null,
+        closed_reason: null,
+        deleted_at: null,
+        deleted_by: null,
+        delete_reason: null,
+        created_by: "coordinator",
+      },
+      {
+        id: "test-project-lf2p4u-epic300",
+        project_key: "/test/project",
+        type: "epic",
+        status: "open",
+        title: "Future epic",
+        description: "Not started yet",
+        priority: 1,
+        parent_id: null,
+        assignee: null,
+        closed_reason: null,
+        deleted_at: null,
+        deleted_by: null,
+        delete_reason: null,
+        created_by: "coordinator",
+      },
+    ],
+    swarmMailState: {
+      agents: 1,
+      reservations: 0,
+      messages: 2,
+    },
+    expected: {
+      confidence: "medium",
+      contextInjected: true,
+      contextType: "full",
+      mustContain: ["SWARM ACTIVE", "COORDINATOR"],
+      mustNotContain: ["bd-xxx"],
+    },
+  },
+
+  // ============================================================================
+  // LOW CONFIDENCE: Cells exist but no active work
+  // ============================================================================
+  {
+    name: "Cells exist but no active swarm",
+    description:
+      "Hive has some cells but no in_progress work. Should inject fallback detection prompt.",
+    hiveCells: [
+      {
+        id: "test-project-lf2p4u-task001",
+        project_key: "/test/project",
+        type: "task",
+        status: "open",
+        title: "Fix typo in README",
+        description: null,
+        priority: 0,
+        parent_id: null,
+        assignee: null,
+        closed_reason: null,
+        deleted_at: null,
+        deleted_by: null,
+        delete_reason: null,
+        created_by: null,
+      },
+    ],
+    swarmMailState: {
+      agents: 0,
+      reservations: 0,
+      messages: 0,
+    },
+    expected: {
+      confidence: "low",
+      contextInjected: true,
+      contextType: "fallback",
+      mustContain: [
+        "Swarm Detection",
+        "Check Your Context",
+        "swarm_decompose",
+        "swarmmail_init",
+      ],
+      mustNotContain: ["SWARM ACTIVE", "COORDINATOR"],
+    },
+  },
+
+  // ============================================================================
+  // NONE: Empty hive, no swarm activity
+  // ============================================================================
+  {
+    name: "Empty hive - no swarm activity",
+    description:
+      "No cells, no swarm-mail activity. Should NOT inject any context.",
+    hiveCells: [],
+    swarmMailState: {
+      agents: 0,
+      reservations: 0,
+      messages: 0,
+    },
+    expected: {
+      confidence: "none",
+      contextInjected: false,
+      contextType: "none",
+      mustContain: [],
+      mustNotContain: ["SWARM", "COORDINATOR", "swarm_status"],
+    },
+  },
+
+  // ============================================================================
+  // EDGE CASE: Blocked epic (should still detect as active swarm)
+  // ============================================================================
+  {
+    name: "Blocked epic with subtasks",
+    description:
+      "Epic is blocked but has in_progress subtasks. Should detect as active swarm.",
+    hiveCells: [
+      {
+        id: "test-project-lf2p4u-epic400",
+        project_key: "/test/project",
+        type: "epic",
+        status: "blocked",
+        title: "Migration to TypeScript",
+        description: "Full codebase migration",
+        priority: 3,
+        parent_id: null,
+        assignee: "coordinator",
+        closed_reason: null,
+        deleted_at: null,
+        deleted_by: null,
+        delete_reason: null,
+        created_by: "coordinator",
+      },
+      {
+        id: "test-project-lf2p4u-epic400.1",
+        project_key: "/test/project",
+        type: "task",
+        status: "in_progress",
+        title: "Migrate utils",
+        description: "Convert utils to TypeScript",
+        priority: 2,
+        parent_id: "test-project-lf2p4u-epic400",
+        assignee: "GreenValley",
+        closed_reason: null,
+        deleted_at: null,
+        deleted_by: null,
+        delete_reason: null,
+        created_by: "coordinator",
+      },
+    ],
+    swarmMailState: {
+      agents: 1,
+      reservations: 1,
+      messages: 3,
+    },
+    expected: {
+      confidence: "high",
+      contextInjected: true,
+      contextType: "full",
+      mustContain: ["SWARM ACTIVE", "COORDINATOR"],
+      mustNotContain: ["bd-xxx"],
+    },
+  },
+];

package/evals/scorers/compaction-scorers.ts

@@ -0,0 +1,305 @@
+/**
+ * Custom scorers for compaction hook evaluation
+ *
+ * These scorers validate that the compaction hook correctly:
+ * 1. Detects swarm state (confidence level)
+ * 2. Injects appropriate context (full/fallback/none)
+ * 3. Includes required patterns in context
+ * 4. Excludes placeholder/generic content
+ */
+
+import { createScorer } from "evalite";
+
+/**
+ * Expected output from compaction hook tests
+ */
+export interface CompactionResult {
+  detected: boolean;
+  confidence: "high" | "medium" | "low" | "none";
+  contextInjected: boolean;
+  contextType: "full" | "fallback" | "none";
+  injectedContext: string;
+}
+
+/**
+ * Expected criteria from test case
+ */
+export interface CompactionExpected {
+  confidence: "high" | "medium" | "low" | "none";
+  contextInjected: boolean;
+  contextType: "full" | "fallback" | "none";
+  mustContain?: string[];
+  mustNotContain?: string[];
+}
+
+/**
+ * Validates that detection confidence matches expected level
+ *
+ * Confidence determines what gets injected:
+ * - HIGH/MEDIUM: Full coordinator context
+ * - LOW: Fallback detection prompt
+ * - NONE: No injection
+ *
+ * Score: 1.0 if confidence matches, 0.0 otherwise
+ */
+export const confidenceAccuracy = createScorer({
+  name: "Confidence Accuracy",
+  description: "Validates detection confidence matches expected level",
+  scorer: ({ output, expected }) => {
+    try {
+      const result = JSON.parse(String(output)) as CompactionResult;
+      const exp = expected as CompactionExpected;
+
+      if (result.confidence === exp.confidence) {
+        return {
+          score: 1,
+          message: `Correct confidence: ${result.confidence}`,
+        };
+      }
+
+      return {
+        score: 0,
+        message: `Wrong confidence: got ${result.confidence}, expected ${exp.confidence}`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse result: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * Validates that context injection matches expected behavior
+ *
+ * Checks:
+ * - Whether context was injected (boolean)
+ * - What type of context (full/fallback/none)
+ *
+ * Score: 1.0 if both match, 0.5 if only injection status matches, 0.0 otherwise
+ */
+export const contextInjectionCorrectness = createScorer({
+  name: "Context Injection Correctness",
+  description: "Validates context injection matches expected behavior",
+  scorer: ({ output, expected }) => {
+    try {
+      const result = JSON.parse(String(output)) as CompactionResult;
+      const exp = expected as CompactionExpected;
+
+      const injectionMatches = result.contextInjected === exp.contextInjected;
+      const typeMatches = result.contextType === exp.contextType;
+
+      if (injectionMatches && typeMatches) {
+        return {
+          score: 1,
+          message: `Correct injection: ${result.contextType}`,
+        };
+      }
+
+      if (injectionMatches) {
+        return {
+          score: 0.5,
+          message: `Injection status correct but wrong type: got ${result.contextType}, expected ${exp.contextType}`,
+        };
+      }
+
+      return {
+        score: 0,
+        message: `Wrong injection: got ${result.contextInjected ? result.contextType : "none"}, expected ${exp.contextInjected ? exp.contextType : "none"}`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse result: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * Validates that injected context contains required patterns
+ *
+ * For coordinator resumption, context MUST include:
+ * - Swarm continuation instructions
+ * - Tool names (swarm_status, swarmmail_inbox)
+ * - Actionable language ("COORDINATOR", "Keep Cooking")
+ *
+ * Score: ratio of required patterns found (0.0 to 1.0)
+ */
+export const requiredPatternsPresent = createScorer({
+  name: "Required Patterns Present",
+  description: "Validates injected context contains required patterns",
+  scorer: ({ output, expected }) => {
+    try {
+      const result = JSON.parse(String(output)) as CompactionResult;
+      const exp = expected as CompactionExpected;
+
+      // If no context injected, check that mustContain is empty
+      if (!result.contextInjected) {
+        if (!exp.mustContain || exp.mustContain.length === 0) {
+          return {
+            score: 1,
+            message: "No context injected (expected)",
+          };
+        }
+        return {
+          score: 0,
+          message: "No context injected but patterns were expected",
+        };
+      }
+
+      // Check required patterns
+      if (!exp.mustContain || exp.mustContain.length === 0) {
+        return {
+          score: 1,
+          message: "No required patterns to check",
+        };
+      }
+
+      const found = exp.mustContain.filter((pattern) =>
+        result.injectedContext.includes(pattern),
+      );
+
+      const score = found.length / exp.mustContain.length;
+
+      if (score === 1) {
+        return {
+          score: 1,
+          message: `All ${exp.mustContain.length} required patterns found`,
+        };
+      }
+
+      const missing = exp.mustContain.filter(
+        (pattern) => !result.injectedContext.includes(pattern),
+      );
+
+      return {
+        score,
+        message: `${found.length}/${exp.mustContain.length} patterns found. Missing: ${missing.join(", ")}`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse result: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * Validates that injected context excludes forbidden patterns
+ *
+ * Context should NOT contain:
+ * - Placeholder IDs ("bd-xxx")
+ * - Generic/template language
+ * - Wrong context type markers
+ *
+ * Score: 1.0 if no forbidden patterns found, 0.0 if any found
+ */
+export const forbiddenPatternsAbsent = createScorer({
+  name: "Forbidden Patterns Absent",
+  description: "Validates injected context excludes forbidden patterns",
+  scorer: ({ output, expected }) => {
+    try {
+      const result = JSON.parse(String(output)) as CompactionResult;
+      const exp = expected as CompactionExpected;
+
+      // If no context injected, all checks pass
+      if (!result.contextInjected) {
+        return {
+          score: 1,
+          message: "No context injected (no forbidden patterns possible)",
+        };
+      }
+
+      // Check forbidden patterns
+      if (!exp.mustNotContain || exp.mustNotContain.length === 0) {
+        return {
+          score: 1,
+          message: "No forbidden patterns to check",
+        };
+      }
+
+      const foundForbidden = exp.mustNotContain.filter((pattern) =>
+        result.injectedContext.includes(pattern),
+      );
+
+      if (foundForbidden.length === 0) {
+        return {
+          score: 1,
+          message: "No forbidden patterns found",
+        };
+      }
+
+      return {
+        score: 0,
+        message: `Forbidden patterns found: ${foundForbidden.join(", ")}`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to parse result: ${error}`,
+      };
+    }
+  },
+});
+
+/**
+ * Composite scorer: Overall compaction quality
+ *
+ * Combines all compaction-specific checks into single score.
+ * Weighted average:
+ * - Confidence accuracy: 25%
+ * - Context injection: 25%
+ * - Required patterns: 30%
+ * - Forbidden patterns: 20%
+ *
+ * Score: 0.0 to 1.0
+ */
+export const compactionQuality = createScorer({
+  name: "Overall Compaction Quality",
+  description: "Composite score for compaction hook correctness",
+  scorer: ({ output, expected }) => {
+    try {
+      // Run all scorers
+      const scores = {
+        confidence: confidenceAccuracy.scorer({ output, expected }),
+        injection: contextInjectionCorrectness.scorer({ output, expected }),
+        required: requiredPatternsPresent.scorer({ output, expected }),
+        forbidden: forbiddenPatternsAbsent.scorer({ output, expected }),
+      };
+
+      // Weighted average
+      const weights = {
+        confidence: 0.25,
+        injection: 0.25,
+        required: 0.3,
+        forbidden: 0.2,
+      };
+
+      const totalScore =
+        scores.confidence.score * weights.confidence +
+        scores.injection.score * weights.injection +
+        scores.required.score * weights.required +
+        scores.forbidden.score * weights.forbidden;
+
+      const details = [
+        `Confidence: ${(scores.confidence.score * 100).toFixed(0)}%`,
+        `Injection: ${(scores.injection.score * 100).toFixed(0)}%`,
+        `Required: ${(scores.required.score * 100).toFixed(0)}%`,
+        `Forbidden: ${(scores.forbidden.score * 100).toFixed(0)}%`,
+      ].join(", ");
+
+      return {
+        score: totalScore,
+        message: `Overall: ${(totalScore * 100).toFixed(0)}% (${details})`,
+      };
+    } catch (error) {
+      return {
+        score: 0,
+        message: `Failed to compute composite score: ${error}`,
+      };
+    }
+  },
+});

package/evals/scorers/index.ts

@@ -66,6 +66,18 @@ export {
   noRework,
 } from "./outcome-scorers.js";
 
+// ============================================================================
+// Compaction-specific scorers
+// ============================================================================
+
+export {
+  confidenceAccuracy,
+  contextInjectionCorrectness,
+  requiredPatternsPresent,
+  forbiddenPatternsAbsent,
+  compactionQuality,
+} from "./compaction-scorers.js";
+
 /**
  * Checks that subtasks cover the full task scope
  *