opencode-swarm-plugin 0.43.0 → 0.44.1

package/evals/lib/llm.ts

@@ -1,115 +0,0 @@
-/**
- * LLM Client for Evalite Evals
- *
- * Uses AI SDK v6 with Vercel AI Gateway.
- * Gateway handles provider routing - just pass "provider/model" string.
- *
- * @module evals/lib/llm
- */
-import { generateText, gateway } from "ai";
-import type { GatewayModelId } from "ai";
-
-/**
- * Default model for decomposition evals
- * Using Claude Sonnet for good balance of quality and cost
- */
-export const DEFAULT_MODEL: GatewayModelId = "anthropic/claude-sonnet-4-5";
-
-/**
- * Generate a decomposition from a task description
- *
- * @param prompt - The full decomposition prompt
- * @param model - Gateway model ID (e.g., "anthropic/claude-sonnet-4-5")
- * @returns The raw text response from the LLM
- */
-export async function generateDecomposition(
-  prompt: string,
-  model: GatewayModelId = DEFAULT_MODEL,
-): Promise<string> {
-  const { text } = await generateText({
-    model: gateway(model),
-    prompt,
-    maxOutputTokens: 4096,
-  });
-
-  return text;
-}
-
-/**
- * Format a decomposition prompt from task and context
- *
- * Uses the same prompt template as swarm_plan_prompt
- */
-export function formatDecompositionPrompt(
-  task: string,
-  context?: string,
-  maxSubtasks: number = 6,
-): string {
-  const contextSection = context ? `## Context\n${context}` : "";
-
-  return `You are decomposing a task into parallelizable subtasks for a swarm of agents.
-
-## Task
-${task}
-
-${contextSection}
-
-## Requirements
-
-1. **Break into 2-${maxSubtasks} independent subtasks** that can run in parallel
-2. **Assign files** - each subtask must specify which files it will modify
-3. **No file overlap** - files cannot appear in multiple subtasks (they get exclusive locks)
-4. **Order by dependency** - if subtask B needs subtask A's output, A must come first in the array
-5. **Estimate complexity** - 1 (trivial) to 5 (complex)
-
-## Response Format
-
-Respond with ONLY a JSON object matching this schema (no markdown, no explanation):
-
-{
-  "epic": {
-    "title": "string",
-    "description": "string"
-  },
-  "subtasks": [
-    {
-      "title": "string",
-      "description": "string",
-      "files": ["string"],
-      "dependencies": [0],
-      "estimated_complexity": 1
-    }
-  ]
-}
-
-## Guidelines
-
-- **Plan aggressively** - when in doubt, split further
-- **Prefer smaller, focused subtasks** over large complex ones
-- **Include test files** in the same subtask as the code they test
-- **Be specific about files** - use actual file paths, not placeholders
-
-Now decompose the task. Respond with JSON only:`;
-}
-
-/**
- * Extract JSON from LLM response
- *
- * Handles responses that may have markdown code blocks or extra text
- */
-export function extractJson(text: string): string {
-  // Try to find JSON in code blocks first
-  const codeBlockMatch = text.match(/```(?:json)?\s*([\s\S]*?)```/);
-  if (codeBlockMatch) {
-    return codeBlockMatch[1].trim();
-  }
-
-  // Try to find raw JSON object
-  const jsonMatch = text.match(/\{[\s\S]*\}/);
-  if (jsonMatch) {
-    return jsonMatch[0];
-  }
-
-  // Return as-is if no JSON found
-  return text;
-}

package/evals/scorers/compaction-prompt-scorers.ts

@@ -1,145 +0,0 @@
-/**
- * Compaction Prompt Quality Scorers - Evalite Wrappers
- *
- * These wrap the pure scoring functions from src/compaction-prompt-scoring.ts
- * for use with evalite's test runner.
- *
- * Weighted scoring:
- * - epicIdSpecificity (0.20) - real IDs not placeholders
- * - actionability (0.20) - swarm_status/inbox with real values
- * - coordinatorIdentity (0.25) - ASCII header + strong mandates
- * - forbiddenToolsPresent (0.15) - lists forbidden tools by name
- * - postCompactionDiscipline (0.20) - first tool correct, no edit/write
- */
-
-import { createScorer } from "evalite";
-import type { CompactionPrompt } from "../../src/compaction-prompt-scoring.js";
-import {
-	scoreActionability,
-	scoreCoordinatorIdentity,
-	scoreEpicIdSpecificity,
-	scoreForbiddenToolsPresent,
-	scorePostCompactionDiscipline,
-} from "../../src/compaction-prompt-scoring.js";
-
-// Re-export types for convenience
-export type { CompactionPrompt, ScorerResult } from "../../src/compaction-prompt-scoring.js";
-
-// Re-export pure functions for direct use
-export {
-	scoreActionability,
-	scoreCoordinatorIdentity,
-	scoreEpicIdSpecificity,
-	scoreForbiddenToolsPresent,
-	scorePostCompactionDiscipline,
-} from "../../src/compaction-prompt-scoring.js";
-
-/**
- * Epic ID Specificity Scorer
- *
- * Validates that epic IDs are REAL, not placeholders.
- * Score: 1.0 if real IDs, 0.0 if placeholders found
- */
-export const epicIdSpecificity = createScorer({
-	name: "Epic ID Specificity",
-	description: "Prompt uses real epic IDs, not placeholders",
-	scorer: ({ output }) => {
-		try {
-			const prompt = JSON.parse(String(output)) as CompactionPrompt;
-			return scoreEpicIdSpecificity(prompt);
-		} catch (error) {
-			return {
-				score: 0,
-				message: `Failed to parse prompt: ${error}`,
-			};
-		}
-	},
-});
-
-/**
- * Actionability Scorer
- *
- * Validates that the prompt includes SPECIFIC actionable tool calls.
- * Score: 1.0 if actionable tool calls with real values, 0.0 otherwise
- */
-export const actionability = createScorer({
-	name: "Actionability",
-	description: "Prompt includes specific tool calls with real values",
-	scorer: ({ output }) => {
-		try {
-			const prompt = JSON.parse(String(output)) as CompactionPrompt;
-			return scoreActionability(prompt);
-		} catch (error) {
-			return {
-				score: 0,
-				message: `Failed to parse prompt: ${error}`,
-			};
-		}
-	},
-});
-
-/**
- * Coordinator Identity Scorer
- *
- * Validates that the prompt has STRONG coordinator identity reinforcement.
- * Score: 1.0 for ASCII header + strong mandates, 0.5 for header only, 0.0 otherwise
- */
-export const coordinatorIdentity = createScorer({
-	name: "Coordinator Identity",
-	description: "Prompt has ASCII header and strong mandates",
-	scorer: ({ output }) => {
-		try {
-			const prompt = JSON.parse(String(output)) as CompactionPrompt;
-			return scoreCoordinatorIdentity(prompt);
-		} catch (error) {
-			return {
-				score: 0,
-				message: `Failed to parse prompt: ${error}`,
-			};
-		}
-	},
-});
-
-/**
- * Forbidden Tools Present Scorer
- *
- * Validates that the prompt LISTS forbidden tools by name.
- * Score: ratio of forbidden tools mentioned (0.0 to 1.0)
- */
-export const forbiddenToolsPresent = createScorer({
-	name: "Forbidden Tools Present",
-	description: "Prompt lists forbidden tools by name",
-	scorer: ({ output }) => {
-		try {
-			const prompt = JSON.parse(String(output)) as CompactionPrompt;
-			return scoreForbiddenToolsPresent(prompt);
-		} catch (error) {
-			return {
-				score: 0,
-				message: `Failed to parse prompt: ${error}`,
-			};
-		}
-	},
-});
-
-/**
- * Post-Compaction Discipline Scorer
- *
- * Validates that the FIRST suggested tool is correct.
- * Score: 1.0 if first tool is swarm_status or inbox, 0.0 otherwise
- */
-export const postCompactionDiscipline = createScorer({
-	name: "Post-Compaction Discipline",
-	description: "First suggested tool is swarm_status or inbox",
-	scorer: ({ output }) => {
-		try {
-			const prompt = JSON.parse(String(output)) as CompactionPrompt;
-			return scorePostCompactionDiscipline(prompt);
-		} catch (error) {
-			return {
-				score: 0,
-				message: `Failed to parse prompt: ${error}`,
-			};
-		}
-	},
-});

package/evals/scorers/compaction-scorers.ts

@@ -1,305 +0,0 @@
-/**
- * Custom scorers for compaction hook evaluation
- *
- * These scorers validate that the compaction hook correctly:
- * 1. Detects swarm state (confidence level)
- * 2. Injects appropriate context (full/fallback/none)
- * 3. Includes required patterns in context
- * 4. Excludes placeholder/generic content
- */
-
-import { createScorer } from "evalite";
-
-/**
- * Expected output from compaction hook tests
- */
-export interface CompactionResult {
-  detected: boolean;
-  confidence: "high" | "medium" | "low" | "none";
-  contextInjected: boolean;
-  contextType: "full" | "fallback" | "none";
-  injectedContext: string;
-}
-
-/**
- * Expected criteria from test case
- */
-export interface CompactionExpected {
-  confidence: "high" | "medium" | "low" | "none";
-  contextInjected: boolean;
-  contextType: "full" | "fallback" | "none";
-  mustContain?: string[];
-  mustNotContain?: string[];
-}
-
-/**
- * Validates that detection confidence matches expected level
- *
- * Confidence determines what gets injected:
- * - HIGH/MEDIUM: Full coordinator context
- * - LOW: Fallback detection prompt
- * - NONE: No injection
- *
- * Score: 1.0 if confidence matches, 0.0 otherwise
- */
-export const confidenceAccuracy = createScorer({
-  name: "Confidence Accuracy",
-  description: "Validates detection confidence matches expected level",
-  scorer: ({ output, expected }) => {
-    try {
-      const result = JSON.parse(String(output)) as CompactionResult;
-      const exp = expected as CompactionExpected;
-
-      if (result.confidence === exp.confidence) {
-        return {
-          score: 1,
-          message: `Correct confidence: ${result.confidence}`,
-        };
-      }
-
-      return {
-        score: 0,
-        message: `Wrong confidence: got ${result.confidence}, expected ${exp.confidence}`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse result: ${error}`,
-      };
-    }
-  },
-});
-
-/**
- * Validates that context injection matches expected behavior
- *
- * Checks:
- * - Whether context was injected (boolean)
- * - What type of context (full/fallback/none)
- *
- * Score: 1.0 if both match, 0.5 if only injection status matches, 0.0 otherwise
- */
-export const contextInjectionCorrectness = createScorer({
-  name: "Context Injection Correctness",
-  description: "Validates context injection matches expected behavior",
-  scorer: ({ output, expected }) => {
-    try {
-      const result = JSON.parse(String(output)) as CompactionResult;
-      const exp = expected as CompactionExpected;
-
-      const injectionMatches = result.contextInjected === exp.contextInjected;
-      const typeMatches = result.contextType === exp.contextType;
-
-      if (injectionMatches && typeMatches) {
-        return {
-          score: 1,
-          message: `Correct injection: ${result.contextType}`,
-        };
-      }
-
-      if (injectionMatches) {
-        return {
-          score: 0.5,
-          message: `Injection status correct but wrong type: got ${result.contextType}, expected ${exp.contextType}`,
-        };
-      }
-
-      return {
-        score: 0,
-        message: `Wrong injection: got ${result.contextInjected ? result.contextType : "none"}, expected ${exp.contextInjected ? exp.contextType : "none"}`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse result: ${error}`,
-      };
-    }
-  },
-});
-
-/**
- * Validates that injected context contains required patterns
- *
- * For coordinator resumption, context MUST include:
- * - Swarm continuation instructions
- * - Tool names (swarm_status, swarmmail_inbox)
- * - Actionable language ("COORDINATOR", "Keep Cooking")
- *
- * Score: ratio of required patterns found (0.0 to 1.0)
- */
-export const requiredPatternsPresent = createScorer({
-  name: "Required Patterns Present",
-  description: "Validates injected context contains required patterns",
-  scorer: ({ output, expected }) => {
-    try {
-      const result = JSON.parse(String(output)) as CompactionResult;
-      const exp = expected as CompactionExpected;
-
-      // If no context injected, check that mustContain is empty
-      if (!result.contextInjected) {
-        if (!exp.mustContain || exp.mustContain.length === 0) {
-          return {
-            score: 1,
-            message: "No context injected (expected)",
-          };
-        }
-        return {
-          score: 0,
-          message: "No context injected but patterns were expected",
-        };
-      }
-
-      // Check required patterns
-      if (!exp.mustContain || exp.mustContain.length === 0) {
-        return {
-          score: 1,
-          message: "No required patterns to check",
-        };
-      }
-
-      const found = exp.mustContain.filter((pattern) =>
-        result.injectedContext.includes(pattern),
-      );
-
-      const score = found.length / exp.mustContain.length;
-
-      if (score === 1) {
-        return {
-          score: 1,
-          message: `All ${exp.mustContain.length} required patterns found`,
-        };
-      }
-
-      const missing = exp.mustContain.filter(
-        (pattern) => !result.injectedContext.includes(pattern),
-      );
-
-      return {
-        score,
-        message: `${found.length}/${exp.mustContain.length} patterns found. Missing: ${missing.join(", ")}`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse result: ${error}`,
-      };
-    }
-  },
-});
-
-/**
- * Validates that injected context excludes forbidden patterns
- *
- * Context should NOT contain:
- * - Placeholder IDs ("bd-xxx")
- * - Generic/template language
- * - Wrong context type markers
- *
- * Score: 1.0 if no forbidden patterns found, 0.0 if any found
- */
-export const forbiddenPatternsAbsent = createScorer({
-  name: "Forbidden Patterns Absent",
-  description: "Validates injected context excludes forbidden patterns",
-  scorer: ({ output, expected }) => {
-    try {
-      const result = JSON.parse(String(output)) as CompactionResult;
-      const exp = expected as CompactionExpected;
-
-      // If no context injected, all checks pass
-      if (!result.contextInjected) {
-        return {
-          score: 1,
-          message: "No context injected (no forbidden patterns possible)",
-        };
-      }
-
-      // Check forbidden patterns
-      if (!exp.mustNotContain || exp.mustNotContain.length === 0) {
-        return {
-          score: 1,
-          message: "No forbidden patterns to check",
-        };
-      }
-
-      const foundForbidden = exp.mustNotContain.filter((pattern) =>
-        result.injectedContext.includes(pattern),
-      );
-
-      if (foundForbidden.length === 0) {
-        return {
-          score: 1,
-          message: "No forbidden patterns found",
-        };
-      }
-
-      return {
-        score: 0,
-        message: `Forbidden patterns found: ${foundForbidden.join(", ")}`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to parse result: ${error}`,
-      };
-    }
-  },
-});
-
-/**
- * Composite scorer: Overall compaction quality
- *
- * Combines all compaction-specific checks into single score.
- * Weighted average:
- * - Confidence accuracy: 25%
- * - Context injection: 25%
- * - Required patterns: 30%
- * - Forbidden patterns: 20%
- *
- * Score: 0.0 to 1.0
- */
-export const compactionQuality = createScorer({
-  name: "Overall Compaction Quality",
-  description: "Composite score for compaction hook correctness",
-  scorer: async ({ output, expected, input }) => {
-    try {
-      // Run all scorers
-      const scores = {
-        confidence: await confidenceAccuracy({ output, expected, input }),
-        injection: await contextInjectionCorrectness({ output, expected, input }),
-        required: await requiredPatternsPresent({ output, expected, input }),
-        forbidden: await forbiddenPatternsAbsent({ output, expected, input }),
-      };
-
-      // Weighted average
-      const weights = {
-        confidence: 0.25,
-        injection: 0.25,
-        required: 0.3,
-        forbidden: 0.2,
-      };
-
-      const totalScore =
-        (scores.confidence.score ?? 0) * weights.confidence +
-        (scores.injection.score ?? 0) * weights.injection +
-        (scores.required.score ?? 0) * weights.required +
-        (scores.forbidden.score ?? 0) * weights.forbidden;
-
-      const details = [
-        `Confidence: ${((scores.confidence.score ?? 0) * 100).toFixed(0)}%`,
-        `Injection: ${((scores.injection.score ?? 0) * 100).toFixed(0)}%`,
-        `Required: ${((scores.required.score ?? 0) * 100).toFixed(0)}%`,
-        `Forbidden: ${((scores.forbidden.score ?? 0) * 100).toFixed(0)}%`,
-      ].join(", ");
-
-      return {
-        score: totalScore,
-        message: `Overall: ${(totalScore * 100).toFixed(0)}% (${details})`,
-      };
-    } catch (error) {
-      return {
-        score: 0,
-        message: `Failed to compute composite score: ${error}`,
-      };
-    }
-  },
-});