opencode-swarm-plugin 0.44.0 → 0.44.2

package/evals/lib/data-loader.ts

@@ -1,281 +0,0 @@
-/**
- * PGlite-backed eval data loader
- *
- * Loads real decomposition outcomes from the eval_records table
- * for use in Evalite evals.
- */
-import * as fs from "node:fs";
-import {
-  getEvalRecords,
-  getEvalStats,
-  type EvalRecord,
-} from "swarm-mail";
-
-export interface EvalCase {
-  input: { task: string; context?: string };
-  expected: {
-    minSubtasks: number;
-    maxSubtasks: number;
-    requiredFiles?: string[];
-    overallSuccess?: boolean;
-  };
-  actual?: EvalRecord;
-}
-
-/**
- * Load eval cases from PGlite
- *
- * @param projectKey - Project key for filtering records
- * @param options - Filter options
- * @returns Array of eval cases ready for Evalite
- */
-export async function loadEvalCases(
-  projectKey: string,
-  options?: {
-    limit?: number;
-    strategy?: "file-based" | "feature-based" | "risk-based";
-    successOnly?: boolean;
-    projectPath?: string;
-  },
-): Promise<EvalCase[]> {
-  const { limit, strategy, successOnly, projectPath } = options ?? {};
-
-  // Query eval records from PGlite
-  const records = await getEvalRecords(
-    projectKey,
-    { limit, strategy },
-    projectPath,
-  );
-
-  // Filter by success if requested
-  const filtered = successOnly
-    ? records.filter((r) => r.overall_success === true)
-    : records;
-
-  // Transform to EvalCase format
-  return filtered.map((record) => ({
-    input: {
-      task: record.task,
-      context: record.context ?? undefined,
-    },
-    expected: {
-      minSubtasks: 2,
-      maxSubtasks: record.subtasks.length,
-      requiredFiles: record.subtasks.flatMap((s) => s.files),
-      overallSuccess: record.overall_success ?? undefined,
-    },
-    actual: record,
-  }));
-}
-
-/**
- * Check if we have enough real data to run evals
- *
- * @param projectKey - Project key to check
- * @param minRecords - Minimum number of records required (default: 5)
- * @param projectPath - Optional project path for database lookup
- * @returns True if enough data exists
- */
-export async function hasRealEvalData(
-  projectKey: string,
-  minRecords: number = 5,
-  projectPath?: string,
-): Promise<boolean> {
-  const stats = await getEvalStats(projectKey, projectPath);
-  return stats.totalRecords >= minRecords;
-}
-
-/**
- * Get eval data stats for reporting
- *
- * @param projectKey - Project key to query
- * @param projectPath - Optional project path for database lookup
- * @returns Summary of available eval data
- */
-export async function getEvalDataSummary(
-  projectKey: string,
-  projectPath?: string,
-): Promise<{
-  totalRecords: number;
-  successRate: number;
-  byStrategy: Record<string, number>;
-  hasEnoughData: boolean;
-}> {
-  const stats = await getEvalStats(projectKey, projectPath);
-
-  return {
-    totalRecords: stats.totalRecords,
-    successRate: stats.successRate,
-    byStrategy: stats.byStrategy,
-    hasEnoughData: stats.totalRecords >= 5,
-  };
-}
-
-/**
- * Check if a session meets quality criteria
- */
-function meetsQualityCriteria(
-  session: import("../../src/eval-capture.js").CoordinatorSession,
-  criteria: {
-    minEvents: number;
-    requireWorkerSpawn: boolean;
-    requireReview: boolean;
-  },
-): boolean {
-  // Filter 1: minEvents
-  if (session.events.length < criteria.minEvents) {
-    return false;
-  }
-
-  // Filter 2: requireWorkerSpawn
-  if (
-    criteria.requireWorkerSpawn &&
-    !session.events.some(
-      (e) => e.event_type === "DECISION" && e.decision_type === "worker_spawned",
-    )
-  ) {
-    return false;
-  }
-
-  // Filter 3: requireReview
-  if (
-    criteria.requireReview &&
-    !session.events.some(
-      (e) =>
-        e.event_type === "DECISION" && e.decision_type === "review_completed",
-    )
-  ) {
-    return false;
-  }
-
-  return true;
-}
-
-/**
- * Load captured coordinator sessions from ~/.config/swarm-tools/sessions/
- *
- * Reads all JSONL session files and returns CoordinatorSession objects.
- *
- * Quality filters are applied to focus on high-signal coordinator sessions:
- * - minEvents: Filter out incomplete/aborted sessions (default: 3)
- * - requireWorkerSpawn: Ensure session delegated to workers (default: true)
- * - requireReview: Ensure coordinator reviewed work (default: true)
- *
- * Filters are applied BEFORE the limit for accurate sampling.
- *
- * @param options - Filter options
- * @returns Array of coordinator sessions that meet quality criteria
- */
-export async function loadCapturedSessions(options?: {
-  sessionIds?: string[];
-  limit?: number;
-  /** Minimum number of events required (default: 3) */
-  minEvents?: number;
-  /** Require at least one worker_spawned event (default: true) */
-  requireWorkerSpawn?: boolean;
-  /** Require at least one review_completed event (default: true) */
-  requireReview?: boolean;
-  /** Override session directory for testing */
-  sessionDir?: string;
-}): Promise<
-  Array<{ session: import("../../src/eval-capture.js").CoordinatorSession }>
-> {
-  const { getSessionDir, readSessionEvents, saveSession } = await import(
-    "../../src/eval-capture.js"
-  );
-  const sessionDir = options?.sessionDir ?? getSessionDir();
-
-  // Default quality filters
-  const qualityCriteria = {
-    minEvents: options?.minEvents ?? 3,
-    requireWorkerSpawn: options?.requireWorkerSpawn ?? true,
-    requireReview: options?.requireReview ?? true,
-  };
-
-  // If session dir doesn't exist, return empty
-  if (!fs.existsSync(sessionDir)) {
-    return [];
-  }
-
-  // Read all .jsonl files in session directory
-  const files = fs
-    .readdirSync(sessionDir)
-    .filter((f) => f.endsWith(".jsonl"));
-
-  // Filter by sessionIds if provided
-  const targetFiles = options?.sessionIds
-    ? files.filter((f) => options.sessionIds?.includes(f.replace(".jsonl", "")))
-    : files;
-
-  // Load each session
-  const sessions: Array<{
-    session: import("../../src/eval-capture.js").CoordinatorSession;
-  }> = [];
-  let filteredOutCount = 0;
-
-  for (const file of targetFiles) {
-    const sessionId = file.replace(".jsonl", "");
-
-    try {
-      let events: import("../../src/eval-capture.js").CoordinatorEvent[];
-
-      // If custom sessionDir, read directly; otherwise use eval-capture functions
-      if (options?.sessionDir) {
-        const sessionPath = `${sessionDir}/${sessionId}.jsonl`;
-        if (!fs.existsSync(sessionPath)) continue;
-
-        const content = fs.readFileSync(sessionPath, "utf-8");
-        const lines = content.trim().split("\n").filter(Boolean);
-        const { CoordinatorEventSchema } = await import(
-          "../../src/eval-capture.js"
-        );
-        events = lines.map((line) => {
-          const parsed = JSON.parse(line);
-          return CoordinatorEventSchema.parse(parsed);
-        });
-      } else {
-        events = readSessionEvents(sessionId);
-      }
-
-      if (events.length === 0) continue;
-
-      // Find epic_id from first event
-      const epicId = events[0]?.epic_id;
-      if (!epicId) continue;
-
-      // Build session object
-      const session: import("../../src/eval-capture.js").CoordinatorSession = {
-        session_id: sessionId,
-        epic_id: epicId,
-        start_time: events[0]?.timestamp ?? new Date().toISOString(),
-        end_time: events[events.length - 1]?.timestamp,
-        events,
-      };
-      if (!session) continue;
-
-      // Apply quality filters BEFORE limit
-      if (meetsQualityCriteria(session, qualityCriteria)) {
-        sessions.push({ session });
-      } else {
-        filteredOutCount++;
-      }
-    } catch (error) {
-      // Skip invalid sessions
-      console.warn(`Failed to load session ${sessionId}:`, error);
-    }
-
-    // Apply limit AFTER filtering
-    if (options?.limit && sessions.length >= options.limit) {
-      break;
-    }
-  }
-
-  // Log filtering stats for visibility
-  if (filteredOutCount > 0) {
-    console.log(
-      `Filtered out ${filteredOutCount} sessions (minEvents=${qualityCriteria.minEvents}, requireWorkerSpawn=${qualityCriteria.requireWorkerSpawn}, requireReview=${qualityCriteria.requireReview})`,
-    );
-  }
-
-  return sessions;
-}

package/evals/lib/llm.ts

@@ -1,115 +0,0 @@
-/**
- * LLM Client for Evalite Evals
- *
- * Uses AI SDK v6 with Vercel AI Gateway.
- * Gateway handles provider routing - just pass "provider/model" string.
- *
- * @module evals/lib/llm
- */
-import { generateText, gateway } from "ai";
-import type { GatewayModelId } from "ai";
-
-/**
- * Default model for decomposition evals
- * Using Claude Sonnet for good balance of quality and cost
- */
-export const DEFAULT_MODEL: GatewayModelId = "anthropic/claude-sonnet-4-5";
-
-/**
- * Generate a decomposition from a task description
- *
- * @param prompt - The full decomposition prompt
- * @param model - Gateway model ID (e.g., "anthropic/claude-sonnet-4-5")
- * @returns The raw text response from the LLM
- */
-export async function generateDecomposition(
-  prompt: string,
-  model: GatewayModelId = DEFAULT_MODEL,
-): Promise<string> {
-  const { text } = await generateText({
-    model: gateway(model),
-    prompt,
-    maxOutputTokens: 4096,
-  });
-
-  return text;
-}
-
-/**
- * Format a decomposition prompt from task and context
- *
- * Uses the same prompt template as swarm_plan_prompt
- */
-export function formatDecompositionPrompt(
-  task: string,
-  context?: string,
-  maxSubtasks: number = 6,
-): string {
-  const contextSection = context ? `## Context\n${context}` : "";
-
-  return `You are decomposing a task into parallelizable subtasks for a swarm of agents.
-
-## Task
-${task}
-
-${contextSection}
-
-## Requirements
-
-1. **Break into 2-${maxSubtasks} independent subtasks** that can run in parallel
-2. **Assign files** - each subtask must specify which files it will modify
-3. **No file overlap** - files cannot appear in multiple subtasks (they get exclusive locks)
-4. **Order by dependency** - if subtask B needs subtask A's output, A must come first in the array
-5. **Estimate complexity** - 1 (trivial) to 5 (complex)
-
-## Response Format
-
-Respond with ONLY a JSON object matching this schema (no markdown, no explanation):
-
-{
-  "epic": {
-    "title": "string",
-    "description": "string"
-  },
-  "subtasks": [
-    {
-      "title": "string",
-      "description": "string",
-      "files": ["string"],
-      "dependencies": [0],
-      "estimated_complexity": 1
-    }
-  ]
-}
-
-## Guidelines
-
-- **Plan aggressively** - when in doubt, split further
-- **Prefer smaller, focused subtasks** over large complex ones
-- **Include test files** in the same subtask as the code they test
-- **Be specific about files** - use actual file paths, not placeholders
-
-Now decompose the task. Respond with JSON only:`;
-}
-
-/**
- * Extract JSON from LLM response
- *
- * Handles responses that may have markdown code blocks or extra text
- */
-export function extractJson(text: string): string {
-  // Try to find JSON in code blocks first
-  const codeBlockMatch = text.match(/```(?:json)?\s*([\s\S]*?)```/);
-  if (codeBlockMatch) {
-    return codeBlockMatch[1].trim();
-  }
-
-  // Try to find raw JSON object
-  const jsonMatch = text.match(/\{[\s\S]*\}/);
-  if (jsonMatch) {
-    return jsonMatch[0];
-  }
-
-  // Return as-is if no JSON found
-  return text;
-}

package/evals/scorers/compaction-prompt-scorers.ts

@@ -1,145 +0,0 @@
-/**
- * Compaction Prompt Quality Scorers - Evalite Wrappers
- *
- * These wrap the pure scoring functions from src/compaction-prompt-scoring.ts
- * for use with evalite's test runner.
- *
- * Weighted scoring:
- * - epicIdSpecificity (0.20) - real IDs not placeholders
- * - actionability (0.20) - swarm_status/inbox with real values
- * - coordinatorIdentity (0.25) - ASCII header + strong mandates
- * - forbiddenToolsPresent (0.15) - lists forbidden tools by name
- * - postCompactionDiscipline (0.20) - first tool correct, no edit/write
- */
-
-import { createScorer } from "evalite";
-import type { CompactionPrompt } from "../../src/compaction-prompt-scoring.js";
-import {
-	scoreActionability,
-	scoreCoordinatorIdentity,
-	scoreEpicIdSpecificity,
-	scoreForbiddenToolsPresent,
-	scorePostCompactionDiscipline,
-} from "../../src/compaction-prompt-scoring.js";
-
-// Re-export types for convenience
-export type { CompactionPrompt, ScorerResult } from "../../src/compaction-prompt-scoring.js";
-
-// Re-export pure functions for direct use
-export {
-	scoreActionability,
-	scoreCoordinatorIdentity,
-	scoreEpicIdSpecificity,
-	scoreForbiddenToolsPresent,
-	scorePostCompactionDiscipline,
-} from "../../src/compaction-prompt-scoring.js";
-
-/**
- * Epic ID Specificity Scorer
- *
- * Validates that epic IDs are REAL, not placeholders.
- * Score: 1.0 if real IDs, 0.0 if placeholders found
- */
-export const epicIdSpecificity = createScorer({
-	name: "Epic ID Specificity",
-	description: "Prompt uses real epic IDs, not placeholders",
-	scorer: ({ output }) => {
-		try {
-			const prompt = JSON.parse(String(output)) as CompactionPrompt;
-			return scoreEpicIdSpecificity(prompt);
-		} catch (error) {
-			return {
-				score: 0,
-				message: `Failed to parse prompt: ${error}`,
-			};
-		}
-	},
-});
-
-/**
- * Actionability Scorer
- *
- * Validates that the prompt includes SPECIFIC actionable tool calls.
- * Score: 1.0 if actionable tool calls with real values, 0.0 otherwise
- */
-export const actionability = createScorer({
-	name: "Actionability",
-	description: "Prompt includes specific tool calls with real values",
-	scorer: ({ output }) => {
-		try {
-			const prompt = JSON.parse(String(output)) as CompactionPrompt;
-			return scoreActionability(prompt);
-		} catch (error) {
-			return {
-				score: 0,
-				message: `Failed to parse prompt: ${error}`,
-			};
-		}
-	},
-});
-
-/**
- * Coordinator Identity Scorer
- *
- * Validates that the prompt has STRONG coordinator identity reinforcement.
- * Score: 1.0 for ASCII header + strong mandates, 0.5 for header only, 0.0 otherwise
- */
-export const coordinatorIdentity = createScorer({
-	name: "Coordinator Identity",
-	description: "Prompt has ASCII header and strong mandates",
-	scorer: ({ output }) => {
-		try {
-			const prompt = JSON.parse(String(output)) as CompactionPrompt;
-			return scoreCoordinatorIdentity(prompt);
-		} catch (error) {
-			return {
-				score: 0,
-				message: `Failed to parse prompt: ${error}`,
-			};
-		}
-	},
-});
-
-/**
- * Forbidden Tools Present Scorer
- *
- * Validates that the prompt LISTS forbidden tools by name.
- * Score: ratio of forbidden tools mentioned (0.0 to 1.0)
- */
-export const forbiddenToolsPresent = createScorer({
-	name: "Forbidden Tools Present",
-	description: "Prompt lists forbidden tools by name",
-	scorer: ({ output }) => {
-		try {
-			const prompt = JSON.parse(String(output)) as CompactionPrompt;
-			return scoreForbiddenToolsPresent(prompt);
-		} catch (error) {
-			return {
-				score: 0,
-				message: `Failed to parse prompt: ${error}`,
-			};
-		}
-	},
-});
-
-/**
- * Post-Compaction Discipline Scorer
- *
- * Validates that the FIRST suggested tool is correct.
- * Score: 1.0 if first tool is swarm_status or inbox, 0.0 otherwise
- */
-export const postCompactionDiscipline = createScorer({
-	name: "Post-Compaction Discipline",
-	description: "First suggested tool is swarm_status or inbox",
-	scorer: ({ output }) => {
-		try {
-			const prompt = JSON.parse(String(output)) as CompactionPrompt;
-			return scorePostCompactionDiscipline(prompt);
-		} catch (error) {
-			return {
-				score: 0,
-				message: `Failed to parse prompt: ${error}`,
-			};
-		}
-	},
-});