opencode-swarm-plugin 0.39.1 → 0.42.0

package/src/eval-runner.test.ts

@@ -0,0 +1,223 @@
+/**
+ * Tests for eval-runner - Programmatic evalite execution
+ *
+ * TDD: These tests MUST fail initially, then pass after implementation.
+ */
+
+import { describe, test, expect, beforeAll, afterEach } from "bun:test";
+import { runEvals } from "./eval-runner";
+import path from "node:path";
+import fs from "node:fs";
+import { getEvalHistoryPath } from "./eval-history";
+
+// Use project root for all tests
+const PROJECT_ROOT = path.resolve(import.meta.dir, "..");
+
+describe("runEvals", () => {
+  test("runs all evals when no suite filter provided", async () => {
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+    });
+
+    // Even if some evals fail, we should get results
+    expect(typeof result.success).toBe("boolean");
+    expect(typeof result.totalSuites).toBe("number");
+    expect(typeof result.totalEvals).toBe("number");
+    expect(typeof result.averageScore).toBe("number");
+    expect(Array.isArray(result.suites)).toBe(true);
+
+    // Should have at least the example.eval.ts suite
+    expect(result.totalSuites).toBeGreaterThan(0);
+    expect(result.suites.length).toBeGreaterThan(0);
+  }, 60000); // 60s timeout for full eval run
+
+  test("filters evals by suite name", async () => {
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+      suiteFilter: "example",
+    });
+
+    expect(result.success).toBe(true);
+    // All suite filepaths should contain "example"
+    for (const suite of result.suites) {
+      expect(suite.filepath.toLowerCase()).toContain("example");
+    }
+  }, 30000);
+
+  test("respects score threshold", async () => {
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+      suiteFilter: "example", // Known good eval
+      scoreThreshold: 0, // Very low threshold, should pass
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.averageScore).toBeGreaterThanOrEqual(0);
+  }, 30000);
+
+  test("returns structured suite results with scores", async () => {
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+      suiteFilter: "example",
+    });
+
+    expect(result.suites.length).toBeGreaterThan(0);
+    
+    const suite = result.suites[0];
+    expect(suite).toMatchObject({
+      name: expect.any(String),
+      filepath: expect.any(String),
+      status: expect.stringMatching(/^(success|fail|running)$/),
+      duration: expect.any(Number),
+      averageScore: expect.any(Number),
+      evalCount: expect.any(Number),
+    });
+  }, 30000);
+
+  test("handles errors gracefully", async () => {
+    const result = await runEvals({
+      cwd: "/nonexistent/path",
+    });
+
+    expect(result.success).toBe(false);
+    expect(result.error).toBeDefined();
+    expect(result.suites).toEqual([]);
+  }, 10000);
+
+  test("returns empty results when no evals match filter", async () => {
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+      suiteFilter: "nonexistent-eval-name-xyz",
+    });
+
+    // Should succeed but with no suites
+    expect(result.success).toBe(true);
+    expect(result.totalSuites).toBe(0);
+    expect(result.suites).toEqual([]);
+  }, 10000);
+
+  test("records eval run to history after execution", async () => {
+    // Clean up any existing history before test
+    const historyPath = getEvalHistoryPath(PROJECT_ROOT);
+    const historyBackup = historyPath + ".backup";
+    
+    // Backup existing history
+    if (fs.existsSync(historyPath)) {
+      fs.copyFileSync(historyPath, historyBackup);
+    }
+    
+    try {
+      // Remove history file to get clean state
+      if (fs.existsSync(historyPath)) {
+        fs.unlinkSync(historyPath);
+      }
+
+      // Run evals
+      const result = await runEvals({
+        cwd: PROJECT_ROOT,
+        suiteFilter: "example",
+      });
+
+      // Should have succeeded
+      expect(result.success).toBe(true);
+      expect(result.suites.length).toBeGreaterThan(0);
+
+      // History file should have been created
+      expect(fs.existsSync(historyPath)).toBe(true);
+
+      // Read history file
+      const historyContent = fs.readFileSync(historyPath, "utf-8");
+      const lines = historyContent.trim().split("\n");
+
+      // Should have one line per suite
+      expect(lines.length).toBe(result.suites.length);
+
+      // Parse first line and verify structure
+      const firstRecord = JSON.parse(lines[0]);
+      
+      // Verify structure has all required fields
+      expect(typeof firstRecord.timestamp).toBe("string");
+      expect(typeof firstRecord.eval_name).toBe("string");
+      expect(typeof firstRecord.score).toBe("number");
+      expect(typeof firstRecord.run_count).toBe("number");
+
+      // Verify eval_name matches suite name
+      expect(firstRecord.eval_name).toBe(result.suites[0].name);
+      
+      // Verify score matches suite averageScore
+      expect(firstRecord.score).toBe(result.suites[0].averageScore);
+      
+      // First run should have run_count = 1
+      expect(firstRecord.run_count).toBe(1);
+    } finally {
+      // Restore backup
+      if (fs.existsSync(historyBackup)) {
+        fs.copyFileSync(historyBackup, historyPath);
+        fs.unlinkSync(historyBackup);
+      }
+    }
+  }, 30000);
+
+  test("checks gates for each suite after recording", async () => {
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+      suiteFilter: "example",
+    });
+
+    expect(result.success).toBe(true);
+    expect(result.gateResults).toBeDefined();
+    expect(Array.isArray(result.gateResults)).toBe(true);
+    
+    // Should have gate result for each suite
+    expect(result.gateResults?.length).toBe(result.suites.length);
+    
+    // Each gate result should have required fields
+    if (result.gateResults && result.gateResults.length > 0) {
+      const gateResult = result.gateResults[0];
+      expect(gateResult).toHaveProperty("suite");
+      expect(gateResult).toHaveProperty("passed");
+      expect(gateResult).toHaveProperty("phase");
+      expect(gateResult).toHaveProperty("message");
+      expect(gateResult).toHaveProperty("currentScore");
+    }
+  }, 30000);
+
+  test("calls learnFromEvalFailure when gate fails", async () => {
+    // This test requires manually creating a history with regression
+    // For now, we just verify the code path exists
+    // In practice, this would be tested with mocked checkGate returning failed=true
+    
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+      suiteFilter: "example",
+    });
+
+    // Gate results should be present even if no failures
+    expect(result.gateResults).toBeDefined();
+  }, 30000);
+
+  test("does NOT call learnFromEvalFailure when gate passes", async () => {
+    // Similar to above - verifies the happy path
+    // Real test would mock checkGate and verify learnFromEvalFailure NOT called
+    
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+      suiteFilter: "example",
+    });
+
+    // Should succeed with gate results
+    expect(result.success).toBe(true);
+    expect(result.gateResults).toBeDefined();
+  }, 30000);
+
+  test("includes gateResults in return value", async () => {
+    const result = await runEvals({
+      cwd: PROJECT_ROOT,
+      suiteFilter: "example",
+    });
+
+    // gateResults should be array (even if empty)
+    expect(result).toHaveProperty("gateResults");
+    expect(Array.isArray(result.gateResults)).toBe(true);
+  }, 30000);
+});

package/src/eval-runner.ts

@@ -0,0 +1,402 @@
+/**
+ * Programmatic Evalite Runner
+ *
+ * Provides a type-safe API for running evalite evals programmatically.
+ * Wraps evalite's runEvalite function with structured result parsing.
+ *
+ * @module eval-runner
+ */
+
+import { tool } from "@opencode-ai/plugin";
+import { runEvalite } from "evalite/runner";
+import { createInMemoryStorage } from "evalite/in-memory-storage";
+import type { Evalite } from "evalite/types";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { recordEvalRun, getScoreHistory } from "./eval-history.js";
+import { checkGate } from "./eval-gates.js";
+import { learnFromEvalFailure } from "./eval-learning.js";
+import { getMemoryAdapter } from "./memory-tools.js";
+
+/**
+ * Options for running evals programmatically
+ */
+export interface RunEvalsOptions {
+  /**
+   * Working directory containing eval files (defaults to process.cwd())
+   */
+  cwd?: string;
+  
+  /**
+   * Optional filter to run specific eval suites (e.g., "coordinator", "compaction")
+   * Matches against eval file paths using substring matching
+   */
+  suiteFilter?: string;
+  
+  /**
+   * Minimum average score threshold (0-100)
+   * If average score falls below this, result.success will be false
+   */
+  scoreThreshold?: number;
+  
+  /**
+   * Optional path to write raw evalite JSON output
+   */
+  outputPath?: string;
+}
+
+/**
+ * Structured suite result with scores
+ */
+export interface SuiteResult {
+  /** Suite name from evalite() call */
+  name: string;
+  
+  /** Absolute path to eval file */
+  filepath: string;
+  
+  /** Suite status: success, fail, or running */
+  status: "success" | "fail" | "running";
+  
+  /** Total duration in milliseconds */
+  duration: number;
+  
+  /** Average score across all evals in suite (0-1 scale) */
+  averageScore: number;
+  
+  /** Number of evals in this suite */
+  evalCount: number;
+  
+  /** Individual eval results (optional, can be large) */
+  evals?: Array<{
+    input: unknown;
+    output: unknown;
+    expected?: unknown;
+    scores: Array<{
+      name: string;
+      score: number;
+      description?: string;
+    }>;
+  }>;
+}
+
+/**
+ * Structured result from running evals
+ */
+export interface RunEvalsResult {
+  /** Whether the run succeeded (all evals passed threshold) */
+  success: boolean;
+  
+  /** Total number of suites executed */
+  totalSuites: number;
+  
+  /** Total number of individual evals executed */
+  totalEvals: number;
+  
+  /** Average score across all suites (0-1 scale) */
+  averageScore: number;
+  
+  /** Individual suite results */
+  suites: SuiteResult[];
+  
+  /** Error message if run failed */
+  error?: string;
+  
+  /** Gate check results per suite */
+  gateResults?: Array<{
+    suite: string;
+    passed: boolean;
+    phase: string;
+    message: string;
+    baseline?: number;
+    currentScore: number;
+    regressionPercent?: number;
+  }>;
+}
+
+/**
+ * Run evalite evals programmatically
+ *
+ * @param options - Configuration for eval run
+ * @returns Structured results with scores per suite
+ *
+ * @example
+ * ```typescript
+ * // Run all evals
+ * const result = await runEvals({ cwd: "/path/to/project" });
+ * console.log(`Average score: ${result.averageScore}`);
+ *
+ * // Run specific suite
+ * const coordResult = await runEvals({
+ *   cwd: "/path/to/project",
+ *   suiteFilter: "coordinator"
+ * });
+ *
+ * // Enforce score threshold
+ * const gatedResult = await runEvals({
+ *   cwd: "/path/to/project",
+ *   scoreThreshold: 80
+ * });
+ * if (!gatedResult.success) {
+ *   throw new Error(`Evals failed threshold: ${gatedResult.averageScore}`);
+ * }
+ * ```
+ */
+export async function runEvals(
+  options: RunEvalsOptions = {}
+): Promise<RunEvalsResult> {
+  const {
+    cwd = process.cwd(),
+    suiteFilter,
+    scoreThreshold,
+    outputPath: userOutputPath,
+  } = options;
+
+  try {
+    // Resolve to project root (evals are in evals/ relative to project root)
+    // If cwd is src/, go up one level
+    const projectRoot = cwd.endsWith("src") ? path.dirname(cwd) : cwd;
+    const evalsDir = path.join(projectRoot, "evals");
+    let evalPath: string | undefined;
+
+    if (suiteFilter) {
+      // Find matching eval files
+      try {
+        const files = await fs.readdir(evalsDir);
+        const matchingFiles = files.filter((f) =>
+          f.toLowerCase().includes(suiteFilter.toLowerCase())
+        );
+
+        if (matchingFiles.length === 0) {
+          // No matches - return empty result (not an error)
+          return {
+            success: true,
+            totalSuites: 0,
+            totalEvals: 0,
+            averageScore: 0,
+            suites: [],
+          };
+        }
+
+        // Use first matching file (evalite will discover all via vitest)
+        evalPath = path.join(evalsDir, matchingFiles[0]);
+      } catch (err) {
+        // Directory doesn't exist or can't be read
+        return {
+          success: false,
+          totalSuites: 0,
+          totalEvals: 0,
+          averageScore: 0,
+          suites: [],
+          error: `Failed to read evals directory: ${err instanceof Error ? err.message : String(err)}`,
+        };
+      }
+    } else {
+      // No filter - run all evals in evals/
+      evalPath = evalsDir;
+    }
+
+    // Use temporary output path if user didn't provide one
+    const outputPath =
+      userOutputPath || path.join(projectRoot, `.evalite-results-${Date.now()}.json`);
+    const isTemporaryOutput = !userOutputPath;
+
+    // Run evalite programmatically
+    const storage = createInMemoryStorage();
+    
+    await runEvalite({
+      path: evalPath, // undefined = run all
+      cwd: projectRoot, // Use project root as working directory
+      mode: "run-once",
+      scoreThreshold,
+      outputPath,
+      hideTable: true, // Suppress terminal output
+      storage,
+      disableServer: true, // No UI server needed
+    });
+
+    // Parse output file for structured results
+    let outputJson: string;
+    try {
+      outputJson = await fs.readFile(outputPath, "utf-8");
+    } catch (err) {
+      // Output file wasn't written - evalite crashed or no tests ran
+      return {
+        success: false,
+        totalSuites: 0,
+        totalEvals: 0,
+        averageScore: 0,
+        suites: [],
+        error: `No results file generated: ${err instanceof Error ? err.message : String(err)}`,
+      };
+    }
+
+    const output: Evalite.Exported.Output = JSON.parse(outputJson);
+
+    // Clean up temporary output file
+    if (isTemporaryOutput) {
+      await fs.unlink(outputPath).catch(() => {
+        /* ignore cleanup errors */
+      });
+    }
+
+    // Transform to structured result
+    const suites: SuiteResult[] = output.suites.map((suite) => ({
+      name: suite.name,
+      filepath: suite.filepath,
+      status: suite.status,
+      duration: suite.duration,
+      averageScore: suite.averageScore,
+      evalCount: suite.evals.length,
+      // Include evals if user wants detailed results
+      evals: suite.evals.map((e) => ({
+        input: e.input,
+        output: e.output,
+        expected: e.expected,
+        scores: e.scores.map((s) => ({
+          name: s.name,
+          score: s.score,
+          description: s.description,
+        })),
+      })),
+    }));
+
+    // Record eval runs to history
+    for (const suite of suites) {
+      const history = getScoreHistory(projectRoot, suite.name);
+      recordEvalRun(projectRoot, {
+        timestamp: new Date().toISOString(),
+        eval_name: suite.name,
+        score: suite.averageScore,
+        run_count: history.length + 1,
+      });
+    }
+
+    // Check gates for each suite
+    const gateResults = [];
+    for (const suite of suites) {
+      const history = getScoreHistory(projectRoot, suite.name);
+      const gate = checkGate(projectRoot, suite.name, suite.averageScore);
+      gateResults.push({ suite: suite.name, ...gate });
+      
+      // If gate failed, trigger learning
+      if (!gate.passed) {
+        try {
+          const memoryAdapter = await getMemoryAdapter();
+          await learnFromEvalFailure(suite.name, suite.averageScore, history, memoryAdapter);
+        } catch (e) {
+          // Learning is best-effort, don't fail the eval run
+          console.warn(`Failed to store learning for ${suite.name}:`, e);
+        }
+      }
+    }
+
+    // Calculate overall metrics
+    const totalEvals = suites.reduce((sum, s) => sum + s.evalCount, 0);
+    const averageScore =
+      suites.length > 0
+        ? suites.reduce((sum, s) => sum + s.averageScore, 0) / suites.length
+        : 0;
+
+    // Determine success based on threshold
+    const thresholdPassed =
+      scoreThreshold === undefined || averageScore * 100 >= scoreThreshold;
+
+    return {
+      success: thresholdPassed,
+      totalSuites: suites.length,
+      totalEvals,
+      averageScore,
+      suites,
+      gateResults,
+    };
+  } catch (error) {
+    // Return error result
+    return {
+      success: false,
+      totalSuites: 0,
+      totalEvals: 0,
+      averageScore: 0,
+      suites: [],
+      error:
+        error instanceof Error
+          ? error.message
+          : String(error),
+    };
+  }
+}
+
+// ============================================================================
+// Plugin Tool
+// ============================================================================
+
+/**
+ * Plugin tool for running evals programmatically
+ */
+const eval_run = tool({
+  description: `Run evalite evals programmatically and get structured results with scores.
+
+Use this to:
+- Run all evals in evals/ directory
+- Filter by specific eval suite (e.g., "coordinator", "compaction")
+- Enforce score thresholds for quality gates
+- Get per-suite and per-eval scores
+
+Returns structured JSON with:
+- success: boolean (true if all tests passed threshold)
+- totalSuites: number of eval suites run
+- totalEvals: number of individual test cases
+- averageScore: 0-1 score across all suites
+- suites: array of suite results with scores
+
+Example usage:
+- Run all evals: eval_run()
+- Run coordinator evals: eval_run({ suiteFilter: "coordinator" })
+- Enforce 80% threshold: eval_run({ scoreThreshold: 80 })`,
+
+  args: {
+    suiteFilter: tool.schema
+      .string()
+      .optional()
+      .describe(
+        'Optional filter to run specific eval suite (e.g., "coordinator", "compaction"). Matches against eval file paths using substring matching.'
+      ),
+    scoreThreshold: tool.schema
+      .number()
+      .optional()
+      .describe(
+        "Optional minimum average score threshold (0-100). If average score falls below this, result.success will be false. Useful for CI quality gates."
+      ),
+    includeDetailedResults: tool.schema
+      .boolean()
+      .optional()
+      .describe(
+        "Include individual eval results with input/output/scores in response. Set to false (default) for summary only to save token usage."
+      ),
+  },
+
+  execute: async (args) => {
+    const result = await runEvals({
+      cwd: process.cwd(),
+      suiteFilter: args.suiteFilter as string | undefined,
+      scoreThreshold: args.scoreThreshold as number | undefined,
+    });
+
+    // Remove detailed evals if not requested (saves tokens)
+    const includeDetails = args.includeDetailedResults === true;
+    if (!includeDetails) {
+      for (const suite of result.suites) {
+        delete suite.evals;
+      }
+    }
+
+    return JSON.stringify(result, null, 2);
+  },
+});
+
+/**
+ * All eval tools exported for registration
+ */
+export const evalTools = {
+  eval_run,
+} as const;