opencode-swarm-plugin 0.43.0 → 0.44.1

package/src/eval-learning.ts

@@ -1,360 +0,0 @@
-/**
- * Eval-to-Learning Feedback Loop
- *
- * Automatically stores eval failures to semantic memory for learning.
- * When eval scores drop significantly from rolling average (default >15%),
- * stores context to semantic-memory with tags for future prompt generation.
- *
- * ## Usage
- *
- * ```typescript
- * import { learnFromEvalFailure } from "./eval-learning";
- * import { getMemoryAdapter } from "./memory-tools";
- * import { getScoreHistory } from "./eval-history";
- *
- * const memoryAdapter = await getMemoryAdapter();
- * const history = getScoreHistory(projectPath, "compaction-test");
- *
- * const result = await learnFromEvalFailure(
- *   "compaction-test",
- *   currentScore,
- *   history,
- *   memoryAdapter
- * );
- *
- * if (result.triggered) {
- *   console.log(`📉 Regression detected: ${(result.drop_percentage * 100).toFixed(1)}% drop`);
- *   console.log(`Memory ID: ${result.memory_id}`);
- * }
- * ```
- *
- * ## Integration Points
- *
- * - **After each eval run**: Call to detect regressions automatically
- * - **Memory tags**: `eval-failure`, `{eval-name}`, `regression`
- * - **Future prompts**: Query memories with these tags for context
- * - **Scorer context**: Optional detail about which scorer failed
- *
- * ## Customization
- *
- * ```typescript
- * const customConfig = {
- *   dropThreshold: 0.10,  // 10% threshold (more sensitive)
- *   windowSize: 10,        // Last 10 runs for baseline
- * };
- *
- * await learnFromEvalFailure(
- *   "test",
- *   score,
- *   history,
- *   adapter,
- *   { config: customConfig }
- * );
- * ```
- *
- * @module eval-learning
- */
-
-import type { EvalRunRecord } from "./eval-history";
-import type { MemoryAdapter } from "./memory-tools";
-
-// ============================================================================
-// Configuration
-// ============================================================================
-
-/**
- * Configuration for eval-to-learning feedback
- */
-export interface EvalLearningConfig {
-	/** Threshold for significant drop (0-1, default 0.15 = 15%) */
-	dropThreshold: number;
-	/** Rolling average window size (default 5 runs) */
-	windowSize: number;
-}
-
-/**
- * Default configuration
- */
-export const DEFAULT_EVAL_LEARNING_CONFIG: EvalLearningConfig = {
-	dropThreshold: 0.15, // 15% drop triggers storage
-	windowSize: 5, // Last 5 runs for baseline
-};
-
-// ============================================================================
-// Result Types
-// ============================================================================
-
-/**
- * Result from learning check
- */
-export interface LearningResult {
-	/** Whether the check triggered memory storage */
-	triggered: boolean;
-	/** Baseline score from rolling average */
-	baseline: number;
-	/** Current score */
-	current: number;
-	/** Drop percentage (0-1, e.g., 0.20 = 20% drop) */
-	drop_percentage: number;
-	/** Memory ID if stored */
-	memory_id?: string;
-}
-
-// ============================================================================
-// Core Functions
-// ============================================================================
-
-/**
- * Calculate rolling average of recent scores
- *
- * Uses last N runs (default 5) to establish baseline.
- * If history shorter than window, uses all available.
- *
- * @param history - Score history (chronological order)
- * @param windowSize - Number of recent runs to average (default 5)
- * @returns Average score (0 if empty)
- */
-export function calculateRollingAverage(
-	history: EvalRunRecord[],
-	windowSize: number = 5,
-): number {
-	if (history.length === 0) {
-		return 0;
-	}
-
-	const recentRuns = history.slice(-windowSize);
-	const sum = recentRuns.reduce((acc, run) => acc + run.score, 0);
-	return sum / recentRuns.length;
-}
-
-/**
- * Check if current score is a significant drop from baseline
- *
- * Significant = drop exceeds threshold (default 15%).
- * Formula: (baseline - current) / baseline >= threshold
- *
- * @param currentScore - Current eval score
- * @param baseline - Baseline score (rolling average)
- * @param threshold - Drop threshold (default 0.15 = 15%)
- * @returns True if drop is significant
- */
-export function isSignificantDrop(
-	currentScore: number,
-	baseline: number,
-	threshold: number = 0.15,
-): boolean {
-	// Avoid division by zero
-	if (baseline === 0) {
-		return false;
-	}
-
-	const drop = (baseline - currentScore) / baseline;
-	return drop >= threshold;
-}
-
-/**
- * Format failure context for semantic memory storage
- *
- * Creates human-readable description of the failure with
- * quantified metrics and optional scorer context.
- *
- * @param evalName - Name of eval that failed
- * @param currentScore - Current score
- * @param baseline - Baseline score
- * @param scorerContext - Optional context about which scorer failed
- * @returns Formatted context string
- */
-export function formatFailureContext(
-	evalName: string,
-	currentScore: number,
-	baseline: number,
-	scorerContext?: string,
-): string {
-	const dropPercentage =
-		baseline > 0 ? ((baseline - currentScore) / baseline) * 100 : 0;
-
-	const lines = [
-		`Eval "${evalName}" regression detected:`,
-		`- Current score: ${currentScore.toFixed(2)}`,
-		`- Baseline (rolling avg): ${baseline.toFixed(2)}`,
-		`- Drop: ${dropPercentage.toFixed(1)}%`,
-	];
-
-	if (scorerContext) {
-		lines.push("", "Scorer context:", scorerContext);
-	}
-
-	lines.push(
-		"",
-		"Action: Review recent changes that may have caused regression.",
-		"Query this memory when generating future prompts for this eval.",
-	);
-
-	return lines.join("\n");
-}
-
-/**
- * Main learning function - automatically stores eval failures to semantic memory
- *
- * **Closed-loop learning**: When eval scores drop significantly from baseline,
- * this function stores failure context to semantic memory. Future prompt generation
- * queries these memories for context, preventing repeated mistakes.
- *
- * **Trigger condition**: Score drops >15% (default) from rolling average baseline.
- * Uses last 5 runs (default) to establish baseline, not just previous run.
- *
- * **What gets stored**:
- * - Eval name, baseline score, current score, drop percentage
- * - Scorer-specific context (which scorer failed, why)
- * - Timestamp and metadata for querying
- * - Tags: `eval-failure`, `{eval-name}`, `regression`
- *
- * **Future use**: Before generating prompts for the same eval, query semantic memory
- * with tags to inject learnings from past failures.
- *
- * **Integration points**:
- * - After each eval run (in evalite runner or CI)
- * - In `checkGate()` when regression detected
- * - Manual calls for custom eval tracking
- *
- * @param evalName - Name of eval (e.g., "compaction-test", "coordinator-behavior")
- * @param currentScore - Current eval score (typically 0-1 range)
- * @param history - Score history in chronological order (oldest first)
- * @param memoryAdapter - Semantic memory adapter (from `getMemoryAdapter()`)
- * @param options - Optional config (thresholds, window size) and scorer context
- * @param options.config - Custom thresholds (dropThreshold, windowSize)
- * @param options.scorerContext - Details about which scorer failed (for context)
- * @returns Learning result with trigger status, baseline, drop percentage, memory ID
- *
- * @example
- * ```typescript
- * import { learnFromEvalFailure } from "./eval-learning.js";
- * import { getMemoryAdapter } from "./memory-tools.js";
- * import { getScoreHistory } from "./eval-history.js";
- *
- * const memoryAdapter = await getMemoryAdapter();
- * const history = getScoreHistory("/path/to/project", "coordinator-behavior");
- *
- * const result = await learnFromEvalFailure(
- *   "coordinator-behavior",
- *   0.68,  // Current score
- *   history,
- *   memoryAdapter,
- *   { scorerContext: "violationCount: 5 violations (coordinator edited files)" }
- * );
- *
- * if (result.triggered) {
- *   console.log(`📉 Regression detected: ${(result.drop_percentage * 100).toFixed(1)}% drop`);
- *   console.log(`Stored to memory: ${result.memory_id}`);
- * }
- * ```
- *
- * @example
- * ```typescript
- * // Custom threshold (more sensitive)
- * const result = await learnFromEvalFailure(
- *   "critical-eval",
- *   0.85,
- *   history,
- *   memoryAdapter,
- *   {
- *     config: {
- *       dropThreshold: 0.10,  // 10% threshold (default is 15%)
- *       windowSize: 10,       // Last 10 runs for baseline (default is 5)
- *     },
- *   }
- * );
- * ```
- */
-export async function learnFromEvalFailure(
-	evalName: string,
-	currentScore: number,
-	history: EvalRunRecord[],
-	memoryAdapter: MemoryAdapter,
-	options?: {
-		config?: EvalLearningConfig;
-		scorerContext?: string;
-	},
-): Promise<LearningResult> {
-	const config = options?.config ?? DEFAULT_EVAL_LEARNING_CONFIG;
-
-	// Calculate baseline from rolling average
-	const baseline = calculateRollingAverage(history, config.windowSize);
-
-	// Check if this is a significant drop
-	const dropPercentage =
-		baseline > 0 ? (baseline - currentScore) / baseline : 0;
-	const significant = isSignificantDrop(
-		currentScore,
-		baseline,
-		config.dropThreshold,
-	);
-
-	const result: LearningResult = {
-		triggered: significant,
-		baseline,
-		current: currentScore,
-		drop_percentage: dropPercentage,
-	};
-
-	// Store to semantic memory if significant
-	if (significant) {
-		const information = formatFailureContext(
-			evalName,
-			currentScore,
-			baseline,
-			options?.scorerContext,
-		);
-
-		const tags = ["eval-failure", evalName, "regression"].join(",");
-
-		const metadata = JSON.stringify({
-			eval_name: evalName,
-			baseline_score: baseline,
-			current_score: currentScore,
-			drop_percentage: dropPercentage,
-			timestamp: new Date().toISOString(),
-		});
-
-		const storeResult = await memoryAdapter.store({
-			information,
-			tags,
-			metadata,
-		});
-
-		if (storeResult.id) {
-			result.memory_id = storeResult.id;
-		}
-	}
-
-	return result;
-}
-
-// ============================================================================
-// Convenience Exports
-// ============================================================================
-
-/**
- * Create custom learning config with specific threshold
- *
- * Helper for common use case: custom drop threshold.
- *
- * @param dropThreshold - Drop threshold (0-1)
- * @param windowSize - Optional window size (default 5)
- * @returns Custom config
- *
- * @example
- * ```typescript
- * const config = createLearningConfig(0.10); // 10% threshold
- * await learnFromEvalFailure("test", score, history, adapter, { config });
- * ```
- */
-export function createLearningConfig(
-	dropThreshold: number,
-	windowSize?: number,
-): EvalLearningConfig {
-	return {
-		dropThreshold,
-		windowSize: windowSize ?? DEFAULT_EVAL_LEARNING_CONFIG.windowSize,
-	};
-}

package/src/eval-runner.test.ts

@@ -1,223 +0,0 @@
-/**
- * Tests for eval-runner - Programmatic evalite execution
- *
- * TDD: These tests MUST fail initially, then pass after implementation.
- */
-
-import { describe, test, expect, beforeAll, afterEach } from "bun:test";
-import { runEvals } from "./eval-runner";
-import path from "node:path";
-import fs from "node:fs";
-import { getEvalHistoryPath } from "./eval-history";
-
-// Use project root for all tests
-const PROJECT_ROOT = path.resolve(import.meta.dir, "..");
-
-describe("runEvals", () => {
-  test("runs all evals when no suite filter provided", async () => {
-    const result = await runEvals({
-      cwd: PROJECT_ROOT,
-    });
-
-    // Even if some evals fail, we should get results
-    expect(typeof result.success).toBe("boolean");
-    expect(typeof result.totalSuites).toBe("number");
-    expect(typeof result.totalEvals).toBe("number");
-    expect(typeof result.averageScore).toBe("number");
-    expect(Array.isArray(result.suites)).toBe(true);
-
-    // Should have at least the example.eval.ts suite
-    expect(result.totalSuites).toBeGreaterThan(0);
-    expect(result.suites.length).toBeGreaterThan(0);
-  }, 60000); // 60s timeout for full eval run
-
-  test("filters evals by suite name", async () => {
-    const result = await runEvals({
-      cwd: PROJECT_ROOT,
-      suiteFilter: "example",
-    });
-
-    expect(result.success).toBe(true);
-    // All suite filepaths should contain "example"
-    for (const suite of result.suites) {
-      expect(suite.filepath.toLowerCase()).toContain("example");
-    }
-  }, 30000);
-
-  test("respects score threshold", async () => {
-    const result = await runEvals({
-      cwd: PROJECT_ROOT,
-      suiteFilter: "example", // Known good eval
-      scoreThreshold: 0, // Very low threshold, should pass
-    });
-
-    expect(result.success).toBe(true);
-    expect(result.averageScore).toBeGreaterThanOrEqual(0);
-  }, 30000);
-
-  test("returns structured suite results with scores", async () => {
-    const result = await runEvals({
-      cwd: PROJECT_ROOT,
-      suiteFilter: "example",
-    });
-
-    expect(result.suites.length).toBeGreaterThan(0);
-    
-    const suite = result.suites[0];
-    expect(suite).toMatchObject({
-      name: expect.any(String),
-      filepath: expect.any(String),
-      status: expect.stringMatching(/^(success|fail|running)$/),
-      duration: expect.any(Number),
-      averageScore: expect.any(Number),
-      evalCount: expect.any(Number),
-    });
-  }, 30000);
-
-  test("handles errors gracefully", async () => {
-    const result = await runEvals({
-      cwd: "/nonexistent/path",
-    });
-
-    expect(result.success).toBe(false);
-    expect(result.error).toBeDefined();
-    expect(result.suites).toEqual([]);
-  }, 10000);
-
-  test("returns empty results when no evals match filter", async () => {
-    const result = await runEvals({
-      cwd: PROJECT_ROOT,
-      suiteFilter: "nonexistent-eval-name-xyz",
-    });
-
-    // Should succeed but with no suites
-    expect(result.success).toBe(true);
-    expect(result.totalSuites).toBe(0);
-    expect(result.suites).toEqual([]);
-  }, 10000);
-
-  test("records eval run to history after execution", async () => {
-    // Clean up any existing history before test
-    const historyPath = getEvalHistoryPath(PROJECT_ROOT);
-    const historyBackup = historyPath + ".backup";
-    
-    // Backup existing history
-    if (fs.existsSync(historyPath)) {
-      fs.copyFileSync(historyPath, historyBackup);
-    }
-    
-    try {
-      // Remove history file to get clean state
-      if (fs.existsSync(historyPath)) {
-        fs.unlinkSync(historyPath);
-      }
-
-      // Run evals
-      const result = await runEvals({
-        cwd: PROJECT_ROOT,
-        suiteFilter: "example",
-      });
-
-      // Should have succeeded
-      expect(result.success).toBe(true);
-      expect(result.suites.length).toBeGreaterThan(0);
-
-      // History file should have been created
-      expect(fs.existsSync(historyPath)).toBe(true);
-
-      // Read history file
-      const historyContent = fs.readFileSync(historyPath, "utf-8");
-      const lines = historyContent.trim().split("\n");
-
-      // Should have one line per suite
-      expect(lines.length).toBe(result.suites.length);
-
-      // Parse first line and verify structure
-      const firstRecord = JSON.parse(lines[0]);
-      
-      // Verify structure has all required fields
-      expect(typeof firstRecord.timestamp).toBe("string");
-      expect(typeof firstRecord.eval_name).toBe("string");
-      expect(typeof firstRecord.score).toBe("number");
-      expect(typeof firstRecord.run_count).toBe("number");
-
-      // Verify eval_name matches suite name
-      expect(firstRecord.eval_name).toBe(result.suites[0].name);
-      
-      // Verify score matches suite averageScore
-      expect(firstRecord.score).toBe(result.suites[0].averageScore);
-      
-      // First run should have run_count = 1
-      expect(firstRecord.run_count).toBe(1);
-    } finally {
-      // Restore backup
-      if (fs.existsSync(historyBackup)) {
-        fs.copyFileSync(historyBackup, historyPath);
-        fs.unlinkSync(historyBackup);
-      }
-    }
-  }, 30000);
-
-  test("checks gates for each suite after recording", async () => {
-    const result = await runEvals({
-      cwd: PROJECT_ROOT,
-      suiteFilter: "example",
-    });
-
-    expect(result.success).toBe(true);
-    expect(result.gateResults).toBeDefined();
-    expect(Array.isArray(result.gateResults)).toBe(true);
-    
-    // Should have gate result for each suite
-    expect(result.gateResults?.length).toBe(result.suites.length);
-    
-    // Each gate result should have required fields
-    if (result.gateResults && result.gateResults.length > 0) {
-      const gateResult = result.gateResults[0];
-      expect(gateResult).toHaveProperty("suite");
-      expect(gateResult).toHaveProperty("passed");
-      expect(gateResult).toHaveProperty("phase");
-      expect(gateResult).toHaveProperty("message");
-      expect(gateResult).toHaveProperty("currentScore");
-    }
-  }, 30000);
-
-  test("calls learnFromEvalFailure when gate fails", async () => {
-    // This test requires manually creating a history with regression
-    // For now, we just verify the code path exists
-    // In practice, this would be tested with mocked checkGate returning failed=true
-    
-    const result = await runEvals({
-      cwd: PROJECT_ROOT,
-      suiteFilter: "example",
-    });
-
-    // Gate results should be present even if no failures
-    expect(result.gateResults).toBeDefined();
-  }, 30000);
-
-  test("does NOT call learnFromEvalFailure when gate passes", async () => {
-    // Similar to above - verifies the happy path
-    // Real test would mock checkGate and verify learnFromEvalFailure NOT called
-    
-    const result = await runEvals({
-      cwd: PROJECT_ROOT,
-      suiteFilter: "example",
-    });
-
-    // Should succeed with gate results
-    expect(result.success).toBe(true);
-    expect(result.gateResults).toBeDefined();
-  }, 30000);
-
-  test("includes gateResults in return value", async () => {
-    const result = await runEvals({
-      cwd: PROJECT_ROOT,
-      suiteFilter: "example",
-    });
-
-    // gateResults should be array (even if empty)
-    expect(result).toHaveProperty("gateResults");
-    expect(Array.isArray(result.gateResults)).toBe(true);
-  }, 30000);
-});