opencode-swarm-plugin 0.37.0 → 0.39.1

package/src/eval-history.ts

@@ -0,0 +1,214 @@
+/**
+ * Eval History Tracker - Progressive gates based on run history
+ *
+ * Tracks eval run scores over time and calculates the current phase:
+ * - Bootstrap (<10 runs): No gates, just collect data
+ * - Stabilization (10-50 runs): Warn on >10% regression
+ * - Production (>50 runs + variance <0.1): Fail on >5% regression
+ *
+ * @module eval-history
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+
+/**
+ * Progressive phases based on run count and variance
+ */
+export type Phase = "bootstrap" | "stabilization" | "production";
+
+/**
+ * Single eval run record
+ */
+export interface EvalRunRecord {
+  /** ISO-8601 timestamp */
+  timestamp: string;
+  /** Name of the eval (e.g., "swarm-decomposition") */
+  eval_name: string;
+  /** Score (0-1 range typically) */
+  score: number;
+  /** Run count (monotonically increasing per eval) */
+  run_count: number;
+}
+
+/**
+ * Default path for eval history
+ */
+export const DEFAULT_EVAL_HISTORY_PATH = ".opencode/eval-history.jsonl";
+
+/**
+ * Variance threshold for production phase
+ */
+export const VARIANCE_THRESHOLD = 0.1;
+
+/**
+ * Run count thresholds for phase transitions
+ */
+export const BOOTSTRAP_THRESHOLD = 10;
+export const STABILIZATION_THRESHOLD = 50;
+
+/**
+ * Get the eval history file path
+ */
+export function getEvalHistoryPath(projectPath: string): string {
+  return path.join(projectPath, DEFAULT_EVAL_HISTORY_PATH);
+}
+
+/**
+ * Ensure the eval history directory exists
+ */
+export function ensureEvalHistoryDir(projectPath: string): void {
+  const historyPath = getEvalHistoryPath(projectPath);
+  const dir = path.dirname(historyPath);
+  if (!fs.existsSync(dir)) {
+    fs.mkdirSync(dir, { recursive: true });
+  }
+}
+
+/**
+ * Record an eval run to JSONL history
+ *
+ * Appends atomically to `.opencode/eval-history.jsonl`. Each line is a complete JSON object
+ * representing one eval run (timestamp, eval name, score, run count).
+ *
+ * **Auto-creates directory** if `.opencode/` doesn't exist.
+ *
+ * **Thread-safe**: Uses `appendFileSync` for atomic writes (safe for concurrent eval runs).
+ *
+ * **Integration**: Called automatically by evalite runner after each eval completes.
+ * Also callable manually for custom eval tracking.
+ *
+ * @param projectPath - Absolute path to project root
+ * @param run - Eval run record with timestamp, eval_name, score, run_count
+ *
+ * @example
+ * ```typescript
+ * import { recordEvalRun } from "./eval-history.js";
+ *
+ * recordEvalRun("/path/to/project", {
+ *   timestamp: new Date().toISOString(),
+ *   eval_name: "swarm-decomposition",
+ *   score: 0.92,
+ *   run_count: 15,
+ * });
+ * ```
+ */
+export function recordEvalRun(
+  projectPath: string,
+  run: EvalRunRecord,
+): void {
+  ensureEvalHistoryDir(projectPath);
+  const historyPath = getEvalHistoryPath(projectPath);
+  const line = `${JSON.stringify(run)}\n`;
+  fs.appendFileSync(historyPath, line, "utf-8");
+}
+
+/**
+ * Read all eval run records from JSONL file
+ *
+ * Internal helper for parsing the history file
+ */
+function readAllRecords(projectPath: string): EvalRunRecord[] {
+  const historyPath = getEvalHistoryPath(projectPath);
+
+  if (!fs.existsSync(historyPath)) {
+    return [];
+  }
+
+  const content = fs.readFileSync(historyPath, "utf-8");
+  const lines = content.trim().split("\n").filter(Boolean);
+
+  return lines.map((line) => JSON.parse(line) as EvalRunRecord);
+}
+
+/**
+ * Get score history for a specific eval
+ *
+ * Returns runs in chronological order (oldest first)
+ */
+export function getScoreHistory(
+  projectPath: string,
+  evalName: string,
+): EvalRunRecord[] {
+  return readAllRecords(projectPath).filter(
+    (run) => run.eval_name === evalName,
+  );
+}
+
+/**
+ * Calculate statistical variance of scores
+ *
+ * Variance = mean of squared deviations from the mean
+ * Formula: Σ((x - μ)²) / n
+ */
+export function calculateVariance(scores: number[]): number {
+  if (scores.length <= 1) {
+    return 0;
+  }
+
+  const mean = scores.reduce((sum, score) => sum + score, 0) / scores.length;
+
+  const variance = scores.reduce((sum, score) => {
+    const deviation = score - mean;
+    return sum + deviation * deviation;
+  }, 0) / scores.length;
+
+  return variance;
+}
+
+/**
+ * Get the current phase for an eval based on run count and score variance
+ *
+ * Progressive phase logic ensures quality gates adapt to data maturity:
+ *
+ * - **Bootstrap (<10 runs)**: No gates, just collect baseline data
+ * - **Stabilization (10-50 runs)**: Warn on >10% regression (but pass)
+ * - **Production (>50 runs AND variance <0.1)**: Fail on >5% regression
+ *
+ * **Variance check**: If >50 runs but variance ≥0.1, stays in stabilization.
+ * This prevents premature production gates when scores are still unstable.
+ *
+ * **Why variance matters**: An eval with wildly fluctuating scores isn't ready for
+ * strict gates. Variance threshold (0.1) ensures the eval is consistent before
+ * enforcing production-level quality control.
+ *
+ * @param projectPath - Absolute path to project root (contains `.opencode/eval-history.jsonl`)
+ * @param evalName - Name of the eval (e.g., "swarm-decomposition")
+ * @returns Current phase: "bootstrap" | "stabilization" | "production"
+ *
+ * @example
+ * ```typescript
+ * import { getPhase } from "./eval-history.js";
+ *
+ * const phase = getPhase("/path/to/project", "swarm-decomposition");
+ *
+ * if (phase === "production") {
+ *   console.log("🚀 Production phase - strict gates enabled");
+ * } else if (phase === "stabilization") {
+ *   console.log("⚙️ Stabilization phase - warnings only");
+ * } else {
+ *   console.log("🌱 Bootstrap phase - collecting data");
+ * }
+ * ```
+ */
+export function getPhase(projectPath: string, evalName: string): Phase {
+  const history = getScoreHistory(projectPath, evalName);
+
+  if (history.length < BOOTSTRAP_THRESHOLD) {
+    return "bootstrap";
+  }
+
+  if (history.length <= STABILIZATION_THRESHOLD) {
+    return "stabilization";
+  }
+
+  // >50 runs - check variance
+  const scores = history.map((run) => run.score);
+  const variance = calculateVariance(scores);
+
+  if (variance < VARIANCE_THRESHOLD) {
+    return "production";
+  }
+
+  // High variance - stay in stabilization
+  return "stabilization";
+}

package/src/eval-learning.test.ts

@@ -0,0 +1,378 @@
+/**
+ * Tests for eval-learning.ts - Eval-to-Learning Feedback Loop
+ *
+ * TDD RED phase: Write failing tests first, then implement.
+ *
+ * Core behavior:
+ * - Detect significant eval score drops (>15% from rolling average)
+ * - Store failure context to semantic-memory with structured tags
+ * - Ignore minor fluctuations (<15% variance)
+ * - Configurable threshold for sensitivity tuning
+ */
+import { describe, test, expect, beforeEach, mock } from "bun:test";
+import {
+	learnFromEvalFailure,
+	type EvalLearningConfig,
+	calculateRollingAverage,
+	isSignificantDrop,
+	formatFailureContext,
+	createLearningConfig,
+	DEFAULT_EVAL_LEARNING_CONFIG,
+} from "./eval-learning";
+import type { EvalRunRecord } from "./eval-history";
+import type { MemoryAdapter } from "./memory-tools";
+
+// ============================================================================
+// Mock Memory Adapter
+// ============================================================================
+
+/**
+ * Create a mock memory adapter for testing
+ *
+ * Tracks store() calls without hitting real storage
+ */
+function createMockMemoryAdapter(): MemoryAdapter {
+	const storedMemories: Array<{
+		information: string;
+		tags?: string;
+		metadata?: string;
+	}> = [];
+
+	return {
+		store: mock(async (args) => {
+			storedMemories.push(args);
+			return {
+				id: `mem_${Date.now()}`,
+				message: "Stored successfully",
+			};
+		}),
+		find: mock(async () => ({ results: [], total: 0 })),
+		get: mock(async () => null),
+		remove: mock(async () => ({ success: true, message: "Removed" })),
+		validate: mock(async () => ({ success: true, message: "Validated" })),
+		list: mock(async () => []),
+		stats: mock(async () => ({
+			total_memories: 0,
+			total_embeddings: 0,
+			collections: {},
+		})),
+		checkHealth: mock(async () => ({ ready: true, message: "OK" })),
+		getStoredMemories: () => storedMemories,
+	} as any;
+}
+
+// ============================================================================
+// Tests: Rolling Average Calculation
+// ============================================================================
+
+describe("calculateRollingAverage", () => {
+	test("returns 0 for empty history", () => {
+		const avg = calculateRollingAverage([]);
+		expect(avg).toBe(0);
+	});
+
+	test("returns single score for history of 1", () => {
+		const history: EvalRunRecord[] = [
+			{
+				eval_name: "test",
+				score: 0.85,
+				timestamp: "2024-12-01T00:00:00Z",
+				run_count: 1,
+			},
+		];
+
+		const avg = calculateRollingAverage(history);
+		expect(avg).toBe(0.85);
+	});
+
+	test("calculates average of last N runs (default 5)", () => {
+		const history: EvalRunRecord[] = [
+			{ eval_name: "test", score: 0.8, timestamp: "2024-12-01", run_count: 1 },
+			{ eval_name: "test", score: 0.82, timestamp: "2024-12-02", run_count: 2 },
+			{ eval_name: "test", score: 0.84, timestamp: "2024-12-03", run_count: 3 },
+			{ eval_name: "test", score: 0.86, timestamp: "2024-12-04", run_count: 4 },
+			{ eval_name: "test", score: 0.88, timestamp: "2024-12-05", run_count: 5 },
+			{ eval_name: "test", score: 0.9, timestamp: "2024-12-06", run_count: 6 },
+		];
+
+		const avg = calculateRollingAverage(history);
+		// Last 5: 0.82, 0.84, 0.86, 0.88, 0.9 => avg = 0.86
+		expect(avg).toBeCloseTo(0.86, 2);
+	});
+
+	test("uses custom window size", () => {
+		const history: EvalRunRecord[] = [
+			{ eval_name: "test", score: 0.8, timestamp: "2024-12-01", run_count: 1 },
+			{ eval_name: "test", score: 0.85, timestamp: "2024-12-02", run_count: 2 },
+			{ eval_name: "test", score: 0.9, timestamp: "2024-12-03", run_count: 3 },
+		];
+
+		const avg = calculateRollingAverage(history, 2);
+		// Last 2: 0.85, 0.9 => avg = 0.875
+		expect(avg).toBeCloseTo(0.875, 3);
+	});
+
+	test("handles window larger than history", () => {
+		const history: EvalRunRecord[] = [
+			{ eval_name: "test", score: 0.8, timestamp: "2024-12-01", run_count: 1 },
+			{ eval_name: "test", score: 0.9, timestamp: "2024-12-02", run_count: 2 },
+		];
+
+		const avg = calculateRollingAverage(history, 10);
+		// Uses all available: (0.8 + 0.9) / 2 = 0.85
+		expect(avg).toBeCloseTo(0.85, 2);
+	});
+});
+
+// ============================================================================
+// Tests: Significant Drop Detection
+// ============================================================================
+
+describe("isSignificantDrop", () => {
+	test("returns false when current equals baseline", () => {
+		expect(isSignificantDrop(0.85, 0.85)).toBe(false);
+	});
+
+	test("returns false when current is higher than baseline", () => {
+		expect(isSignificantDrop(0.9, 0.85)).toBe(false);
+	});
+
+	test("returns false for drop below threshold (default 15%)", () => {
+		// Drop of 10%: 0.85 -> 0.765 (90% of 0.85)
+		expect(isSignificantDrop(0.765, 0.85)).toBe(false);
+	});
+
+	test("returns true for drop at threshold (15%)", () => {
+		// Drop of exactly 15%: 0.85 -> 0.7225 (85% of 0.85)
+		// Use slightly lower to account for floating point precision
+		expect(isSignificantDrop(0.722, 0.85)).toBe(true);
+	});
+
+	test("returns true for drop above threshold (20%)", () => {
+		// Drop of 20%: 0.85 -> 0.68 (80% of 0.85)
+		expect(isSignificantDrop(0.68, 0.85)).toBe(true);
+	});
+
+	test("uses custom threshold", () => {
+		// Drop of 8%: 0.85 -> 0.782 (92% of 0.85)
+		// Default (15%) => false
+		expect(isSignificantDrop(0.782, 0.85)).toBe(false);
+
+		// Custom threshold (5%) => true
+		expect(isSignificantDrop(0.782, 0.85, 0.05)).toBe(true);
+	});
+
+	test("returns false when baseline is 0 (avoid division by zero)", () => {
+		expect(isSignificantDrop(0, 0)).toBe(false);
+		expect(isSignificantDrop(0.5, 0)).toBe(false);
+	});
+});
+
+// ============================================================================
+// Tests: Failure Context Formatting
+// ============================================================================
+
+describe("formatFailureContext", () => {
+	test("includes eval name, scores, and drop percentage", () => {
+		const context = formatFailureContext("compaction-test", 0.68, 0.85);
+
+		expect(context).toContain("compaction-test");
+		expect(context).toContain("0.68");
+		expect(context).toContain("0.85");
+		expect(context).toContain("20.0%"); // (0.85 - 0.68) / 0.85 = 20%
+	});
+
+	test("includes optional scorer context", () => {
+		const scorerContext = "violationCount scorer failed: 5 violations detected";
+		const context = formatFailureContext(
+			"coordinator-behavior",
+			0.5,
+			0.8,
+			scorerContext,
+		);
+
+		expect(context).toContain("coordinator-behavior");
+		expect(context).toContain(scorerContext);
+	});
+
+	test("handles baseline of 0 gracefully", () => {
+		const context = formatFailureContext("test", 0.5, 0);
+		expect(context).not.toContain("NaN");
+		expect(context).not.toContain("Infinity");
+	});
+});
+
+// ============================================================================
+// Tests: Main learnFromEvalFailure Function
+// ============================================================================
+
+describe("learnFromEvalFailure", () => {
+	let mockAdapter: MemoryAdapter;
+
+	beforeEach(() => {
+		mockAdapter = createMockMemoryAdapter();
+	});
+
+	test("stores memory when score drops significantly", async () => {
+		const history: EvalRunRecord[] = [
+			{ eval_name: "test", score: 0.85, timestamp: "2024-12-01", run_count: 1 },
+			{ eval_name: "test", score: 0.84, timestamp: "2024-12-02", run_count: 2 },
+			{ eval_name: "test", score: 0.86, timestamp: "2024-12-03", run_count: 3 },
+			{ eval_name: "test", score: 0.85, timestamp: "2024-12-04", run_count: 4 },
+			{ eval_name: "test", score: 0.84, timestamp: "2024-12-05", run_count: 5 },
+		];
+		const currentScore = 0.68; // Drop of ~20%
+
+		const result = await learnFromEvalFailure(
+			"test-eval",
+			currentScore,
+			history,
+			mockAdapter,
+		);
+
+		expect(result.triggered).toBe(true);
+		expect(result.baseline).toBeCloseTo(0.848, 2);
+		expect(result.drop_percentage).toBeCloseTo(0.198, 2); // ~20%
+
+		// Verify memory was stored
+		expect(mockAdapter.store).toHaveBeenCalledTimes(1);
+
+		const storedMemory = (mockAdapter as any).getStoredMemories()[0];
+		expect(storedMemory.information).toContain("test-eval");
+		expect(storedMemory.information).toContain("0.68");
+		expect(storedMemory.tags).toContain("eval-failure");
+		expect(storedMemory.tags).toContain("test-eval");
+	});
+
+	test("does not store memory for minor fluctuations", async () => {
+		const history: EvalRunRecord[] = [
+			{ eval_name: "test", score: 0.85, timestamp: "2024-12-01", run_count: 1 },
+			{ eval_name: "test", score: 0.84, timestamp: "2024-12-02", run_count: 2 },
+		];
+		const currentScore = 0.8; // Drop of ~5%, below 15% threshold
+
+		const result = await learnFromEvalFailure(
+			"test-eval",
+			currentScore,
+			history,
+			mockAdapter,
+		);
+
+		expect(result.triggered).toBe(false);
+		expect(mockAdapter.store).not.toHaveBeenCalled();
+	});
+
+	test("includes scorer context in memory if provided", async () => {
+		const history: EvalRunRecord[] = [
+			{ eval_name: "test", score: 0.9, timestamp: "2024-12-01", run_count: 1 },
+		];
+		const currentScore = 0.7; // Drop of ~22%
+		const scorerContext = "violationCount: 8 protocol violations";
+
+		await learnFromEvalFailure(
+			"coordinator-behavior",
+			currentScore,
+			history,
+			mockAdapter,
+			{ scorerContext },
+		);
+
+		const storedMemory = (mockAdapter as any).getStoredMemories()[0];
+		expect(storedMemory.information).toContain(scorerContext);
+	});
+
+	test("uses custom threshold when provided", async () => {
+		const history: EvalRunRecord[] = [
+			{ eval_name: "test", score: 0.9, timestamp: "2024-12-01", run_count: 1 },
+		];
+		const currentScore = 0.85; // Drop of ~5.5%
+
+		const customConfig: EvalLearningConfig = {
+			...DEFAULT_EVAL_LEARNING_CONFIG,
+			dropThreshold: 0.05, // 5% threshold
+		};
+
+		const result = await learnFromEvalFailure(
+			"test-eval",
+			currentScore,
+			history,
+			mockAdapter,
+			{ config: customConfig },
+		);
+
+		expect(result.triggered).toBe(true);
+		expect(mockAdapter.store).toHaveBeenCalledTimes(1);
+	});
+
+	test("handles empty history gracefully", async () => {
+		const result = await learnFromEvalFailure(
+			"test-eval",
+			0.5,
+			[],
+			mockAdapter,
+		);
+
+		expect(result.triggered).toBe(false);
+		expect(result.baseline).toBe(0);
+		expect(mockAdapter.store).not.toHaveBeenCalled();
+	});
+
+	test("generates structured tags for semantic search", async () => {
+		const history: EvalRunRecord[] = [
+			{ eval_name: "test", score: 0.9, timestamp: "2024-12-01", run_count: 1 },
+		];
+		const currentScore = 0.7; // Significant drop
+
+		await learnFromEvalFailure(
+			"compaction-test",
+			currentScore,
+			history,
+			mockAdapter,
+		);
+
+		const storedMemory = (mockAdapter as any).getStoredMemories()[0];
+		const tags = storedMemory.tags;
+
+		expect(tags).toContain("eval-failure");
+		expect(tags).toContain("compaction-test");
+		expect(tags).toContain("regression");
+	});
+
+	test("stores metadata for future prompt generation", async () => {
+		const history: EvalRunRecord[] = [
+			{ eval_name: "test", score: 0.9, timestamp: "2024-12-01", run_count: 1 },
+		];
+		const currentScore = 0.7;
+
+		await learnFromEvalFailure("test-eval", currentScore, history, mockAdapter);
+
+		const storedMemory = (mockAdapter as any).getStoredMemories()[0];
+		expect(storedMemory.metadata).toBeDefined();
+
+		const metadata = JSON.parse(storedMemory.metadata!);
+		expect(metadata.eval_name).toBe("test-eval");
+		expect(metadata.baseline_score).toBeCloseTo(0.9, 2);
+		expect(metadata.current_score).toBe(0.7);
+		expect(metadata.drop_percentage).toBeCloseTo(0.222, 2);
+	});
+});
+
+// ============================================================================
+// Tests: Convenience Helpers
+// ============================================================================
+
+describe("createLearningConfig", () => {
+	test("creates config with custom threshold", () => {
+		const config = createLearningConfig(0.1);
+
+		expect(config.dropThreshold).toBe(0.1);
+		expect(config.windowSize).toBe(DEFAULT_EVAL_LEARNING_CONFIG.windowSize);
+	});
+
+	test("accepts custom window size", () => {
+		const config = createLearningConfig(0.2, 10);
+
+		expect(config.dropThreshold).toBe(0.2);
+		expect(config.windowSize).toBe(10);
+	});
+});