opencode-swarm-plugin 0.38.0 → 0.39.1

package/src/eval-gates.test.ts

@@ -0,0 +1,306 @@
+/**
+ * Tests for progressive eval gates
+ *
+ * TDD approach:
+ * RED: Tests written first, all failing
+ * GREEN: Minimal implementation to pass
+ * REFACTOR: Clean up while keeping tests green
+ */
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+import * as fs from "node:fs";
+import { checkGate } from "./eval-gates.js";
+import { recordEvalRun } from "./eval-history.js";
+
+const TEST_PROJECT = "/tmp/eval-gates-test";
+
+beforeEach(() => {
+	// Clean slate for each test
+	if (fs.existsSync(TEST_PROJECT)) {
+		fs.rmSync(TEST_PROJECT, { recursive: true });
+	}
+	fs.mkdirSync(TEST_PROJECT, { recursive: true });
+});
+
+afterEach(() => {
+	// Cleanup
+	if (fs.existsSync(TEST_PROJECT)) {
+		fs.rmSync(TEST_PROJECT, { recursive: true });
+	}
+});
+
+/**
+ * Helper to create run history
+ */
+function seedHistory(evalName: string, scores: number[]): void {
+	for (let i = 0; i < scores.length; i++) {
+		recordEvalRun(TEST_PROJECT, {
+			timestamp: new Date(Date.now() + i * 1000).toISOString(),
+			eval_name: evalName,
+			score: scores[i],
+			run_count: i + 1,
+		});
+	}
+}
+
+describe("checkGate - Bootstrap Phase (<10 runs)", () => {
+	test("always passes with 0 runs", () => {
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.5);
+
+		expect(result.passed).toBe(true);
+		expect(result.phase).toBe("bootstrap");
+		expect(result.message).toContain("Bootstrap phase");
+	});
+
+	test("always passes with 9 runs, even with score drop", () => {
+		seedHistory("my-eval", [0.9, 0.88, 0.87, 0.86, 0.85, 0.84, 0.83, 0.82, 0.81]);
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.5); // 50% drop
+
+		expect(result.passed).toBe(true);
+		expect(result.phase).toBe("bootstrap");
+		expect(result.message).toContain("Bootstrap phase");
+	});
+
+	test("provides run count in message", () => {
+		seedHistory("my-eval", [0.8, 0.8, 0.8]);
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.75);
+
+		expect(result.message).toContain("3/10");
+	});
+});
+
+describe("checkGate - Stabilization Phase (10-50 runs)", () => {
+	test("exactly 10 runs enters stabilization", () => {
+		seedHistory("my-eval", Array(10).fill(0.85));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.84);
+
+		expect(result.phase).toBe("stabilization");
+	});
+
+	test("passes with <10% regression", () => {
+		seedHistory("my-eval", Array(15).fill(0.9));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.82); // 8.8% drop
+
+		expect(result.passed).toBe(true);
+		expect(result.phase).toBe("stabilization");
+		expect(result.message).toContain("acceptable");
+	});
+
+	test("WARNS on >10% regression but still passes", () => {
+		seedHistory("my-eval", Array(15).fill(0.9));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.8); // 11.1% drop
+
+		expect(result.passed).toBe(true); // Still passes in stabilization
+		expect(result.phase).toBe("stabilization");
+		expect(result.message).toContain("regression");
+		expect(result.message).toMatch(/10%|11%/); // Should mention threshold
+	});
+
+	test("edge case: exactly 10% regression", () => {
+		seedHistory("my-eval", Array(20).fill(0.9));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.81); // Exactly 10% drop
+
+		expect(result.passed).toBe(true);
+		expect(result.phase).toBe("stabilization");
+	});
+
+	test("passes with score improvement", () => {
+		seedHistory("my-eval", Array(25).fill(0.8));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.95);
+
+		expect(result.passed).toBe(true);
+		expect(result.message).toContain("acceptable");
+	});
+
+	test("exactly 50 runs still in stabilization", () => {
+		seedHistory("my-eval", Array(50).fill(0.85));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.84);
+
+		expect(result.phase).toBe("stabilization");
+	});
+});
+
+describe("checkGate - Production Phase (>50 runs + variance <0.1)", () => {
+	test("enters production with 51 stable runs", () => {
+		seedHistory("my-eval", Array(51).fill(0.85));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.84);
+
+		expect(result.phase).toBe("production");
+	});
+
+	test("FAILS on >5% regression in production", () => {
+		seedHistory("my-eval", Array(60).fill(0.9));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.84); // 6.7% drop
+
+		expect(result.passed).toBe(false);
+		expect(result.phase).toBe("production");
+		expect(result.message).toContain("FAIL");
+		expect(result.message).toMatch(/5%|6%/);
+	});
+
+	test("passes with <5% regression in production", () => {
+		seedHistory("my-eval", Array(60).fill(0.9));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.86); // 4.4% drop
+
+		expect(result.passed).toBe(true);
+		expect(result.phase).toBe("production");
+		expect(result.message).toContain("acceptable");
+	});
+
+	test("edge case: exactly 5% regression", () => {
+		seedHistory("my-eval", Array(60).fill(0.9));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.855); // Exactly 5% drop
+
+		expect(result.passed).toBe(true);
+		expect(result.phase).toBe("production");
+	});
+
+	test("stays in stabilization if variance too high (>0.1) despite >50 runs", () => {
+		// Need significant wild variance to push above 0.1
+		// From memory: 60 stable + 50 alternating wild = variance ~0.103
+		const stableRuns = Array(60).fill(0.85);
+		const wildRuns = Array(50)
+			.fill(0)
+			.map((_, i) => (i % 2 === 0 ? 0.1 : 0.9));
+		seedHistory("my-eval", [...stableRuns, ...wildRuns]);
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.84);
+
+		expect(result.phase).toBe("stabilization");
+		expect(result.message).toContain("variance");
+	});
+
+	test("passes with score improvement in production", () => {
+		seedHistory("my-eval", Array(60).fill(0.8));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.95);
+
+		expect(result.passed).toBe(true);
+		expect(result.phase).toBe("production");
+	});
+});
+
+describe("checkGate - Baseline Calculation", () => {
+	test("uses mean of all historical scores as baseline", () => {
+		// Need 10+ runs to exit bootstrap and see baseline in message
+		seedHistory("my-eval", [0.8, 0.85, 0.9, 0.95, 1.0, 0.9, 0.9, 0.9, 0.9, 0.9]); // mean = 0.9
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.88);
+
+		// 0.88 is ~2.2% drop from 0.9 mean (within stabilization tolerance)
+		expect(result.passed).toBe(true);
+		expect(result.message).toContain("0.90"); // Should show baseline
+	});
+
+	test("handles different eval names independently", () => {
+		seedHistory("eval-a", Array(15).fill(0.9));
+		seedHistory("eval-b", Array(15).fill(0.5));
+
+		const resultA = checkGate(TEST_PROJECT, "eval-a", 0.88);
+		const resultB = checkGate(TEST_PROJECT, "eval-b", 0.48);
+
+		expect(resultA.passed).toBe(true);
+		expect(resultB.passed).toBe(true);
+	});
+});
+
+describe("checkGate - Edge Cases", () => {
+	test("handles score of 0", () => {
+		seedHistory("my-eval", Array(15).fill(0.8));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0);
+
+		expect(result.passed).toBe(true); // Still passes in stabilization with warning
+		expect(result.message).toContain("regression");
+	});
+
+	test("handles perfect score of 1.0", () => {
+		seedHistory("my-eval", Array(15).fill(0.9));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 1.0);
+
+		expect(result.passed).toBe(true);
+	});
+
+	test("handles no history file (first run)", () => {
+		// No seedHistory call - empty project
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.75);
+
+		expect(result.passed).toBe(true);
+		expect(result.phase).toBe("bootstrap");
+	});
+
+	test("handles baseline of 0 (avoid division by zero)", () => {
+		seedHistory("my-eval", Array(15).fill(0));
+
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.5);
+
+		expect(result.passed).toBe(true);
+		expect(result.message).not.toContain("NaN");
+		expect(result.message).not.toContain("Infinity");
+	});
+});
+
+describe("checkGate - Configurable Thresholds", () => {
+	test("accepts custom stabilization threshold", () => {
+		seedHistory("my-eval", Array(15).fill(0.9));
+
+		// 15% regression with custom 20% threshold - should pass
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.765, {
+			stabilizationThreshold: 0.2, // 20% instead of default 10%
+		});
+
+		expect(result.passed).toBe(true);
+		expect(result.phase).toBe("stabilization");
+		expect(result.message).toContain("acceptable");
+	});
+
+	test("accepts custom production threshold", () => {
+		seedHistory("my-eval", Array(60).fill(0.9));
+
+		// 7% regression with custom 10% threshold - should pass
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.837, {
+			productionThreshold: 0.1, // 10% instead of default 5%
+		});
+
+		expect(result.passed).toBe(true);
+		expect(result.phase).toBe("production");
+	});
+
+	test("custom threshold makes test fail when exceeded", () => {
+		seedHistory("my-eval", Array(60).fill(0.9));
+
+		// 7% regression with custom 3% threshold - should fail
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.837, {
+			productionThreshold: 0.03, // 3% instead of default 5%
+		});
+
+		expect(result.passed).toBe(false);
+		expect(result.phase).toBe("production");
+		expect(result.message).toContain("FAIL");
+	});
+
+	test("partial config uses defaults for unspecified thresholds", () => {
+		seedHistory("my-eval", Array(15).fill(0.9));
+
+		// Only override production threshold
+		const result = checkGate(TEST_PROJECT, "my-eval", 0.88, {
+			productionThreshold: 0.01,
+			// stabilizationThreshold not specified - uses default 0.1
+		});
+
+		expect(result.passed).toBe(true); // 2.2% regression < 10% stabilization default
+	});
+});

package/src/eval-gates.ts

@@ -0,0 +1,218 @@
+/**
+ * Progressive Eval Gates
+ *
+ * Enforces quality gates based on eval history and current phase:
+ * - Bootstrap (<10 runs): Always pass, collect data
+ * - Stabilization (10-50 runs): Pass but warn on >10% regression
+ * - Production (>50 runs + variance <0.1): Fail on >5% regression
+ *
+ * @module eval-gates
+ */
+import { calculateVariance, getPhase, getScoreHistory } from "./eval-history.js";
+
+/**
+ * Result from a gate check
+ */
+export interface GateResult {
+	/** Whether the gate passed */
+	passed: boolean;
+	/** Current phase */
+	phase: "bootstrap" | "stabilization" | "production";
+	/** Human-readable message */
+	message: string;
+	/** Baseline score (mean of history) */
+	baseline?: number;
+	/** Current score */
+	currentScore: number;
+	/** Regression percentage (negative = improvement) */
+	regressionPercent?: number;
+}
+
+/**
+ * Configuration for gate thresholds
+ */
+export interface GateConfig {
+	/** Regression threshold for stabilization phase (default: 0.1 = 10%) */
+	stabilizationThreshold?: number;
+	/** Regression threshold for production phase (default: 0.05 = 5%) */
+	productionThreshold?: number;
+}
+
+/**
+ * Default regression thresholds by phase
+ */
+export const DEFAULT_THRESHOLDS = {
+	stabilization: 0.1, // 10% regression warning
+	production: 0.05, // 5% regression failure
+} as const;
+
+/**
+ * Calculate baseline score (mean of all historical scores)
+ */
+function calculateBaseline(history: ReturnType<typeof getScoreHistory>, currentScore: number): number {
+	if (history.length === 0) {
+		return currentScore;
+	}
+	return history.reduce((sum, run) => sum + run.score, 0) / history.length;
+}
+
+/**
+ * Calculate regression percentage from baseline
+ *
+ * Returns 0 if baseline is 0 to avoid division by zero.
+ * Positive = regression (score dropped), negative = improvement
+ */
+function calculateRegression(baseline: number, currentScore: number): number {
+	if (baseline === 0) {
+		return 0;
+	}
+	return (baseline - currentScore) / baseline;
+}
+
+/**
+ * Format regression message with scores
+ */
+function formatRegressionMessage(
+	regressionPercent: number,
+	baseline: number,
+	currentScore: number,
+): string {
+	return `${(regressionPercent * 100).toFixed(1)}% regression (baseline: ${baseline.toFixed(2)}, current: ${currentScore.toFixed(2)})`;
+}
+
+/**
+ * Check if the current eval score passes the quality gate
+ *
+ * Progressive gates adapt based on data maturity:
+ * - **Bootstrap (<10 runs)**: Always pass, focus on collecting baseline data
+ * - **Stabilization (10-50 runs)**: Warn on >10% regression (default), but pass
+ * - **Production (>50 runs + variance <0.1)**: Fail on >5% regression (default)
+ *
+ * **Baseline calculation**: Mean of all historical scores for this eval (not just last run).
+ *
+ * **Regression formula**: `(baseline - current) / baseline`
+ * - Positive = regression (score dropped)
+ * - Negative = improvement
+ * - Returns 0 if baseline is 0 (avoids division by zero)
+ *
+ * **Variance threshold (0.1)**: High variance keeps eval in stabilization phase even with >50 runs.
+ * This prevents premature production gates when scores are still unstable.
+ *
+ * **CI Integration**: Production gates can fail PRs. Use `swarm eval status` to check phase before merging.
+ *
+ * @param projectPath - Absolute path to project root (contains `.opencode/eval-history.jsonl`)
+ * @param evalName - Name of the eval (e.g., "swarm-decomposition", "coordinator-behavior")
+ * @param currentScore - Current score to check (typically 0-1 range)
+ * @param config - Optional threshold configuration (defaults: stabilization=0.1, production=0.05)
+ * @returns Gate check result with pass/fail, phase, baseline, regression details
+ *
+ * @example
+ * ```typescript
+ * import { checkGate } from "./eval-gates.js";
+ *
+ * const result = checkGate("/path/to/project", "swarm-decomposition", 0.89);
+ *
+ * if (!result.passed) {
+ *   console.error(`❌ Gate FAILED: ${result.message}`);
+ *   process.exit(1); // Fail CI
+ * }
+ *
+ * console.log(`✅ ${result.phase} phase: ${result.message}`);
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Custom thresholds for sensitive eval
+ * const result = checkGate("/path", "critical-eval", 0.92, {
+ *   stabilizationThreshold: 0.05,  // 5% threshold in stabilization
+ *   productionThreshold: 0.02,     // 2% threshold in production
+ * });
+ * ```
+ */
+export function checkGate(
+	projectPath: string,
+	evalName: string,
+	currentScore: number,
+	config?: GateConfig,
+): GateResult {
+	const thresholds = {
+		stabilization: config?.stabilizationThreshold ?? DEFAULT_THRESHOLDS.stabilization,
+		production: config?.productionThreshold ?? DEFAULT_THRESHOLDS.production,
+	};
+
+	const phase = getPhase(projectPath, evalName);
+	const history = getScoreHistory(projectPath, evalName);
+
+	// Bootstrap phase - always pass
+	if (phase === "bootstrap") {
+		return {
+			passed: true,
+			phase: "bootstrap",
+			message: `Bootstrap phase (${history.length}/10 runs) - collecting data`,
+			currentScore,
+		};
+	}
+
+	// Calculate baseline and regression
+	const baseline = calculateBaseline(history, currentScore);
+	const regressionPercent = calculateRegression(baseline, currentScore);
+	const regressionMsg = formatRegressionMessage(regressionPercent, baseline, currentScore);
+
+	// Stabilization phase - warn on regression but pass
+	if (phase === "stabilization") {
+		if (regressionPercent > thresholds.stabilization) {
+			return {
+				passed: true,
+				phase: "stabilization",
+				message: `Stabilization phase: ${regressionMsg} - exceeds ${(thresholds.stabilization * 100).toFixed(0)}% threshold but still passing`,
+				baseline,
+				currentScore,
+				regressionPercent,
+			};
+		}
+
+		// Check if we have high variance (>50 runs but can't enter production)
+		if (history.length > 50) {
+			const scores = history.map((run) => run.score);
+			const variance = calculateVariance(scores);
+			return {
+				passed: true,
+				phase: "stabilization",
+				message: `Stabilization phase: ${regressionMsg} - acceptable. High variance (${variance.toFixed(3)}) prevents production phase.`,
+				baseline,
+				currentScore,
+				regressionPercent,
+			};
+		}
+
+		return {
+			passed: true,
+			phase: "stabilization",
+			message: `Stabilization phase: ${regressionMsg} - acceptable`,
+			baseline,
+			currentScore,
+			regressionPercent,
+		};
+	}
+
+	// Production phase - fail on regression exceeding threshold
+	if (regressionPercent > thresholds.production) {
+		return {
+			passed: false,
+			phase: "production",
+			message: `Production phase FAIL: ${regressionMsg} - exceeds ${(thresholds.production * 100).toFixed(0)}% threshold`,
+			baseline,
+			currentScore,
+			regressionPercent,
+		};
+	}
+
+	return {
+		passed: true,
+		phase: "production",
+		message: `Production phase: ${regressionMsg} - acceptable`,
+		baseline,
+		currentScore,
+		regressionPercent,
+	};
+}