opencode-swarm-plugin 0.44.0 → 0.44.2

package/src/eval-gates.test.ts

@@ -1,306 +0,0 @@
-/**
- * Tests for progressive eval gates
- *
- * TDD approach:
- * RED: Tests written first, all failing
- * GREEN: Minimal implementation to pass
- * REFACTOR: Clean up while keeping tests green
- */
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import * as fs from "node:fs";
-import { checkGate } from "./eval-gates.js";
-import { recordEvalRun } from "./eval-history.js";
-
-const TEST_PROJECT = "/tmp/eval-gates-test";
-
-beforeEach(() => {
-	// Clean slate for each test
-	if (fs.existsSync(TEST_PROJECT)) {
-		fs.rmSync(TEST_PROJECT, { recursive: true });
-	}
-	fs.mkdirSync(TEST_PROJECT, { recursive: true });
-});
-
-afterEach(() => {
-	// Cleanup
-	if (fs.existsSync(TEST_PROJECT)) {
-		fs.rmSync(TEST_PROJECT, { recursive: true });
-	}
-});
-
-/**
- * Helper to create run history
- */
-function seedHistory(evalName: string, scores: number[]): void {
-	for (let i = 0; i < scores.length; i++) {
-		recordEvalRun(TEST_PROJECT, {
-			timestamp: new Date(Date.now() + i * 1000).toISOString(),
-			eval_name: evalName,
-			score: scores[i],
-			run_count: i + 1,
-		});
-	}
-}
-
-describe("checkGate - Bootstrap Phase (<10 runs)", () => {
-	test("always passes with 0 runs", () => {
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.5);
-
-		expect(result.passed).toBe(true);
-		expect(result.phase).toBe("bootstrap");
-		expect(result.message).toContain("Bootstrap phase");
-	});
-
-	test("always passes with 9 runs, even with score drop", () => {
-		seedHistory("my-eval", [0.9, 0.88, 0.87, 0.86, 0.85, 0.84, 0.83, 0.82, 0.81]);
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.5); // 50% drop
-
-		expect(result.passed).toBe(true);
-		expect(result.phase).toBe("bootstrap");
-		expect(result.message).toContain("Bootstrap phase");
-	});
-
-	test("provides run count in message", () => {
-		seedHistory("my-eval", [0.8, 0.8, 0.8]);
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.75);
-
-		expect(result.message).toContain("3/10");
-	});
-});
-
-describe("checkGate - Stabilization Phase (10-50 runs)", () => {
-	test("exactly 10 runs enters stabilization", () => {
-		seedHistory("my-eval", Array(10).fill(0.85));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.84);
-
-		expect(result.phase).toBe("stabilization");
-	});
-
-	test("passes with <10% regression", () => {
-		seedHistory("my-eval", Array(15).fill(0.9));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.82); // 8.8% drop
-
-		expect(result.passed).toBe(true);
-		expect(result.phase).toBe("stabilization");
-		expect(result.message).toContain("acceptable");
-	});
-
-	test("WARNS on >10% regression but still passes", () => {
-		seedHistory("my-eval", Array(15).fill(0.9));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.8); // 11.1% drop
-
-		expect(result.passed).toBe(true); // Still passes in stabilization
-		expect(result.phase).toBe("stabilization");
-		expect(result.message).toContain("regression");
-		expect(result.message).toMatch(/10%|11%/); // Should mention threshold
-	});
-
-	test("edge case: exactly 10% regression", () => {
-		seedHistory("my-eval", Array(20).fill(0.9));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.81); // Exactly 10% drop
-
-		expect(result.passed).toBe(true);
-		expect(result.phase).toBe("stabilization");
-	});
-
-	test("passes with score improvement", () => {
-		seedHistory("my-eval", Array(25).fill(0.8));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.95);
-
-		expect(result.passed).toBe(true);
-		expect(result.message).toContain("acceptable");
-	});
-
-	test("exactly 50 runs still in stabilization", () => {
-		seedHistory("my-eval", Array(50).fill(0.85));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.84);
-
-		expect(result.phase).toBe("stabilization");
-	});
-});
-
-describe("checkGate - Production Phase (>50 runs + variance <0.1)", () => {
-	test("enters production with 51 stable runs", () => {
-		seedHistory("my-eval", Array(51).fill(0.85));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.84);
-
-		expect(result.phase).toBe("production");
-	});
-
-	test("FAILS on >5% regression in production", () => {
-		seedHistory("my-eval", Array(60).fill(0.9));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.84); // 6.7% drop
-
-		expect(result.passed).toBe(false);
-		expect(result.phase).toBe("production");
-		expect(result.message).toContain("FAIL");
-		expect(result.message).toMatch(/5%|6%/);
-	});
-
-	test("passes with <5% regression in production", () => {
-		seedHistory("my-eval", Array(60).fill(0.9));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.86); // 4.4% drop
-
-		expect(result.passed).toBe(true);
-		expect(result.phase).toBe("production");
-		expect(result.message).toContain("acceptable");
-	});
-
-	test("edge case: exactly 5% regression", () => {
-		seedHistory("my-eval", Array(60).fill(0.9));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.855); // Exactly 5% drop
-
-		expect(result.passed).toBe(true);
-		expect(result.phase).toBe("production");
-	});
-
-	test("stays in stabilization if variance too high (>0.1) despite >50 runs", () => {
-		// Need significant wild variance to push above 0.1
-		// From memory: 60 stable + 50 alternating wild = variance ~0.103
-		const stableRuns = Array(60).fill(0.85);
-		const wildRuns = Array(50)
-			.fill(0)
-			.map((_, i) => (i % 2 === 0 ? 0.1 : 0.9));
-		seedHistory("my-eval", [...stableRuns, ...wildRuns]);
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.84);
-
-		expect(result.phase).toBe("stabilization");
-		expect(result.message).toContain("variance");
-	});
-
-	test("passes with score improvement in production", () => {
-		seedHistory("my-eval", Array(60).fill(0.8));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.95);
-
-		expect(result.passed).toBe(true);
-		expect(result.phase).toBe("production");
-	});
-});
-
-describe("checkGate - Baseline Calculation", () => {
-	test("uses mean of all historical scores as baseline", () => {
-		// Need 10+ runs to exit bootstrap and see baseline in message
-		seedHistory("my-eval", [0.8, 0.85, 0.9, 0.95, 1.0, 0.9, 0.9, 0.9, 0.9, 0.9]); // mean = 0.9
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.88);
-
-		// 0.88 is ~2.2% drop from 0.9 mean (within stabilization tolerance)
-		expect(result.passed).toBe(true);
-		expect(result.message).toContain("0.90"); // Should show baseline
-	});
-
-	test("handles different eval names independently", () => {
-		seedHistory("eval-a", Array(15).fill(0.9));
-		seedHistory("eval-b", Array(15).fill(0.5));
-
-		const resultA = checkGate(TEST_PROJECT, "eval-a", 0.88);
-		const resultB = checkGate(TEST_PROJECT, "eval-b", 0.48);
-
-		expect(resultA.passed).toBe(true);
-		expect(resultB.passed).toBe(true);
-	});
-});
-
-describe("checkGate - Edge Cases", () => {
-	test("handles score of 0", () => {
-		seedHistory("my-eval", Array(15).fill(0.8));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0);
-
-		expect(result.passed).toBe(true); // Still passes in stabilization with warning
-		expect(result.message).toContain("regression");
-	});
-
-	test("handles perfect score of 1.0", () => {
-		seedHistory("my-eval", Array(15).fill(0.9));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 1.0);
-
-		expect(result.passed).toBe(true);
-	});
-
-	test("handles no history file (first run)", () => {
-		// No seedHistory call - empty project
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.75);
-
-		expect(result.passed).toBe(true);
-		expect(result.phase).toBe("bootstrap");
-	});
-
-	test("handles baseline of 0 (avoid division by zero)", () => {
-		seedHistory("my-eval", Array(15).fill(0));
-
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.5);
-
-		expect(result.passed).toBe(true);
-		expect(result.message).not.toContain("NaN");
-		expect(result.message).not.toContain("Infinity");
-	});
-});
-
-describe("checkGate - Configurable Thresholds", () => {
-	test("accepts custom stabilization threshold", () => {
-		seedHistory("my-eval", Array(15).fill(0.9));
-
-		// 15% regression with custom 20% threshold - should pass
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.765, {
-			stabilizationThreshold: 0.2, // 20% instead of default 10%
-		});
-
-		expect(result.passed).toBe(true);
-		expect(result.phase).toBe("stabilization");
-		expect(result.message).toContain("acceptable");
-	});
-
-	test("accepts custom production threshold", () => {
-		seedHistory("my-eval", Array(60).fill(0.9));
-
-		// 7% regression with custom 10% threshold - should pass
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.837, {
-			productionThreshold: 0.1, // 10% instead of default 5%
-		});
-
-		expect(result.passed).toBe(true);
-		expect(result.phase).toBe("production");
-	});
-
-	test("custom threshold makes test fail when exceeded", () => {
-		seedHistory("my-eval", Array(60).fill(0.9));
-
-		// 7% regression with custom 3% threshold - should fail
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.837, {
-			productionThreshold: 0.03, // 3% instead of default 5%
-		});
-
-		expect(result.passed).toBe(false);
-		expect(result.phase).toBe("production");
-		expect(result.message).toContain("FAIL");
-	});
-
-	test("partial config uses defaults for unspecified thresholds", () => {
-		seedHistory("my-eval", Array(15).fill(0.9));
-
-		// Only override production threshold
-		const result = checkGate(TEST_PROJECT, "my-eval", 0.88, {
-			productionThreshold: 0.01,
-			// stabilizationThreshold not specified - uses default 0.1
-		});
-
-		expect(result.passed).toBe(true); // 2.2% regression < 10% stabilization default
-	});
-});

package/src/eval-gates.ts

@@ -1,218 +0,0 @@
-/**
- * Progressive Eval Gates
- *
- * Enforces quality gates based on eval history and current phase:
- * - Bootstrap (<10 runs): Always pass, collect data
- * - Stabilization (10-50 runs): Pass but warn on >10% regression
- * - Production (>50 runs + variance <0.1): Fail on >5% regression
- *
- * @module eval-gates
- */
-import { calculateVariance, getPhase, getScoreHistory } from "./eval-history.js";
-
-/**
- * Result from a gate check
- */
-export interface GateResult {
-	/** Whether the gate passed */
-	passed: boolean;
-	/** Current phase */
-	phase: "bootstrap" | "stabilization" | "production";
-	/** Human-readable message */
-	message: string;
-	/** Baseline score (mean of history) */
-	baseline?: number;
-	/** Current score */
-	currentScore: number;
-	/** Regression percentage (negative = improvement) */
-	regressionPercent?: number;
-}
-
-/**
- * Configuration for gate thresholds
- */
-export interface GateConfig {
-	/** Regression threshold for stabilization phase (default: 0.1 = 10%) */
-	stabilizationThreshold?: number;
-	/** Regression threshold for production phase (default: 0.05 = 5%) */
-	productionThreshold?: number;
-}
-
-/**
- * Default regression thresholds by phase
- */
-export const DEFAULT_THRESHOLDS = {
-	stabilization: 0.1, // 10% regression warning
-	production: 0.05, // 5% regression failure
-} as const;
-
-/**
- * Calculate baseline score (mean of all historical scores)
- */
-function calculateBaseline(history: ReturnType<typeof getScoreHistory>, currentScore: number): number {
-	if (history.length === 0) {
-		return currentScore;
-	}
-	return history.reduce((sum, run) => sum + run.score, 0) / history.length;
-}
-
-/**
- * Calculate regression percentage from baseline
- *
- * Returns 0 if baseline is 0 to avoid division by zero.
- * Positive = regression (score dropped), negative = improvement
- */
-function calculateRegression(baseline: number, currentScore: number): number {
-	if (baseline === 0) {
-		return 0;
-	}
-	return (baseline - currentScore) / baseline;
-}
-
-/**
- * Format regression message with scores
- */
-function formatRegressionMessage(
-	regressionPercent: number,
-	baseline: number,
-	currentScore: number,
-): string {
-	return `${(regressionPercent * 100).toFixed(1)}% regression (baseline: ${baseline.toFixed(2)}, current: ${currentScore.toFixed(2)})`;
-}
-
-/**
- * Check if the current eval score passes the quality gate
- *
- * Progressive gates adapt based on data maturity:
- * - **Bootstrap (<10 runs)**: Always pass, focus on collecting baseline data
- * - **Stabilization (10-50 runs)**: Warn on >10% regression (default), but pass
- * - **Production (>50 runs + variance <0.1)**: Fail on >5% regression (default)
- *
- * **Baseline calculation**: Mean of all historical scores for this eval (not just last run).
- *
- * **Regression formula**: `(baseline - current) / baseline`
- * - Positive = regression (score dropped)
- * - Negative = improvement
- * - Returns 0 if baseline is 0 (avoids division by zero)
- *
- * **Variance threshold (0.1)**: High variance keeps eval in stabilization phase even with >50 runs.
- * This prevents premature production gates when scores are still unstable.
- *
- * **CI Integration**: Production gates can fail PRs. Use `swarm eval status` to check phase before merging.
- *
- * @param projectPath - Absolute path to project root (contains `.opencode/eval-history.jsonl`)
- * @param evalName - Name of the eval (e.g., "swarm-decomposition", "coordinator-behavior")
- * @param currentScore - Current score to check (typically 0-1 range)
- * @param config - Optional threshold configuration (defaults: stabilization=0.1, production=0.05)
- * @returns Gate check result with pass/fail, phase, baseline, regression details
- *
- * @example
- * ```typescript
- * import { checkGate } from "./eval-gates.js";
- *
- * const result = checkGate("/path/to/project", "swarm-decomposition", 0.89);
- *
- * if (!result.passed) {
- *   console.error(`❌ Gate FAILED: ${result.message}`);
- *   process.exit(1); // Fail CI
- * }
- *
- * console.log(`✅ ${result.phase} phase: ${result.message}`);
- * ```
- *
- * @example
- * ```typescript
- * // Custom thresholds for sensitive eval
- * const result = checkGate("/path", "critical-eval", 0.92, {
- *   stabilizationThreshold: 0.05,  // 5% threshold in stabilization
- *   productionThreshold: 0.02,     // 2% threshold in production
- * });
- * ```
- */
-export function checkGate(
-	projectPath: string,
-	evalName: string,
-	currentScore: number,
-	config?: GateConfig,
-): GateResult {
-	const thresholds = {
-		stabilization: config?.stabilizationThreshold ?? DEFAULT_THRESHOLDS.stabilization,
-		production: config?.productionThreshold ?? DEFAULT_THRESHOLDS.production,
-	};
-
-	const phase = getPhase(projectPath, evalName);
-	const history = getScoreHistory(projectPath, evalName);
-
-	// Bootstrap phase - always pass
-	if (phase === "bootstrap") {
-		return {
-			passed: true,
-			phase: "bootstrap",
-			message: `Bootstrap phase (${history.length}/10 runs) - collecting data`,
-			currentScore,
-		};
-	}
-
-	// Calculate baseline and regression
-	const baseline = calculateBaseline(history, currentScore);
-	const regressionPercent = calculateRegression(baseline, currentScore);
-	const regressionMsg = formatRegressionMessage(regressionPercent, baseline, currentScore);
-
-	// Stabilization phase - warn on regression but pass
-	if (phase === "stabilization") {
-		if (regressionPercent > thresholds.stabilization) {
-			return {
-				passed: true,
-				phase: "stabilization",
-				message: `Stabilization phase: ${regressionMsg} - exceeds ${(thresholds.stabilization * 100).toFixed(0)}% threshold but still passing`,
-				baseline,
-				currentScore,
-				regressionPercent,
-			};
-		}
-
-		// Check if we have high variance (>50 runs but can't enter production)
-		if (history.length > 50) {
-			const scores = history.map((run) => run.score);
-			const variance = calculateVariance(scores);
-			return {
-				passed: true,
-				phase: "stabilization",
-				message: `Stabilization phase: ${regressionMsg} - acceptable. High variance (${variance.toFixed(3)}) prevents production phase.`,
-				baseline,
-				currentScore,
-				regressionPercent,
-			};
-		}
-
-		return {
-			passed: true,
-			phase: "stabilization",
-			message: `Stabilization phase: ${regressionMsg} - acceptable`,
-			baseline,
-			currentScore,
-			regressionPercent,
-		};
-	}
-
-	// Production phase - fail on regression exceeding threshold
-	if (regressionPercent > thresholds.production) {
-		return {
-			passed: false,
-			phase: "production",
-			message: `Production phase FAIL: ${regressionMsg} - exceeds ${(thresholds.production * 100).toFixed(0)}% threshold`,
-			baseline,
-			currentScore,
-			regressionPercent,
-		};
-	}
-
-	return {
-		passed: true,
-		phase: "production",
-		message: `Production phase: ${regressionMsg} - acceptable`,
-		baseline,
-		currentScore,
-		regressionPercent,
-	};
-}