opencode-swarm-plugin 0.56.0 → 0.57.0

package/claude-plugin/dist/eval-gates.d.ts

@@ -0,0 +1,84 @@
+/**
+ * Result from a gate check
+ */
+export interface GateResult {
+    /** Whether the gate passed */
+    passed: boolean;
+    /** Current phase */
+    phase: "bootstrap" | "stabilization" | "production";
+    /** Human-readable message */
+    message: string;
+    /** Baseline score (mean of history) */
+    baseline?: number;
+    /** Current score */
+    currentScore: number;
+    /** Regression percentage (negative = improvement) */
+    regressionPercent?: number;
+}
+/**
+ * Configuration for gate thresholds
+ */
+export interface GateConfig {
+    /** Regression threshold for stabilization phase (default: 0.1 = 10%) */
+    stabilizationThreshold?: number;
+    /** Regression threshold for production phase (default: 0.05 = 5%) */
+    productionThreshold?: number;
+}
+/**
+ * Default regression thresholds by phase
+ */
+export declare const DEFAULT_THRESHOLDS: {
+    readonly stabilization: 0.1;
+    readonly production: 0.05;
+};
+/**
+ * Check if the current eval score passes the quality gate
+ *
+ * Progressive gates adapt based on data maturity:
+ * - **Bootstrap (<10 runs)**: Always pass, focus on collecting baseline data
+ * - **Stabilization (10-50 runs)**: Warn on >10% regression (default), but pass
+ * - **Production (>50 runs + variance <0.1)**: Fail on >5% regression (default)
+ *
+ * **Baseline calculation**: Mean of all historical scores for this eval (not just last run).
+ *
+ * **Regression formula**: `(baseline - current) / baseline`
+ * - Positive = regression (score dropped)
+ * - Negative = improvement
+ * - Returns 0 if baseline is 0 (avoids division by zero)
+ *
+ * **Variance threshold (0.1)**: High variance keeps eval in stabilization phase even with >50 runs.
+ * This prevents premature production gates when scores are still unstable.
+ *
+ * **CI Integration**: Production gates can fail PRs. Use `swarm eval status` to check phase before merging.
+ *
+ * @param projectPath - Absolute path to project root (contains `.opencode/eval-history.jsonl`)
+ * @param evalName - Name of the eval (e.g., "swarm-decomposition", "coordinator-behavior")
+ * @param currentScore - Current score to check (typically 0-1 range)
+ * @param config - Optional threshold configuration (defaults: stabilization=0.1, production=0.05)
+ * @returns Gate check result with pass/fail, phase, baseline, regression details
+ *
+ * @example
+ * ```typescript
+ * import { checkGate } from "./eval-gates.js";
+ *
+ * const result = checkGate("/path/to/project", "swarm-decomposition", 0.89);
+ *
+ * if (!result.passed) {
+ *   console.error(`❌ Gate FAILED: ${result.message}`);
+ *   process.exit(1); // Fail CI
+ * }
+ *
+ * console.log(`✅ ${result.phase} phase: ${result.message}`);
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Custom thresholds for sensitive eval
+ * const result = checkGate("/path", "critical-eval", 0.92, {
+ *   stabilizationThreshold: 0.05,  // 5% threshold in stabilization
+ *   productionThreshold: 0.02,     // 2% threshold in production
+ * });
+ * ```
+ */
+export declare function checkGate(projectPath: string, evalName: string, currentScore: number, config?: GateConfig): GateResult;
+//# sourceMappingURL=eval-gates.d.ts.map

package/claude-plugin/dist/eval-gates.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-gates.d.ts","sourceRoot":"","sources":["../src/eval-gates.ts"],"names":[],"mappings":"AAYA;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,8BAA8B;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,oBAAoB;IACpB,KAAK,EAAE,WAAW,GAAG,eAAe,GAAG,YAAY,CAAC;IACpD,6BAA6B;IAC7B,OAAO,EAAE,MAAM,CAAC;IAChB,uCAAuC;IACvC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,qDAAqD;IACrD,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,UAAU;IAC1B,wEAAwE;IACxE,sBAAsB,CAAC,EAAE,MAAM,CAAC;IAChC,qEAAqE;IACrE,mBAAmB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED;;GAEG;AACH,eAAO,MAAM,kBAAkB;;;CAGrB,CAAC;AAoCX;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AACH,wBAAgB,SAAS,CACxB,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,MAAM,CAAC,EAAE,UAAU,GACjB,UAAU,CAiFZ"}

package/claude-plugin/dist/eval-history.d.ts

@@ -0,0 +1,117 @@
+/**
+ * Progressive phases based on run count and variance
+ */
+export type Phase = "bootstrap" | "stabilization" | "production";
+/**
+ * Single eval run record
+ */
+export interface EvalRunRecord {
+    /** ISO-8601 timestamp */
+    timestamp: string;
+    /** Name of the eval (e.g., "swarm-decomposition") */
+    eval_name: string;
+    /** Score (0-1 range typically) */
+    score: number;
+    /** Run count (monotonically increasing per eval) */
+    run_count: number;
+}
+/**
+ * Default path for eval history
+ */
+export declare const DEFAULT_EVAL_HISTORY_PATH = ".opencode/eval-history.jsonl";
+/**
+ * Variance threshold for production phase
+ */
+export declare const VARIANCE_THRESHOLD = 0.1;
+/**
+ * Run count thresholds for phase transitions
+ */
+export declare const BOOTSTRAP_THRESHOLD = 10;
+export declare const STABILIZATION_THRESHOLD = 50;
+/**
+ * Get the eval history file path
+ */
+export declare function getEvalHistoryPath(projectPath: string): string;
+/**
+ * Ensure the eval history directory exists
+ */
+export declare function ensureEvalHistoryDir(projectPath: string): void;
+/**
+ * Record an eval run to JSONL history
+ *
+ * Appends atomically to `.opencode/eval-history.jsonl`. Each line is a complete JSON object
+ * representing one eval run (timestamp, eval name, score, run count).
+ *
+ * **Auto-creates directory** if `.opencode/` doesn't exist.
+ *
+ * **Thread-safe**: Uses `appendFileSync` for atomic writes (safe for concurrent eval runs).
+ *
+ * **Integration**: Called automatically by evalite runner after each eval completes.
+ * Also callable manually for custom eval tracking.
+ *
+ * @param projectPath - Absolute path to project root
+ * @param run - Eval run record with timestamp, eval_name, score, run_count
+ *
+ * @example
+ * ```typescript
+ * import { recordEvalRun } from "./eval-history.js";
+ *
+ * recordEvalRun("/path/to/project", {
+ *   timestamp: new Date().toISOString(),
+ *   eval_name: "swarm-decomposition",
+ *   score: 0.92,
+ *   run_count: 15,
+ * });
+ * ```
+ */
+export declare function recordEvalRun(projectPath: string, run: EvalRunRecord): void;
+/**
+ * Get score history for a specific eval
+ *
+ * Returns runs in chronological order (oldest first)
+ */
+export declare function getScoreHistory(projectPath: string, evalName: string): EvalRunRecord[];
+/**
+ * Calculate statistical variance of scores
+ *
+ * Variance = mean of squared deviations from the mean
+ * Formula: Σ((x - μ)²) / n
+ */
+export declare function calculateVariance(scores: number[]): number;
+/**
+ * Get the current phase for an eval based on run count and score variance
+ *
+ * Progressive phase logic ensures quality gates adapt to data maturity:
+ *
+ * - **Bootstrap (<10 runs)**: No gates, just collect baseline data
+ * - **Stabilization (10-50 runs)**: Warn on >10% regression (but pass)
+ * - **Production (>50 runs AND variance <0.1)**: Fail on >5% regression
+ *
+ * **Variance check**: If >50 runs but variance ≥0.1, stays in stabilization.
+ * This prevents premature production gates when scores are still unstable.
+ *
+ * **Why variance matters**: An eval with wildly fluctuating scores isn't ready for
+ * strict gates. Variance threshold (0.1) ensures the eval is consistent before
+ * enforcing production-level quality control.
+ *
+ * @param projectPath - Absolute path to project root (contains `.opencode/eval-history.jsonl`)
+ * @param evalName - Name of the eval (e.g., "swarm-decomposition")
+ * @returns Current phase: "bootstrap" | "stabilization" | "production"
+ *
+ * @example
+ * ```typescript
+ * import { getPhase } from "./eval-history.js";
+ *
+ * const phase = getPhase("/path/to/project", "swarm-decomposition");
+ *
+ * if (phase === "production") {
+ *   console.log("🚀 Production phase - strict gates enabled");
+ * } else if (phase === "stabilization") {
+ *   console.log("⚙️ Stabilization phase - warnings only");
+ * } else {
+ *   console.log("🌱 Bootstrap phase - collecting data");
+ * }
+ * ```
+ */
+export declare function getPhase(projectPath: string, evalName: string): Phase;
+//# sourceMappingURL=eval-history.d.ts.map

package/claude-plugin/dist/eval-history.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-history.d.ts","sourceRoot":"","sources":["../src/eval-history.ts"],"names":[],"mappings":"AAaA;;GAEG;AACH,MAAM,MAAM,KAAK,GAAG,WAAW,GAAG,eAAe,GAAG,YAAY,CAAC;AAEjE;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B,yBAAyB;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;IAClB,kCAAkC;IAClC,KAAK,EAAE,MAAM,CAAC;IACd,oDAAoD;IACpD,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,eAAO,MAAM,yBAAyB,iCAAiC,CAAC;AAExE;;GAEG;AACH,eAAO,MAAM,kBAAkB,MAAM,CAAC;AAEtC;;GAEG;AACH,eAAO,MAAM,mBAAmB,KAAK,CAAC;AACtC,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAE1C;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM,CAE9D;AAED;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAM9D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAgB,aAAa,CAC3B,WAAW,EAAE,MAAM,EACnB,GAAG,EAAE,aAAa,GACjB,IAAI,CAKN;AAoBD;;;;GAIG;AACH,wBAAgB,eAAe,CAC7B,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,GACf,aAAa,EAAE,CAIjB;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAa1D;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,wBAAgB,QAAQ,CAAC,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,KAAK,CAqBrE"}

package/claude-plugin/dist/eval-learning.d.ts

@@ -0,0 +1,216 @@
+/**
+ * Eval-to-Learning Feedback Loop
+ *
+ * Automatically stores eval failures to semantic memory for learning.
+ * When eval scores drop significantly from rolling average (default >15%),
+ * stores context to semantic-memory with tags for future prompt generation.
+ *
+ * ## Usage
+ *
+ * ```typescript
+ * import { learnFromEvalFailure } from "./eval-learning";
+ * import { getHivemindAdapter } from "./hivemind-tools";
+ * import { getScoreHistory } from "./eval-history";
+ *
+ * const memoryAdapter = await getHivemindAdapter();
+ * const history = getScoreHistory(projectPath, "compaction-test");
+ *
+ * const result = await learnFromEvalFailure(
+ *   "compaction-test",
+ *   currentScore,
+ *   history,
+ *   memoryAdapter
+ * );
+ *
+ * if (result.triggered) {
+ *   console.log(`📉 Regression detected: ${(result.drop_percentage * 100).toFixed(1)}% drop`);
+ *   console.log(`Memory ID: ${result.memory_id}`);
+ * }
+ * ```
+ *
+ * ## Integration Points
+ *
+ * - **After each eval run**: Call to detect regressions automatically
+ * - **Memory tags**: `eval-failure`, `{eval-name}`, `regression`
+ * - **Future prompts**: Query memories with these tags for context
+ * - **Scorer context**: Optional detail about which scorer failed
+ *
+ * ## Customization
+ *
+ * ```typescript
+ * const customConfig = {
+ *   dropThreshold: 0.10,  // 10% threshold (more sensitive)
+ *   windowSize: 10,        // Last 10 runs for baseline
+ * };
+ *
+ * await learnFromEvalFailure(
+ *   "test",
+ *   score,
+ *   history,
+ *   adapter,
+ *   { config: customConfig }
+ * );
+ * ```
+ *
+ * @module eval-learning
+ */
+import type { EvalRunRecord } from "./eval-history";
+import type { MemoryAdapter } from "./hivemind-tools";
+/**
+ * Configuration for eval-to-learning feedback
+ */
+export interface EvalLearningConfig {
+    /** Threshold for significant drop (0-1, default 0.15 = 15%) */
+    dropThreshold: number;
+    /** Rolling average window size (default 5 runs) */
+    windowSize: number;
+}
+/**
+ * Default configuration
+ */
+export declare const DEFAULT_EVAL_LEARNING_CONFIG: EvalLearningConfig;
+/**
+ * Result from learning check
+ */
+export interface LearningResult {
+    /** Whether the check triggered memory storage */
+    triggered: boolean;
+    /** Baseline score from rolling average */
+    baseline: number;
+    /** Current score */
+    current: number;
+    /** Drop percentage (0-1, e.g., 0.20 = 20% drop) */
+    drop_percentage: number;
+    /** Memory ID if stored */
+    memory_id?: string;
+}
+/**
+ * Calculate rolling average of recent scores
+ *
+ * Uses last N runs (default 5) to establish baseline.
+ * If history shorter than window, uses all available.
+ *
+ * @param history - Score history (chronological order)
+ * @param windowSize - Number of recent runs to average (default 5)
+ * @returns Average score (0 if empty)
+ */
+export declare function calculateRollingAverage(history: EvalRunRecord[], windowSize?: number): number;
+/**
+ * Check if current score is a significant drop from baseline
+ *
+ * Significant = drop exceeds threshold (default 15%).
+ * Formula: (baseline - current) / baseline >= threshold
+ *
+ * @param currentScore - Current eval score
+ * @param baseline - Baseline score (rolling average)
+ * @param threshold - Drop threshold (default 0.15 = 15%)
+ * @returns True if drop is significant
+ */
+export declare function isSignificantDrop(currentScore: number, baseline: number, threshold?: number): boolean;
+/**
+ * Format failure context for semantic memory storage
+ *
+ * Creates human-readable description of the failure with
+ * quantified metrics and optional scorer context.
+ *
+ * @param evalName - Name of eval that failed
+ * @param currentScore - Current score
+ * @param baseline - Baseline score
+ * @param scorerContext - Optional context about which scorer failed
+ * @returns Formatted context string
+ */
+export declare function formatFailureContext(evalName: string, currentScore: number, baseline: number, scorerContext?: string): string;
+/**
+ * Main learning function - automatically stores eval failures to semantic memory
+ *
+ * **Closed-loop learning**: When eval scores drop significantly from baseline,
+ * this function stores failure context to semantic memory. Future prompt generation
+ * queries these memories for context, preventing repeated mistakes.
+ *
+ * **Trigger condition**: Score drops >15% (default) from rolling average baseline.
+ * Uses last 5 runs (default) to establish baseline, not just previous run.
+ *
+ * **What gets stored**:
+ * - Eval name, baseline score, current score, drop percentage
+ * - Scorer-specific context (which scorer failed, why)
+ * - Timestamp and metadata for querying
+ * - Tags: `eval-failure`, `{eval-name}`, `regression`
+ *
+ * **Future use**: Before generating prompts for the same eval, query semantic memory
+ * with tags to inject learnings from past failures.
+ *
+ * **Integration points**:
+ * - After each eval run (in evalite runner or CI)
+ * - In `checkGate()` when regression detected
+ * - Manual calls for custom eval tracking
+ *
+ * @param evalName - Name of eval (e.g., "compaction-test", "coordinator-behavior")
+ * @param currentScore - Current eval score (typically 0-1 range)
+ * @param history - Score history in chronological order (oldest first)
+ * @param memoryAdapter - Semantic memory adapter (from `getMemoryAdapter()`)
+ * @param options - Optional config (thresholds, window size) and scorer context
+ * @param options.config - Custom thresholds (dropThreshold, windowSize)
+ * @param options.scorerContext - Details about which scorer failed (for context)
+ * @returns Learning result with trigger status, baseline, drop percentage, memory ID
+ *
+ * @example
+ * ```typescript
+ * import { learnFromEvalFailure } from "./eval-learning.js";
+ * import { getHivemindAdapter } from "./hivemind-tools.js";
+ * import { getScoreHistory } from "./eval-history.js";
+ *
+ * const memoryAdapter = await getHivemindAdapter();
+ * const history = getScoreHistory("/path/to/project", "coordinator-behavior");
+ *
+ * const result = await learnFromEvalFailure(
+ *   "coordinator-behavior",
+ *   0.68,  // Current score
+ *   history,
+ *   memoryAdapter,
+ *   { scorerContext: "violationCount: 5 violations (coordinator edited files)" }
+ * );
+ *
+ * if (result.triggered) {
+ *   console.log(`📉 Regression detected: ${(result.drop_percentage * 100).toFixed(1)}% drop`);
+ *   console.log(`Stored to memory: ${result.memory_id}`);
+ * }
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Custom threshold (more sensitive)
+ * const result = await learnFromEvalFailure(
+ *   "critical-eval",
+ *   0.85,
+ *   history,
+ *   memoryAdapter,
+ *   {
+ *     config: {
+ *       dropThreshold: 0.10,  // 10% threshold (default is 15%)
+ *       windowSize: 10,       // Last 10 runs for baseline (default is 5)
+ *     },
+ *   }
+ * );
+ * ```
+ */
+export declare function learnFromEvalFailure(evalName: string, currentScore: number, history: EvalRunRecord[], memoryAdapter: MemoryAdapter, options?: {
+    config?: EvalLearningConfig;
+    scorerContext?: string;
+}): Promise<LearningResult>;
+/**
+ * Create custom learning config with specific threshold
+ *
+ * Helper for common use case: custom drop threshold.
+ *
+ * @param dropThreshold - Drop threshold (0-1)
+ * @param windowSize - Optional window size (default 5)
+ * @returns Custom config
+ *
+ * @example
+ * ```typescript
+ * const config = createLearningConfig(0.10); // 10% threshold
+ * await learnFromEvalFailure("test", score, history, adapter, { config });
+ * ```
+ */
+export declare function createLearningConfig(dropThreshold: number, windowSize?: number): EvalLearningConfig;
+//# sourceMappingURL=eval-learning.d.ts.map

package/claude-plugin/dist/eval-learning.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-learning.d.ts","sourceRoot":"","sources":["../src/eval-learning.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAuDG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAMtD;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,+DAA+D;IAC/D,aAAa,EAAE,MAAM,CAAC;IACtB,mDAAmD;IACnD,UAAU,EAAE,MAAM,CAAC;CACnB;AAED;;GAEG;AACH,eAAO,MAAM,4BAA4B,EAAE,kBAG1C,CAAC;AAMF;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,iDAAiD;IACjD,SAAS,EAAE,OAAO,CAAC;IACnB,0CAA0C;IAC1C,QAAQ,EAAE,MAAM,CAAC;IACjB,oBAAoB;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,mDAAmD;IACnD,eAAe,EAAE,MAAM,CAAC;IACxB,0BAA0B;IAC1B,SAAS,CAAC,EAAE,MAAM,CAAC;CACnB;AAMD;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CACtC,OAAO,EAAE,aAAa,EAAE,EACxB,UAAU,GAAE,MAAU,GACpB,MAAM,CAQR;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAChC,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,EAChB,SAAS,GAAE,MAAa,GACtB,OAAO,CAQT;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,oBAAoB,CACnC,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,QAAQ,EAAE,MAAM,EAChB,aAAa,CAAC,EAAE,MAAM,GACpB,MAAM,CAsBR;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwEG;AACH,wBAAsB,oBAAoB,CACzC,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,EACpB,OAAO,EAAE,aAAa,EAAE,EACxB,aAAa,EAAE,aAAa,EAC5B,OAAO,CAAC,EAAE;IACT,MAAM,CAAC,EAAE,kBAAkB,CAAC;IAC5B,aAAa,CAAC,EAAE,MAAM,CAAC;CACvB,GACC,OAAO,CAAC,cAAc,CAAC,CAqDzB;AAMD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,oBAAoB,CACnC,aAAa,EAAE,MAAM,EACrB,UAAU,CAAC,EAAE,MAAM,GACjB,kBAAkB,CAKpB"}

package/claude-plugin/dist/eval-runner.d.ts

@@ -0,0 +1,134 @@
+/**
+ * Programmatic Evalite Runner
+ *
+ * Provides a type-safe API for running evalite evals programmatically.
+ * Wraps evalite's runEvalite function with structured result parsing.
+ *
+ * @module eval-runner
+ */
+/**
+ * Options for running evals programmatically
+ */
+export interface RunEvalsOptions {
+    /**
+     * Working directory containing eval files (defaults to process.cwd())
+     */
+    cwd?: string;
+    /**
+     * Optional filter to run specific eval suites (e.g., "coordinator", "compaction")
+     * Matches against eval file paths using substring matching
+     */
+    suiteFilter?: string;
+    /**
+     * Minimum average score threshold (0-100)
+     * If average score falls below this, result.success will be false
+     */
+    scoreThreshold?: number;
+    /**
+     * Optional path to write raw evalite JSON output
+     */
+    outputPath?: string;
+}
+/**
+ * Structured suite result with scores
+ */
+export interface SuiteResult {
+    /** Suite name from evalite() call */
+    name: string;
+    /** Absolute path to eval file */
+    filepath: string;
+    /** Suite status: success, fail, or running */
+    status: "success" | "fail" | "running";
+    /** Total duration in milliseconds */
+    duration: number;
+    /** Average score across all evals in suite (0-1 scale) */
+    averageScore: number;
+    /** Number of evals in this suite */
+    evalCount: number;
+    /** Individual eval results (optional, can be large) */
+    evals?: Array<{
+        input: unknown;
+        output: unknown;
+        expected?: unknown;
+        scores: Array<{
+            name: string;
+            score: number;
+            description?: string;
+        }>;
+    }>;
+}
+/**
+ * Structured result from running evals
+ */
+export interface RunEvalsResult {
+    /** Whether the run succeeded (all evals passed threshold) */
+    success: boolean;
+    /** Total number of suites executed */
+    totalSuites: number;
+    /** Total number of individual evals executed */
+    totalEvals: number;
+    /** Average score across all suites (0-1 scale) */
+    averageScore: number;
+    /** Individual suite results */
+    suites: SuiteResult[];
+    /** Error message if run failed */
+    error?: string;
+    /** Gate check results per suite */
+    gateResults?: Array<{
+        suite: string;
+        passed: boolean;
+        phase: string;
+        message: string;
+        baseline?: number;
+        currentScore: number;
+        regressionPercent?: number;
+    }>;
+}
+/**
+ * Run evalite evals programmatically
+ *
+ * @param options - Configuration for eval run
+ * @returns Structured results with scores per suite
+ *
+ * @example
+ * ```typescript
+ * // Run all evals
+ * const result = await runEvals({ cwd: "/path/to/project" });
+ * console.log(`Average score: ${result.averageScore}`);
+ *
+ * // Run specific suite
+ * const coordResult = await runEvals({
+ *   cwd: "/path/to/project",
+ *   suiteFilter: "coordinator"
+ * });
+ *
+ * // Enforce score threshold
+ * const gatedResult = await runEvals({
+ *   cwd: "/path/to/project",
+ *   scoreThreshold: 80
+ * });
+ * if (!gatedResult.success) {
+ *   throw new Error(`Evals failed threshold: ${gatedResult.averageScore}`);
+ * }
+ * ```
+ */
+export declare function runEvals(options?: RunEvalsOptions): Promise<RunEvalsResult>;
+/**
+ * All eval tools exported for registration
+ */
+export declare const evalTools: {
+    readonly eval_run: {
+        description: string;
+        args: {
+            suiteFilter: import("zod").ZodOptional<import("zod").ZodString>;
+            scoreThreshold: import("zod").ZodOptional<import("zod").ZodNumber>;
+            includeDetailedResults: import("zod").ZodOptional<import("zod").ZodBoolean>;
+        };
+        execute(args: {
+            suiteFilter?: string | undefined;
+            scoreThreshold?: number | undefined;
+            includeDetailedResults?: boolean | undefined;
+        }, context: import("@opencode-ai/plugin").ToolContext): Promise<string>;
+    };
+};
+//# sourceMappingURL=eval-runner.d.ts.map

package/claude-plugin/dist/eval-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.d.ts","sourceRoot":"","sources":["../src/eval-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAaH;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;OAEG;IACH,GAAG,CAAC,EAAE,MAAM,CAAC;IAEb;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,qCAAqC;IACrC,IAAI,EAAE,MAAM,CAAC;IAEb,iCAAiC;IACjC,QAAQ,EAAE,MAAM,CAAC;IAEjB,8CAA8C;IAC9C,MAAM,EAAE,SAAS,GAAG,MAAM,GAAG,SAAS,CAAC;IAEvC,qCAAqC;IACrC,QAAQ,EAAE,MAAM,CAAC;IAEjB,0DAA0D;IAC1D,YAAY,EAAE,MAAM,CAAC;IAErB,oCAAoC;IACpC,SAAS,EAAE,MAAM,CAAC;IAElB,uDAAuD;IACvD,KAAK,CAAC,EAAE,KAAK,CAAC;QACZ,KAAK,EAAE,OAAO,CAAC;QACf,MAAM,EAAE,OAAO,CAAC;QAChB,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,MAAM,EAAE,KAAK,CAAC;YACZ,IAAI,EAAE,MAAM,CAAC;YACb,KAAK,EAAE,MAAM,CAAC;YACd,WAAW,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC,CAAC;KACJ,CAAC,CAAC;CACJ;AAED;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6DAA6D;IAC7D,OAAO,EAAE,OAAO,CAAC;IAEjB,sCAAsC;IACtC,WAAW,EAAE,MAAM,CAAC;IAEpB,gDAAgD;IAChD,UAAU,EAAE,MAAM,CAAC;IAEnB,kDAAkD;IAClD,YAAY,EAAE,MAAM,CAAC;IAErB,+BAA+B;IAC/B,MAAM,EAAE,WAAW,EAAE,CAAC;IAEtB,kCAAkC;IAClC,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,mCAAmC;IACnC,WAAW,CAAC,EAAE,KAAK,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,OAAO,CAAC;QAChB,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,YAAY,EAAE,MAAM,CAAC;QACrB,iBAAiB,CAAC,EAAE,MAAM,CAAC;KAC5B,CAAC,CAAC;CACJ;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,wBAAsB,QAAQ,CAC5B,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC,cAAc,CAAC,CAoLzB;AAsED;;GAEG;AACH,eAAO,MAAM,SAAS;;;;;;;;;;;;;;CAEZ,CAAC"}