opencode-swarm-plugin 0.38.0 → 0.40.0

package/src/eval-runner.ts

@@ -0,0 +1,356 @@
+/**
+ * Programmatic Evalite Runner
+ *
+ * Provides a type-safe API for running evalite evals programmatically.
+ * Wraps evalite's runEvalite function with structured result parsing.
+ *
+ * @module eval-runner
+ */
+
+import { tool } from "@opencode-ai/plugin";
+import { runEvalite } from "evalite/runner";
+import { createInMemoryStorage } from "evalite/in-memory-storage";
+import type { Evalite } from "evalite/types";
+import fs from "node:fs/promises";
+import path from "node:path";
+
+/**
+ * Options for running evals programmatically
+ */
+export interface RunEvalsOptions {
+  /**
+   * Working directory containing eval files (defaults to process.cwd())
+   */
+  cwd?: string;
+  
+  /**
+   * Optional filter to run specific eval suites (e.g., "coordinator", "compaction")
+   * Matches against eval file paths using substring matching
+   */
+  suiteFilter?: string;
+  
+  /**
+   * Minimum average score threshold (0-100)
+   * If average score falls below this, result.success will be false
+   */
+  scoreThreshold?: number;
+  
+  /**
+   * Optional path to write raw evalite JSON output
+   */
+  outputPath?: string;
+}
+
+/**
+ * Structured suite result with scores
+ */
+export interface SuiteResult {
+  /** Suite name from evalite() call */
+  name: string;
+  
+  /** Absolute path to eval file */
+  filepath: string;
+  
+  /** Suite status: success, fail, or running */
+  status: "success" | "fail" | "running";
+  
+  /** Total duration in milliseconds */
+  duration: number;
+  
+  /** Average score across all evals in suite (0-1 scale) */
+  averageScore: number;
+  
+  /** Number of evals in this suite */
+  evalCount: number;
+  
+  /** Individual eval results (optional, can be large) */
+  evals?: Array<{
+    input: unknown;
+    output: unknown;
+    expected?: unknown;
+    scores: Array<{
+      name: string;
+      score: number;
+      description?: string;
+    }>;
+  }>;
+}
+
+/**
+ * Structured result from running evals
+ */
+export interface RunEvalsResult {
+  /** Whether the run succeeded (all evals passed threshold) */
+  success: boolean;
+  
+  /** Total number of suites executed */
+  totalSuites: number;
+  
+  /** Total number of individual evals executed */
+  totalEvals: number;
+  
+  /** Average score across all suites (0-1 scale) */
+  averageScore: number;
+  
+  /** Individual suite results */
+  suites: SuiteResult[];
+  
+  /** Error message if run failed */
+  error?: string;
+}
+
+/**
+ * Run evalite evals programmatically
+ *
+ * @param options - Configuration for eval run
+ * @returns Structured results with scores per suite
+ *
+ * @example
+ * ```typescript
+ * // Run all evals
+ * const result = await runEvals({ cwd: "/path/to/project" });
+ * console.log(`Average score: ${result.averageScore}`);
+ *
+ * // Run specific suite
+ * const coordResult = await runEvals({
+ *   cwd: "/path/to/project",
+ *   suiteFilter: "coordinator"
+ * });
+ *
+ * // Enforce score threshold
+ * const gatedResult = await runEvals({
+ *   cwd: "/path/to/project",
+ *   scoreThreshold: 80
+ * });
+ * if (!gatedResult.success) {
+ *   throw new Error(`Evals failed threshold: ${gatedResult.averageScore}`);
+ * }
+ * ```
+ */
+export async function runEvals(
+  options: RunEvalsOptions = {}
+): Promise<RunEvalsResult> {
+  const {
+    cwd = process.cwd(),
+    suiteFilter,
+    scoreThreshold,
+    outputPath: userOutputPath,
+  } = options;
+
+  try {
+    // Resolve to project root (evals are in evals/ relative to project root)
+    // If cwd is src/, go up one level
+    const projectRoot = cwd.endsWith("src") ? path.dirname(cwd) : cwd;
+    const evalsDir = path.join(projectRoot, "evals");
+    let evalPath: string | undefined;
+
+    if (suiteFilter) {
+      // Find matching eval files
+      try {
+        const files = await fs.readdir(evalsDir);
+        const matchingFiles = files.filter((f) =>
+          f.toLowerCase().includes(suiteFilter.toLowerCase())
+        );
+
+        if (matchingFiles.length === 0) {
+          // No matches - return empty result (not an error)
+          return {
+            success: true,
+            totalSuites: 0,
+            totalEvals: 0,
+            averageScore: 0,
+            suites: [],
+          };
+        }
+
+        // Use first matching file (evalite will discover all via vitest)
+        evalPath = path.join(evalsDir, matchingFiles[0]);
+      } catch (err) {
+        // Directory doesn't exist or can't be read
+        return {
+          success: false,
+          totalSuites: 0,
+          totalEvals: 0,
+          averageScore: 0,
+          suites: [],
+          error: `Failed to read evals directory: ${err instanceof Error ? err.message : String(err)}`,
+        };
+      }
+    } else {
+      // No filter - run all evals in evals/
+      evalPath = evalsDir;
+    }
+
+    // Use temporary output path if user didn't provide one
+    const outputPath =
+      userOutputPath || path.join(projectRoot, `.evalite-results-${Date.now()}.json`);
+    const isTemporaryOutput = !userOutputPath;
+
+    // Run evalite programmatically
+    const storage = createInMemoryStorage();
+    
+    await runEvalite({
+      path: evalPath, // undefined = run all
+      cwd: projectRoot, // Use project root as working directory
+      mode: "run-once",
+      scoreThreshold,
+      outputPath,
+      hideTable: true, // Suppress terminal output
+      storage,
+      disableServer: true, // No UI server needed
+    });
+
+    // Parse output file for structured results
+    let outputJson: string;
+    try {
+      outputJson = await fs.readFile(outputPath, "utf-8");
+    } catch (err) {
+      // Output file wasn't written - evalite crashed or no tests ran
+      return {
+        success: false,
+        totalSuites: 0,
+        totalEvals: 0,
+        averageScore: 0,
+        suites: [],
+        error: `No results file generated: ${err instanceof Error ? err.message : String(err)}`,
+      };
+    }
+
+    const output: Evalite.Exported.Output = JSON.parse(outputJson);
+
+    // Clean up temporary output file
+    if (isTemporaryOutput) {
+      await fs.unlink(outputPath).catch(() => {
+        /* ignore cleanup errors */
+      });
+    }
+
+    // Transform to structured result
+    const suites: SuiteResult[] = output.suites.map((suite) => ({
+      name: suite.name,
+      filepath: suite.filepath,
+      status: suite.status,
+      duration: suite.duration,
+      averageScore: suite.averageScore,
+      evalCount: suite.evals.length,
+      // Include evals if user wants detailed results
+      evals: suite.evals.map((e) => ({
+        input: e.input,
+        output: e.output,
+        expected: e.expected,
+        scores: e.scores.map((s) => ({
+          name: s.name,
+          score: s.score,
+          description: s.description,
+        })),
+      })),
+    }));
+
+    // Calculate overall metrics
+    const totalEvals = suites.reduce((sum, s) => sum + s.evalCount, 0);
+    const averageScore =
+      suites.length > 0
+        ? suites.reduce((sum, s) => sum + s.averageScore, 0) / suites.length
+        : 0;
+
+    // Determine success based on threshold
+    const thresholdPassed =
+      scoreThreshold === undefined || averageScore * 100 >= scoreThreshold;
+
+    return {
+      success: thresholdPassed,
+      totalSuites: suites.length,
+      totalEvals,
+      averageScore,
+      suites,
+    };
+  } catch (error) {
+    // Return error result
+    return {
+      success: false,
+      totalSuites: 0,
+      totalEvals: 0,
+      averageScore: 0,
+      suites: [],
+      error:
+        error instanceof Error
+          ? error.message
+          : String(error),
+    };
+  }
+}
+
+// ============================================================================
+// Plugin Tool
+// ============================================================================
+
+/**
+ * Plugin tool for running evals programmatically
+ */
+const eval_run = tool({
+  description: `Run evalite evals programmatically and get structured results with scores.
+
+Use this to:
+- Run all evals in evals/ directory
+- Filter by specific eval suite (e.g., "coordinator", "compaction")
+- Enforce score thresholds for quality gates
+- Get per-suite and per-eval scores
+
+Returns structured JSON with:
+- success: boolean (true if all tests passed threshold)
+- totalSuites: number of eval suites run
+- totalEvals: number of individual test cases
+- averageScore: 0-1 score across all suites
+- suites: array of suite results with scores
+
+Example usage:
+- Run all evals: eval_run()
+- Run coordinator evals: eval_run({ suiteFilter: "coordinator" })
+- Enforce 80% threshold: eval_run({ scoreThreshold: 80 })`,
+
+  args: {
+    suiteFilter: tool.schema
+      .string()
+      .optional()
+      .describe(
+        'Optional filter to run specific eval suite (e.g., "coordinator", "compaction"). Matches against eval file paths using substring matching.'
+      ),
+    scoreThreshold: tool.schema
+      .number()
+      .optional()
+      .describe(
+        "Optional minimum average score threshold (0-100). If average score falls below this, result.success will be false. Useful for CI quality gates."
+      ),
+    includeDetailedResults: tool.schema
+      .boolean()
+      .optional()
+      .describe(
+        "Include individual eval results with input/output/scores in response. Set to false (default) for summary only to save token usage."
+      ),
+  },
+
+  execute: async (args) => {
+    const result = await runEvals({
+      cwd: process.cwd(),
+      suiteFilter: args.suiteFilter as string | undefined,
+      scoreThreshold: args.scoreThreshold as number | undefined,
+    });
+
+    // Remove detailed evals if not requested (saves tokens)
+    const includeDetails = args.includeDetailedResults === true;
+    if (!includeDetails) {
+      for (const suite of result.suites) {
+        delete suite.evals;
+      }
+    }
+
+    return JSON.stringify(result, null, 2);
+  },
+});
+
+/**
+ * All eval tools exported for registration
+ */
+export const evalTools = {
+  eval_run,
+} as const;

package/src/hive.ts

@@ -765,6 +765,40 @@ export const hive_create_epic = tool({
             error,
           );
         }
+
+        // Capture decomposition_complete event for eval scoring
+        try {
+          const { captureCoordinatorEvent } = await import("./eval-capture.js");
+          
+          // Build files_per_subtask map (indexed by subtask index)
+          const filesPerSubtask: Record<number, string[]> = {};
+          validated.subtasks.forEach((subtask, index) => {
+            if (subtask.files && subtask.files.length > 0) {
+              filesPerSubtask[index] = subtask.files;
+            }
+          });
+
+          captureCoordinatorEvent({
+            session_id: ctx.sessionID || "unknown",
+            epic_id: epic.id,
+            timestamp: new Date().toISOString(),
+            event_type: "DECISION",
+            decision_type: "decomposition_complete",
+            payload: {
+              subtask_count: validated.subtasks.length,
+              strategy_used: args.strategy || "feature-based",
+              files_per_subtask: filesPerSubtask,
+              epic_title: validated.epic_title,
+              task: args.task,
+            },
+          });
+        } catch (error) {
+          // Non-fatal - log and continue
+          console.warn(
+            "[hive_create_epic] Failed to capture decomposition_complete event:",
+            error,
+          );
+        }
       }
 
       // Sync cells to JSONL so spawned workers can see them immediately

package/src/index.ts

@@ -49,6 +49,7 @@ import { mandateTools } from "./mandates";
 import { memoryTools } from "./memory-tools";
 import { observabilityTools } from "./observability-tools";
 import { researchTools } from "./swarm-research";
+import { evalTools } from "./eval-runner";
 import {
   guardrailOutput,
   DEFAULT_GUARDRAIL_CONFIG,
@@ -175,6 +176,7 @@ const SwarmPlugin: Plugin = async (
       ...memoryTools,
       ...observabilityTools,
       ...researchTools,
+      ...evalTools,
     },
 
     /**
@@ -213,7 +215,7 @@ const SwarmPlugin: Plugin = async (
       if (isInCoordinatorContext()) {
         const ctx = getCoordinatorContext();
         const violation = detectCoordinatorViolation({
-          sessionId: ctx.sessionId || "unknown",
+          sessionId: input.sessionID || "unknown",
           epicId: ctx.epicId || "unknown",
           toolName,
           toolArgs: output.args as Record<string, unknown>,
@@ -729,6 +731,8 @@ export {
  * Includes:
  * - SWARM_COMPACTION_CONTEXT - Prompt text for swarm state preservation
  * - createCompactionHook - Factory function for the compaction hook
+ * - scanSessionMessages - Scan session for swarm state
+ * - ScannedSwarmState - Scanned state interface
  *
  * Usage:
  * ```typescript
@@ -739,7 +743,56 @@ export {
  * };
  * ```
  */
-export { SWARM_COMPACTION_CONTEXT, createCompactionHook } from "./compaction-hook";
+export { 
+  SWARM_COMPACTION_CONTEXT, 
+  createCompactionHook,
+  scanSessionMessages,
+  type ScannedSwarmState,
+} from "./compaction-hook";
+
+/**
+ * Re-export compaction-observability module
+ *
+ * Includes:
+ * - CompactionPhase - Enum of compaction phases
+ * - createMetricsCollector - Create a metrics collector
+ * - recordPhaseStart, recordPhaseComplete - Phase timing
+ * - recordPatternExtracted, recordPatternSkipped - Pattern tracking
+ * - getMetricsSummary - Get metrics summary
+ *
+ * Types:
+ * - CompactionMetrics - Mutable metrics collector
+ * - CompactionMetricsSummary - Read-only summary snapshot
+ *
+ * Features:
+ * - Phase timing breakdown (START, GATHER, DETECT, INJECT, COMPLETE)
+ * - Pattern extraction tracking with reasons
+ * - Success rate calculation
+ * - Debug mode for verbose details
+ * - JSON serializable for persistence
+ *
+ * Usage:
+ * ```typescript
+ * import { createMetricsCollector, CompactionPhase, recordPhaseStart } from "opencode-swarm-plugin";
+ *
+ * const metrics = createMetricsCollector({ session_id: "abc123" });
+ * recordPhaseStart(metrics, CompactionPhase.DETECT);
+ * // ... work ...
+ * recordPhaseComplete(metrics, CompactionPhase.DETECT);
+ * const summary = getMetricsSummary(metrics);
+ * ```
+ */
+export {
+  CompactionPhase,
+  createMetricsCollector,
+  recordPhaseStart,
+  recordPhaseComplete,
+  recordPatternExtracted,
+  recordPatternSkipped,
+  getMetricsSummary,
+  type CompactionMetrics,
+  type CompactionMetricsSummary,
+} from "./compaction-observability";
 
 /**
  * Re-export memory module
@@ -771,6 +824,66 @@ export {
 } from "./memory-tools";
 export type { Memory, SearchResult, SearchOptions } from "swarm-mail";
 
+/**
+ * Re-export eval-history module
+ *
+ * Includes:
+ * - recordEvalRun - Record eval run to JSONL history
+ * - getScoreHistory - Get score history for a specific eval
+ * - getPhase - Get current phase based on run count and variance
+ * - calculateVariance - Calculate statistical variance of scores
+ * - ensureEvalHistoryDir - Ensure history directory exists
+ * - getEvalHistoryPath - Get path to eval history file
+ *
+ * Constants:
+ * - DEFAULT_EVAL_HISTORY_PATH - Default path (.opencode/eval-history.jsonl)
+ * - VARIANCE_THRESHOLD - Variance threshold for production phase (0.1)
+ * - BOOTSTRAP_THRESHOLD - Run count for bootstrap phase (10)
+ * - STABILIZATION_THRESHOLD - Run count for stabilization phase (50)
+ *
+ * Types:
+ * - Phase - Progressive phases (bootstrap | stabilization | production)
+ * - EvalRunRecord - Single eval run record
+ */
+export {
+  recordEvalRun,
+  getScoreHistory,
+  getPhase,
+  calculateVariance,
+  ensureEvalHistoryDir,
+  getEvalHistoryPath,
+  DEFAULT_EVAL_HISTORY_PATH,
+  VARIANCE_THRESHOLD,
+  BOOTSTRAP_THRESHOLD,
+  STABILIZATION_THRESHOLD,
+  type Phase,
+  type EvalRunRecord,
+} from "./eval-history";
+
+/**
+ * Re-export eval-gates module
+ *
+ * Includes:
+ * - checkGate - Check if current score passes quality gate
+ * - DEFAULT_THRESHOLDS - Default regression thresholds by phase
+ *
+ * Types:
+ * - GateResult - Result from gate check
+ * - GateConfig - Configuration for gate thresholds
+ *
+ * Features:
+ * - Phase-based regression thresholds (Bootstrap: none, Stabilization: 10%, Production: 5%)
+ * - Configurable thresholds via GateConfig
+ * - Clear pass/fail messages with baseline comparison
+ * - Handles edge cases (division by zero, no history)
+ */
+export {
+  checkGate,
+  DEFAULT_THRESHOLDS,
+  type GateResult,
+  type GateConfig,
+} from "./eval-gates";
+
 /**
  * Re-export logger infrastructure
  *

package/src/memory.test.ts

@@ -120,6 +120,116 @@ describe("memory adapter", () => {
 			}
 		});
 	});
+
+	describe("upsert", () => {
+		test("returns ADD operation for new memory", async () => {
+			const result = await adapter.upsert({
+				information: "Completely new information about quantum computing",
+				tags: "quantum,physics",
+			});
+
+			expect(result.operation).toBe("ADD");
+			expect(result.reason).toBeDefined();
+			expect(result.memoryId).toBeDefined();
+			expect(result.memoryId).toMatch(/^mem_/);
+		});
+
+		test("returns UPDATE operation when refining existing memory", async () => {
+			// Store initial memory
+			await adapter.store({
+				information: "OAuth tokens need buffer",
+				tags: "auth",
+			});
+
+			// Try to upsert refined version
+			const result = await adapter.upsert({
+				information: "OAuth refresh tokens need 5min buffer before expiry to avoid race conditions",
+				tags: "auth,oauth,tokens",
+			});
+
+			expect(result.operation).toBe("UPDATE");
+			expect(result.reason).toBeDefined();
+			expect(result.memoryId).toBeDefined();
+		});
+
+		test("returns NOOP operation when information already exists", async () => {
+			// Store a memory
+			await adapter.store({
+				information: "Next.js 16 requires Suspense for Cache Components",
+				tags: "nextjs",
+			});
+
+			// Try to upsert same information
+			const result = await adapter.upsert({
+				information: "Next.js 16 requires Suspense for Cache Components",
+				tags: "nextjs",
+			});
+
+			expect(result.operation).toBe("NOOP");
+			expect(result.reason).toContain("already captured");
+		});
+
+		test("autoTag generates tags when enabled", async () => {
+			const result = await adapter.upsert({
+				information: "TypeScript interfaces are better than type aliases for object types",
+				autoTag: true,
+			});
+
+			expect(result.autoTags).toBeDefined();
+			expect(result.autoTags?.tags).toBeInstanceOf(Array);
+			expect(result.autoTags?.tags.length).toBeGreaterThan(0);
+		});
+
+		test("autoLink creates links when enabled", async () => {
+			// Store a base memory
+			await adapter.store({
+				information: "TypeScript supports structural typing",
+				tags: "typescript",
+			});
+
+			// Upsert related memory with autoLink
+			const result = await adapter.upsert({
+				information: "TypeScript interfaces use structural typing for shape matching",
+				autoLink: true,
+			});
+
+			if (result.linksCreated !== undefined) {
+				expect(result.linksCreated).toBeGreaterThanOrEqual(0);
+			}
+		});
+
+		test("extractEntities extracts entities when enabled", async () => {
+			const result = await adapter.upsert({
+				information: "React 19 introduces Server Components for Next.js 15",
+				extractEntities: true,
+			});
+
+			if (result.entitiesExtracted !== undefined) {
+				expect(result.entitiesExtracted).toBeGreaterThanOrEqual(0);
+			}
+		});
+
+		test("respects collection parameter", async () => {
+			const result = await adapter.upsert({
+				information: "Test memory in custom collection",
+				collection: "test-collection",
+			});
+
+			expect(result.memoryId).toBeDefined();
+			// Verify memory was stored in correct collection
+			const memory = await adapter.get({ id: result.memoryId! });
+			expect(memory?.collection).toBe("test-collection");
+		});
+
+		test("handles errors gracefully when information is missing", async () => {
+			await expect(async () => {
+				await (adapter.upsert as any)({
+					// Missing required information field
+					tags: "test",
+				});
+			}).toThrow();
+		});
+	});
 });
 
 describe("auto-migration on createMemoryAdapter", () => {