opencode-swarm-plugin 0.38.0 → 0.39.1

package/src/eval-learning.ts

@@ -0,0 +1,360 @@
+/**
+ * Eval-to-Learning Feedback Loop
+ *
+ * Automatically stores eval failures to semantic memory for learning.
+ * When eval scores drop significantly from rolling average (default >15%),
+ * stores context to semantic-memory with tags for future prompt generation.
+ *
+ * ## Usage
+ *
+ * ```typescript
+ * import { learnFromEvalFailure } from "./eval-learning";
+ * import { getMemoryAdapter } from "./memory-tools";
+ * import { getScoreHistory } from "./eval-history";
+ *
+ * const memoryAdapter = await getMemoryAdapter();
+ * const history = getScoreHistory(projectPath, "compaction-test");
+ *
+ * const result = await learnFromEvalFailure(
+ *   "compaction-test",
+ *   currentScore,
+ *   history,
+ *   memoryAdapter
+ * );
+ *
+ * if (result.triggered) {
+ *   console.log(`📉 Regression detected: ${(result.drop_percentage * 100).toFixed(1)}% drop`);
+ *   console.log(`Memory ID: ${result.memory_id}`);
+ * }
+ * ```
+ *
+ * ## Integration Points
+ *
+ * - **After each eval run**: Call to detect regressions automatically
+ * - **Memory tags**: `eval-failure`, `{eval-name}`, `regression`
+ * - **Future prompts**: Query memories with these tags for context
+ * - **Scorer context**: Optional detail about which scorer failed
+ *
+ * ## Customization
+ *
+ * ```typescript
+ * const customConfig = {
+ *   dropThreshold: 0.10,  // 10% threshold (more sensitive)
+ *   windowSize: 10,        // Last 10 runs for baseline
+ * };
+ *
+ * await learnFromEvalFailure(
+ *   "test",
+ *   score,
+ *   history,
+ *   adapter,
+ *   { config: customConfig }
+ * );
+ * ```
+ *
+ * @module eval-learning
+ */
+
+import type { EvalRunRecord } from "./eval-history";
+import type { MemoryAdapter } from "./memory-tools";
+
+// ============================================================================
+// Configuration
+// ============================================================================
+
+/**
+ * Configuration for eval-to-learning feedback
+ */
+export interface EvalLearningConfig {
+	/** Threshold for significant drop (0-1, default 0.15 = 15%) */
+	dropThreshold: number;
+	/** Rolling average window size (default 5 runs) */
+	windowSize: number;
+}
+
+/**
+ * Default configuration
+ */
+export const DEFAULT_EVAL_LEARNING_CONFIG: EvalLearningConfig = {
+	dropThreshold: 0.15, // 15% drop triggers storage
+	windowSize: 5, // Last 5 runs for baseline
+};
+
+// ============================================================================
+// Result Types
+// ============================================================================
+
+/**
+ * Result from learning check
+ */
+export interface LearningResult {
+	/** Whether the check triggered memory storage */
+	triggered: boolean;
+	/** Baseline score from rolling average */
+	baseline: number;
+	/** Current score */
+	current: number;
+	/** Drop percentage (0-1, e.g., 0.20 = 20% drop) */
+	drop_percentage: number;
+	/** Memory ID if stored */
+	memory_id?: string;
+}
+
+// ============================================================================
+// Core Functions
+// ============================================================================
+
+/**
+ * Calculate rolling average of recent scores
+ *
+ * Uses last N runs (default 5) to establish baseline.
+ * If history shorter than window, uses all available.
+ *
+ * @param history - Score history (chronological order)
+ * @param windowSize - Number of recent runs to average (default 5)
+ * @returns Average score (0 if empty)
+ */
+export function calculateRollingAverage(
+	history: EvalRunRecord[],
+	windowSize: number = 5,
+): number {
+	if (history.length === 0) {
+		return 0;
+	}
+
+	const recentRuns = history.slice(-windowSize);
+	const sum = recentRuns.reduce((acc, run) => acc + run.score, 0);
+	return sum / recentRuns.length;
+}
+
+/**
+ * Check if current score is a significant drop from baseline
+ *
+ * Significant = drop exceeds threshold (default 15%).
+ * Formula: (baseline - current) / baseline >= threshold
+ *
+ * @param currentScore - Current eval score
+ * @param baseline - Baseline score (rolling average)
+ * @param threshold - Drop threshold (default 0.15 = 15%)
+ * @returns True if drop is significant
+ */
+export function isSignificantDrop(
+	currentScore: number,
+	baseline: number,
+	threshold: number = 0.15,
+): boolean {
+	// Avoid division by zero
+	if (baseline === 0) {
+		return false;
+	}
+
+	const drop = (baseline - currentScore) / baseline;
+	return drop >= threshold;
+}
+
+/**
+ * Format failure context for semantic memory storage
+ *
+ * Creates human-readable description of the failure with
+ * quantified metrics and optional scorer context.
+ *
+ * @param evalName - Name of eval that failed
+ * @param currentScore - Current score
+ * @param baseline - Baseline score
+ * @param scorerContext - Optional context about which scorer failed
+ * @returns Formatted context string
+ */
+export function formatFailureContext(
+	evalName: string,
+	currentScore: number,
+	baseline: number,
+	scorerContext?: string,
+): string {
+	const dropPercentage =
+		baseline > 0 ? ((baseline - currentScore) / baseline) * 100 : 0;
+
+	const lines = [
+		`Eval "${evalName}" regression detected:`,
+		`- Current score: ${currentScore.toFixed(2)}`,
+		`- Baseline (rolling avg): ${baseline.toFixed(2)}`,
+		`- Drop: ${dropPercentage.toFixed(1)}%`,
+	];
+
+	if (scorerContext) {
+		lines.push("", "Scorer context:", scorerContext);
+	}
+
+	lines.push(
+		"",
+		"Action: Review recent changes that may have caused regression.",
+		"Query this memory when generating future prompts for this eval.",
+	);
+
+	return lines.join("\n");
+}
+
+/**
+ * Main learning function - automatically stores eval failures to semantic memory
+ *
+ * **Closed-loop learning**: When eval scores drop significantly from baseline,
+ * this function stores failure context to semantic memory. Future prompt generation
+ * queries these memories for context, preventing repeated mistakes.
+ *
+ * **Trigger condition**: Score drops >15% (default) from rolling average baseline.
+ * Uses last 5 runs (default) to establish baseline, not just previous run.
+ *
+ * **What gets stored**:
+ * - Eval name, baseline score, current score, drop percentage
+ * - Scorer-specific context (which scorer failed, why)
+ * - Timestamp and metadata for querying
+ * - Tags: `eval-failure`, `{eval-name}`, `regression`
+ *
+ * **Future use**: Before generating prompts for the same eval, query semantic memory
+ * with tags to inject learnings from past failures.
+ *
+ * **Integration points**:
+ * - After each eval run (in evalite runner or CI)
+ * - In `checkGate()` when regression detected
+ * - Manual calls for custom eval tracking
+ *
+ * @param evalName - Name of eval (e.g., "compaction-test", "coordinator-behavior")
+ * @param currentScore - Current eval score (typically 0-1 range)
+ * @param history - Score history in chronological order (oldest first)
+ * @param memoryAdapter - Semantic memory adapter (from `getMemoryAdapter()`)
+ * @param options - Optional config (thresholds, window size) and scorer context
+ * @param options.config - Custom thresholds (dropThreshold, windowSize)
+ * @param options.scorerContext - Details about which scorer failed (for context)
+ * @returns Learning result with trigger status, baseline, drop percentage, memory ID
+ *
+ * @example
+ * ```typescript
+ * import { learnFromEvalFailure } from "./eval-learning.js";
+ * import { getMemoryAdapter } from "./memory-tools.js";
+ * import { getScoreHistory } from "./eval-history.js";
+ *
+ * const memoryAdapter = await getMemoryAdapter();
+ * const history = getScoreHistory("/path/to/project", "coordinator-behavior");
+ *
+ * const result = await learnFromEvalFailure(
+ *   "coordinator-behavior",
+ *   0.68,  // Current score
+ *   history,
+ *   memoryAdapter,
+ *   { scorerContext: "violationCount: 5 violations (coordinator edited files)" }
+ * );
+ *
+ * if (result.triggered) {
+ *   console.log(`📉 Regression detected: ${(result.drop_percentage * 100).toFixed(1)}% drop`);
+ *   console.log(`Stored to memory: ${result.memory_id}`);
+ * }
+ * ```
+ *
+ * @example
+ * ```typescript
+ * // Custom threshold (more sensitive)
+ * const result = await learnFromEvalFailure(
+ *   "critical-eval",
+ *   0.85,
+ *   history,
+ *   memoryAdapter,
+ *   {
+ *     config: {
+ *       dropThreshold: 0.10,  // 10% threshold (default is 15%)
+ *       windowSize: 10,       // Last 10 runs for baseline (default is 5)
+ *     },
+ *   }
+ * );
+ * ```
+ */
+export async function learnFromEvalFailure(
+	evalName: string,
+	currentScore: number,
+	history: EvalRunRecord[],
+	memoryAdapter: MemoryAdapter,
+	options?: {
+		config?: EvalLearningConfig;
+		scorerContext?: string;
+	},
+): Promise<LearningResult> {
+	const config = options?.config ?? DEFAULT_EVAL_LEARNING_CONFIG;
+
+	// Calculate baseline from rolling average
+	const baseline = calculateRollingAverage(history, config.windowSize);
+
+	// Check if this is a significant drop
+	const dropPercentage =
+		baseline > 0 ? (baseline - currentScore) / baseline : 0;
+	const significant = isSignificantDrop(
+		currentScore,
+		baseline,
+		config.dropThreshold,
+	);
+
+	const result: LearningResult = {
+		triggered: significant,
+		baseline,
+		current: currentScore,
+		drop_percentage: dropPercentage,
+	};
+
+	// Store to semantic memory if significant
+	if (significant) {
+		const information = formatFailureContext(
+			evalName,
+			currentScore,
+			baseline,
+			options?.scorerContext,
+		);
+
+		const tags = ["eval-failure", evalName, "regression"].join(",");
+
+		const metadata = JSON.stringify({
+			eval_name: evalName,
+			baseline_score: baseline,
+			current_score: currentScore,
+			drop_percentage: dropPercentage,
+			timestamp: new Date().toISOString(),
+		});
+
+		const storeResult = await memoryAdapter.store({
+			information,
+			tags,
+			metadata,
+		});
+
+		if (storeResult.id) {
+			result.memory_id = storeResult.id;
+		}
+	}
+
+	return result;
+}
+
+// ============================================================================
+// Convenience Exports
+// ============================================================================
+
+/**
+ * Create custom learning config with specific threshold
+ *
+ * Helper for common use case: custom drop threshold.
+ *
+ * @param dropThreshold - Drop threshold (0-1)
+ * @param windowSize - Optional window size (default 5)
+ * @returns Custom config
+ *
+ * @example
+ * ```typescript
+ * const config = createLearningConfig(0.10); // 10% threshold
+ * await learnFromEvalFailure("test", score, history, adapter, { config });
+ * ```
+ */
+export function createLearningConfig(
+	dropThreshold: number,
+	windowSize?: number,
+): EvalLearningConfig {
+	return {
+		dropThreshold,
+		windowSize: windowSize ?? DEFAULT_EVAL_LEARNING_CONFIG.windowSize,
+	};
+}

package/src/index.ts

@@ -213,7 +213,7 @@ const SwarmPlugin: Plugin = async (
       if (isInCoordinatorContext()) {
         const ctx = getCoordinatorContext();
         const violation = detectCoordinatorViolation({
-          sessionId: ctx.sessionId || "unknown",
+          sessionId: input.sessionID || "unknown",
           epicId: ctx.epicId || "unknown",
           toolName,
           toolArgs: output.args as Record<string, unknown>,
@@ -771,6 +771,66 @@ export {
 } from "./memory-tools";
 export type { Memory, SearchResult, SearchOptions } from "swarm-mail";
 
+/**
+ * Re-export eval-history module
+ *
+ * Includes:
+ * - recordEvalRun - Record eval run to JSONL history
+ * - getScoreHistory - Get score history for a specific eval
+ * - getPhase - Get current phase based on run count and variance
+ * - calculateVariance - Calculate statistical variance of scores
+ * - ensureEvalHistoryDir - Ensure history directory exists
+ * - getEvalHistoryPath - Get path to eval history file
+ *
+ * Constants:
+ * - DEFAULT_EVAL_HISTORY_PATH - Default path (.opencode/eval-history.jsonl)
+ * - VARIANCE_THRESHOLD - Variance threshold for production phase (0.1)
+ * - BOOTSTRAP_THRESHOLD - Run count for bootstrap phase (10)
+ * - STABILIZATION_THRESHOLD - Run count for stabilization phase (50)
+ *
+ * Types:
+ * - Phase - Progressive phases (bootstrap | stabilization | production)
+ * - EvalRunRecord - Single eval run record
+ */
+export {
+  recordEvalRun,
+  getScoreHistory,
+  getPhase,
+  calculateVariance,
+  ensureEvalHistoryDir,
+  getEvalHistoryPath,
+  DEFAULT_EVAL_HISTORY_PATH,
+  VARIANCE_THRESHOLD,
+  BOOTSTRAP_THRESHOLD,
+  STABILIZATION_THRESHOLD,
+  type Phase,
+  type EvalRunRecord,
+} from "./eval-history";
+
+/**
+ * Re-export eval-gates module
+ *
+ * Includes:
+ * - checkGate - Check if current score passes quality gate
+ * - DEFAULT_THRESHOLDS - Default regression thresholds by phase
+ *
+ * Types:
+ * - GateResult - Result from gate check
+ * - GateConfig - Configuration for gate thresholds
+ *
+ * Features:
+ * - Phase-based regression thresholds (Bootstrap: none, Stabilization: 10%, Production: 5%)
+ * - Configurable thresholds via GateConfig
+ * - Clear pass/fail messages with baseline comparison
+ * - Handles edge cases (division by zero, no history)
+ */
+export {
+  checkGate,
+  DEFAULT_THRESHOLDS,
+  type GateResult,
+  type GateConfig,
+} from "./eval-gates";
+
 /**
  * Re-export logger infrastructure
  *

package/src/post-compaction-tracker.test.ts

@@ -0,0 +1,251 @@
+/**
+ * Post-Compaction Tool Call Tracker Tests
+ *
+ * TDD: RED → GREEN → REFACTOR
+ *
+ * Tests tracking of tool calls after compaction resumption.
+ * Emits resumption_started on first tool call, then tool_call_tracked for each call (max 20).
+ * Detects coordinator violations: Edit, Write, swarmmail_reserve are forbidden.
+ */
+import { describe, test, expect, beforeEach, mock } from "bun:test";
+import {
+  createPostCompactionTracker,
+  type PostCompactionTracker,
+  type ToolCallEvent,
+} from "./post-compaction-tracker";
+
+describe("PostCompactionTracker - TDD", () => {
+  let tracker: PostCompactionTracker;
+  let mockCapture: ReturnType<typeof mock>;
+
+  beforeEach(() => {
+    mockCapture = mock((event: any) => {});
+    tracker = createPostCompactionTracker({
+      sessionId: "test-session",
+      epicId: "mjkwehsqnbm",
+      onEvent: mockCapture,
+    });
+  });
+
+  // ============================================================================
+  // RED: Test resumption_started event
+  // ============================================================================
+
+  test("emits resumption_started on first tool call", () => {
+    const toolCall: ToolCallEvent = {
+      tool: "read",
+      args: { filePath: "/test/file.ts" },
+      timestamp: Date.now(),
+    };
+
+    tracker.trackToolCall(toolCall);
+
+    expect(mockCapture).toHaveBeenCalledTimes(2); // resumption_started + tool_call_tracked
+    const firstCall = mockCapture.mock.calls[0][0];
+    expect(firstCall.compaction_type).toBe("resumption_started");
+    expect(firstCall.payload.session_id).toBe("test-session");
+    expect(firstCall.payload.epic_id).toBe("mjkwehsqnbm");
+  });
+
+  test("resumption_started only emitted once", () => {
+    tracker.trackToolCall({
+      tool: "read",
+      args: {},
+      timestamp: Date.now(),
+    });
+    tracker.trackToolCall({
+      tool: "glob",
+      args: {},
+      timestamp: Date.now(),
+    });
+
+    // First call: resumption_started + tool_call_tracked
+    // Second call: tool_call_tracked only
+    expect(mockCapture).toHaveBeenCalledTimes(3);
+
+    const calls = mockCapture.mock.calls;
+    expect(calls[0][0].compaction_type).toBe("resumption_started");
+    expect(calls[1][0].compaction_type).toBe("tool_call_tracked");
+    expect(calls[2][0].compaction_type).toBe("tool_call_tracked");
+  });
+
+  // ============================================================================
+  // RED: Test tool_call_tracked event
+  // ============================================================================
+
+  test("emits tool_call_tracked for each of first 20 calls", () => {
+    for (let i = 0; i < 20; i++) {
+      tracker.trackToolCall({
+        tool: `tool-${i}`,
+        args: {},
+        timestamp: Date.now(),
+      });
+    }
+
+    // First call: resumption_started + tool_call_tracked = 2
+    // Next 19 calls: tool_call_tracked only = 19
+    // Total: 21 events (1 resumption_started + 20 tool_call_tracked)
+    expect(mockCapture).toHaveBeenCalledTimes(21);
+
+    const trackedEvents = mockCapture.mock.calls.filter(
+      (call: any) => call[0].compaction_type === "tool_call_tracked",
+    );
+    expect(trackedEvents).toHaveLength(20);
+  });
+
+  test("tool_call_tracked includes tool name and args", () => {
+    tracker.trackToolCall({
+      tool: "edit",
+      args: { filePath: "/test.ts", oldString: "foo", newString: "bar" },
+      timestamp: Date.now(),
+    });
+
+    const trackedEvent = mockCapture.mock.calls.find(
+      (call: any) => call[0].compaction_type === "tool_call_tracked",
+    )?.[0];
+
+    expect(trackedEvent).toBeDefined();
+    expect(trackedEvent.payload.tool).toBe("edit");
+    expect(trackedEvent.payload.args.filePath).toBe("/test.ts");
+    expect(trackedEvent.payload.call_number).toBe(1);
+  });
+
+  // ============================================================================
+  // RED: Test coordinator violation detection
+  // ============================================================================
+
+  test("detects Edit as coordinator violation", () => {
+    tracker.trackToolCall({
+      tool: "edit",
+      args: { filePath: "/test.ts", oldString: "a", newString: "b" },
+      timestamp: Date.now(),
+    });
+
+    const trackedEvent = mockCapture.mock.calls.find(
+      (call: any) => call[0].compaction_type === "tool_call_tracked",
+    )?.[0];
+
+    expect(trackedEvent.payload.is_coordinator_violation).toBe(true);
+    expect(trackedEvent.payload.violation_reason).toBe(
+      "Coordinators NEVER edit files - spawn worker instead",
+    );
+  });
+
+  test("detects Write as coordinator violation", () => {
+    tracker.trackToolCall({
+      tool: "write",
+      args: { filePath: "/new.ts", content: "export {}" },
+      timestamp: Date.now(),
+    });
+
+    const trackedEvent = mockCapture.mock.calls.find(
+      (call: any) => call[0].compaction_type === "tool_call_tracked",
+    )?.[0];
+
+    expect(trackedEvent.payload.is_coordinator_violation).toBe(true);
+    expect(trackedEvent.payload.violation_reason).toBe(
+      "Coordinators NEVER write files - spawn worker instead",
+    );
+  });
+
+  test("detects swarmmail_reserve as coordinator violation", () => {
+    tracker.trackToolCall({
+      tool: "swarmmail_reserve",
+      args: { paths: ["/src/**"], reason: "test" },
+      timestamp: Date.now(),
+    });
+
+    const trackedEvent = mockCapture.mock.calls.find(
+      (call: any) => call[0].compaction_type === "tool_call_tracked",
+    )?.[0];
+
+    expect(trackedEvent.payload.is_coordinator_violation).toBe(true);
+    expect(trackedEvent.payload.violation_reason).toBe(
+      "Coordinators NEVER reserve files - workers reserve files",
+    );
+  });
+
+  test("does not flag Read as violation", () => {
+    tracker.trackToolCall({
+      tool: "read",
+      args: { filePath: "/test.ts" },
+      timestamp: Date.now(),
+    });
+
+    const trackedEvent = mockCapture.mock.calls.find(
+      (call: any) => call[0].compaction_type === "tool_call_tracked",
+    )?.[0];
+
+    expect(trackedEvent.payload.is_coordinator_violation).toBe(false);
+    expect(trackedEvent.payload.violation_reason).toBeUndefined();
+  });
+
+  test("does not flag swarm_spawn_subtask as violation", () => {
+    tracker.trackToolCall({
+      tool: "swarm_spawn_subtask",
+      args: { bead_id: "bd-123", subtask_title: "Test" },
+      timestamp: Date.now(),
+    });
+
+    const trackedEvent = mockCapture.mock.calls.find(
+      (call: any) => call[0].compaction_type === "tool_call_tracked",
+    )?.[0];
+
+    expect(trackedEvent.payload.is_coordinator_violation).toBe(false);
+  });
+
+  // ============================================================================
+  // RED: Test tracking stops after 20 calls
+  // ============================================================================
+
+  test("stops tracking after 20 calls", () => {
+    for (let i = 0; i < 25; i++) {
+      tracker.trackToolCall({
+        tool: `tool-${i}`,
+        args: {},
+        timestamp: Date.now(),
+      });
+    }
+
+    // Should only track first 20: 1 resumption_started + 20 tool_call_tracked
+    expect(mockCapture).toHaveBeenCalledTimes(21);
+  });
+
+  test("returns tracking status", () => {
+    expect(tracker.isTracking()).toBe(true);
+
+    for (let i = 0; i < 20; i++) {
+      tracker.trackToolCall({
+        tool: `tool-${i}`,
+        args: {},
+        timestamp: Date.now(),
+      });
+    }
+
+    expect(tracker.isTracking()).toBe(false);
+  });
+
+  // ============================================================================
+  // RED: Test configurable limit
+  // ============================================================================
+
+  test("respects custom call limit", () => {
+    const customTracker = createPostCompactionTracker({
+      sessionId: "test",
+      epicId: "test",
+      onEvent: mockCapture,
+      maxCalls: 5,
+    });
+
+    for (let i = 0; i < 10; i++) {
+      customTracker.trackToolCall({
+        tool: `tool-${i}`,
+        args: {},
+        timestamp: Date.now(),
+      });
+    }
+
+    // 1 resumption_started + 5 tool_call_tracked
+    expect(mockCapture).toHaveBeenCalledTimes(6);
+  });
+});