opencode-swarm-plugin 0.43.0 → 0.44.1

package/evals/lib/data-loader.test.ts

@@ -1,345 +0,0 @@
-/**
- * Tests for data-loader quality filters
- *
- * TDD approach: RED → GREEN → REFACTOR
- */
-import { afterEach, beforeEach, describe, expect, test } from "bun:test";
-import * as fs from "node:fs";
-import * as os from "node:os";
-import * as path from "node:path";
-import type { CoordinatorEvent } from "../../src/eval-capture.js";
-import { loadCapturedSessions } from "./data-loader.js";
-
-// Test helper: create a temp session directory
-let tempSessionDir: string;
-
-beforeEach(() => {
-  tempSessionDir = fs.mkdtempSync(path.join(os.tmpdir(), "test-sessions-"));
-});
-
-afterEach(() => {
-  if (fs.existsSync(tempSessionDir)) {
-    fs.rmSync(tempSessionDir, { recursive: true });
-  }
-});
-
-/**
- * Helper: create a session JSONL file with events
- */
-function createSessionFile(
-  sessionId: string,
-  events: CoordinatorEvent[],
-): void {
-  const filePath = path.join(tempSessionDir, `${sessionId}.jsonl`);
-  const lines = events.map((e) => JSON.stringify(e)).join("\n") + "\n";
-  fs.writeFileSync(filePath, lines, "utf-8");
-}
-
-/**
- * Helper: create minimal events
- */
-function createEvent(
-  sessionId: string,
-  epicId: string,
-  type: "DECISION" | "VIOLATION" | "OUTCOME",
-  subtype: string,
-): CoordinatorEvent {
-  const base = {
-    session_id: sessionId,
-    epic_id: epicId,
-    timestamp: new Date().toISOString(),
-    payload: {},
-  };
-
-  if (type === "DECISION") {
-    return {
-      ...base,
-      event_type: "DECISION" as const,
-      decision_type: subtype as any,
-    };
-  } else if (type === "VIOLATION") {
-    return {
-      ...base,
-      event_type: "VIOLATION" as const,
-      violation_type: subtype as any,
-    };
-  } else {
-    return {
-      ...base,
-      event_type: "OUTCOME" as const,
-      outcome_type: subtype as any,
-    };
-  }
-}
-
-describe("loadCapturedSessions - quality filters", () => {
-  test("filters out sessions with fewer than minEvents (default: 3)", async () => {
-    // Create sessions with different event counts
-    createSessionFile("session-2-events", [
-      createEvent("session-2-events", "epic-1", "DECISION", "worker_spawned"),
-      createEvent("session-2-events", "epic-1", "OUTCOME", "subtask_success"),
-    ]);
-
-    createSessionFile("session-3-events", [
-      createEvent("session-3-events", "epic-2", "DECISION", "worker_spawned"),
-      createEvent("session-3-events", "epic-2", "DECISION", "review_completed"),
-      createEvent("session-3-events", "epic-2", "OUTCOME", "subtask_success"),
-    ]);
-
-    createSessionFile("session-5-events", [
-      createEvent("session-5-events", "epic-3", "DECISION", "worker_spawned"),
-      createEvent("session-5-events", "epic-3", "DECISION", "review_completed"),
-      createEvent("session-5-events", "epic-3", "OUTCOME", "subtask_success"),
-      createEvent("session-5-events", "epic-3", "OUTCOME", "subtask_success"),
-      createEvent("session-5-events", "epic-3", "OUTCOME", "epic_complete"),
-    ]);
-
-    const sessions = await loadCapturedSessions({
-      minEvents: 3,
-      sessionDir: tempSessionDir,
-    });
-
-    // Should only get sessions with >= 3 events
-    expect(sessions.length).toBe(2);
-    expect(
-      sessions.some((s) => s.session.session_id === "session-3-events"),
-    ).toBe(true);
-    expect(
-      sessions.some((s) => s.session.session_id === "session-5-events"),
-    ).toBe(true);
-    expect(
-      sessions.some((s) => s.session.session_id === "session-2-events"),
-    ).toBe(false);
-  });
-
-  test("filters out sessions without worker_spawned event when requireWorkerSpawn=true", async () => {
-    // Session WITH worker_spawned
-    createSessionFile("session-with-spawn", [
-      createEvent("session-with-spawn", "epic-1", "DECISION", "worker_spawned"),
-      createEvent(
-        "session-with-spawn",
-        "epic-1",
-        "DECISION",
-        "review_completed",
-      ),
-      createEvent("session-with-spawn", "epic-1", "OUTCOME", "subtask_success"),
-    ]);
-
-    // Session WITHOUT worker_spawned
-    createSessionFile("session-no-spawn", [
-      createEvent(
-        "session-no-spawn",
-        "epic-2",
-        "DECISION",
-        "strategy_selected",
-      ),
-      createEvent(
-        "session-no-spawn",
-        "epic-2",
-        "DECISION",
-        "decomposition_complete",
-      ),
-      createEvent("session-no-spawn", "epic-2", "OUTCOME", "epic_complete"),
-    ]);
-
-    const sessions = await loadCapturedSessions({
-      requireWorkerSpawn: true,
-      sessionDir: tempSessionDir,
-    });
-
-    expect(sessions.length).toBe(1);
-    expect(sessions[0]?.session.session_id).toBe("session-with-spawn");
-  });
-
-  test("filters out sessions without review_completed event when requireReview=true", async () => {
-    // Session WITH review
-    createSessionFile("session-with-review", [
-      createEvent(
-        "session-with-review",
-        "epic-1",
-        "DECISION",
-        "worker_spawned",
-      ),
-      createEvent(
-        "session-with-review",
-        "epic-1",
-        "DECISION",
-        "review_completed",
-      ),
-      createEvent("session-with-review", "epic-1", "OUTCOME", "subtask_success"),
-    ]);
-
-    // Session WITHOUT review
-    createSessionFile("session-no-review", [
-      createEvent("session-no-review", "epic-2", "DECISION", "worker_spawned"),
-      createEvent("session-no-review", "epic-2", "OUTCOME", "subtask_success"),
-      createEvent("session-no-review", "epic-2", "OUTCOME", "epic_complete"),
-    ]);
-
-    const sessions = await loadCapturedSessions({
-      requireReview: true,
-      sessionDir: tempSessionDir,
-    });
-
-    expect(sessions.length).toBe(1);
-    expect(sessions[0]?.session.session_id).toBe("session-with-review");
-  });
-
-  test("allows disabling filters individually", async () => {
-    // Session with only 2 events, no worker_spawned, no review
-    createSessionFile("session-low-quality", [
-      createEvent(
-        "session-low-quality",
-        "epic-1",
-        "DECISION",
-        "strategy_selected",
-      ),
-      createEvent("session-low-quality", "epic-1", "OUTCOME", "epic_complete"),
-    ]);
-
-    // Disable all filters
-    const sessions = await loadCapturedSessions({
-      minEvents: 0,
-      requireWorkerSpawn: false,
-      requireReview: false,
-      sessionDir: tempSessionDir,
-    });
-
-    expect(sessions.length).toBe(1);
-    expect(sessions[0]?.session.session_id).toBe("session-low-quality");
-  });
-
-  test("applies limit AFTER filtering", async () => {
-    // Create 5 high-quality sessions
-    for (let i = 1; i <= 5; i++) {
-      createSessionFile(`session-${i}`, [
-        createEvent(`session-${i}`, `epic-${i}`, "DECISION", "worker_spawned"),
-        createEvent(
-          `session-${i}`,
-          `epic-${i}`,
-          "DECISION",
-          "review_completed",
-        ),
-        createEvent(`session-${i}`, `epic-${i}`, "OUTCOME", "subtask_success"),
-      ]);
-    }
-
-    // Create 3 low-quality sessions (will be filtered out)
-    for (let i = 6; i <= 8; i++) {
-      createSessionFile(`session-${i}`, [
-        createEvent(`session-${i}`, `epic-${i}`, "DECISION", "strategy_selected"),
-      ]);
-    }
-
-    // Filter first (remove 3 low-quality), then limit to 2
-    const sessions = await loadCapturedSessions({
-      minEvents: 3,
-      requireWorkerSpawn: true,
-      requireReview: true,
-      limit: 2,
-      sessionDir: tempSessionDir,
-    });
-
-    // Should get 2 sessions from the 5 high-quality ones
-    expect(sessions.length).toBe(2);
-    expect(sessions.every((s) => s.session.events.length >= 3)).toBe(true);
-  });
-
-  test("combines all filters correctly", async () => {
-    // High-quality session (passes all filters)
-    createSessionFile("session-high-quality", [
-      createEvent(
-        "session-high-quality",
-        "epic-1",
-        "DECISION",
-        "worker_spawned",
-      ),
-      createEvent(
-        "session-high-quality",
-        "epic-1",
-        "DECISION",
-        "review_completed",
-      ),
-      createEvent("session-high-quality", "epic-1", "OUTCOME", "subtask_success"),
-      createEvent("session-high-quality", "epic-1", "OUTCOME", "epic_complete"),
-    ]);
-
-    // Missing worker_spawned
-    createSessionFile("session-no-spawn", [
-      createEvent(
-        "session-no-spawn",
-        "epic-2",
-        "DECISION",
-        "review_completed",
-      ),
-      createEvent("session-no-spawn", "epic-2", "OUTCOME", "subtask_success"),
-      createEvent("session-no-spawn", "epic-2", "OUTCOME", "epic_complete"),
-    ]);
-
-    // Missing review_completed
-    createSessionFile("session-no-review", [
-      createEvent("session-no-review", "epic-3", "DECISION", "worker_spawned"),
-      createEvent("session-no-review", "epic-3", "OUTCOME", "subtask_success"),
-      createEvent("session-no-review", "epic-3", "OUTCOME", "epic_complete"),
-    ]);
-
-    // Too few events
-    createSessionFile("session-too-few", [
-      createEvent("session-too-few", "epic-4", "DECISION", "worker_spawned"),
-      createEvent("session-too-few", "epic-4", "DECISION", "review_completed"),
-    ]);
-
-    const sessions = await loadCapturedSessions({
-      minEvents: 3,
-      requireWorkerSpawn: true,
-      requireReview: true,
-      sessionDir: tempSessionDir,
-    });
-
-    // Only high-quality session should pass
-    expect(sessions.length).toBe(1);
-    expect(sessions[0]?.session.session_id).toBe("session-high-quality");
-  });
-
-  test("defaults are: minEvents=3, requireWorkerSpawn=true, requireReview=true", async () => {
-    // Create one session that meets defaults
-    createSessionFile("session-meets-defaults", [
-      createEvent(
-        "session-meets-defaults",
-        "epic-1",
-        "DECISION",
-        "worker_spawned",
-      ),
-      createEvent(
-        "session-meets-defaults",
-        "epic-1",
-        "DECISION",
-        "review_completed",
-      ),
-      createEvent(
-        "session-meets-defaults",
-        "epic-1",
-        "OUTCOME",
-        "subtask_success",
-      ),
-    ]);
-
-    // Create one that doesn't
-    createSessionFile("session-fails-defaults", [
-      createEvent(
-        "session-fails-defaults",
-        "epic-2",
-        "DECISION",
-        "strategy_selected",
-      ),
-    ]);
-
-    // Call with NO options except sessionDir - should use defaults
-    const sessions = await loadCapturedSessions({
-      sessionDir: tempSessionDir,
-    });
-
-    expect(sessions.length).toBe(1);
-    expect(sessions[0]?.session.session_id).toBe("session-meets-defaults");
-  });
-});

package/evals/lib/data-loader.ts

@@ -1,281 +0,0 @@
-/**
- * PGlite-backed eval data loader
- *
- * Loads real decomposition outcomes from the eval_records table
- * for use in Evalite evals.
- */
-import * as fs from "node:fs";
-import {
-  getEvalRecords,
-  getEvalStats,
-  type EvalRecord,
-} from "swarm-mail";
-
-export interface EvalCase {
-  input: { task: string; context?: string };
-  expected: {
-    minSubtasks: number;
-    maxSubtasks: number;
-    requiredFiles?: string[];
-    overallSuccess?: boolean;
-  };
-  actual?: EvalRecord;
-}
-
-/**
- * Load eval cases from PGlite
- *
- * @param projectKey - Project key for filtering records
- * @param options - Filter options
- * @returns Array of eval cases ready for Evalite
- */
-export async function loadEvalCases(
-  projectKey: string,
-  options?: {
-    limit?: number;
-    strategy?: "file-based" | "feature-based" | "risk-based";
-    successOnly?: boolean;
-    projectPath?: string;
-  },
-): Promise<EvalCase[]> {
-  const { limit, strategy, successOnly, projectPath } = options ?? {};
-
-  // Query eval records from PGlite
-  const records = await getEvalRecords(
-    projectKey,
-    { limit, strategy },
-    projectPath,
-  );
-
-  // Filter by success if requested
-  const filtered = successOnly
-    ? records.filter((r) => r.overall_success === true)
-    : records;
-
-  // Transform to EvalCase format
-  return filtered.map((record) => ({
-    input: {
-      task: record.task,
-      context: record.context ?? undefined,
-    },
-    expected: {
-      minSubtasks: 2,
-      maxSubtasks: record.subtasks.length,
-      requiredFiles: record.subtasks.flatMap((s) => s.files),
-      overallSuccess: record.overall_success ?? undefined,
-    },
-    actual: record,
-  }));
-}
-
-/**
- * Check if we have enough real data to run evals
- *
- * @param projectKey - Project key to check
- * @param minRecords - Minimum number of records required (default: 5)
- * @param projectPath - Optional project path for database lookup
- * @returns True if enough data exists
- */
-export async function hasRealEvalData(
-  projectKey: string,
-  minRecords: number = 5,
-  projectPath?: string,
-): Promise<boolean> {
-  const stats = await getEvalStats(projectKey, projectPath);
-  return stats.totalRecords >= minRecords;
-}
-
-/**
- * Get eval data stats for reporting
- *
- * @param projectKey - Project key to query
- * @param projectPath - Optional project path for database lookup
- * @returns Summary of available eval data
- */
-export async function getEvalDataSummary(
-  projectKey: string,
-  projectPath?: string,
-): Promise<{
-  totalRecords: number;
-  successRate: number;
-  byStrategy: Record<string, number>;
-  hasEnoughData: boolean;
-}> {
-  const stats = await getEvalStats(projectKey, projectPath);
-
-  return {
-    totalRecords: stats.totalRecords,
-    successRate: stats.successRate,
-    byStrategy: stats.byStrategy,
-    hasEnoughData: stats.totalRecords >= 5,
-  };
-}
-
-/**
- * Check if a session meets quality criteria
- */
-function meetsQualityCriteria(
-  session: import("../../src/eval-capture.js").CoordinatorSession,
-  criteria: {
-    minEvents: number;
-    requireWorkerSpawn: boolean;
-    requireReview: boolean;
-  },
-): boolean {
-  // Filter 1: minEvents
-  if (session.events.length < criteria.minEvents) {
-    return false;
-  }
-
-  // Filter 2: requireWorkerSpawn
-  if (
-    criteria.requireWorkerSpawn &&
-    !session.events.some(
-      (e) => e.event_type === "DECISION" && e.decision_type === "worker_spawned",
-    )
-  ) {
-    return false;
-  }
-
-  // Filter 3: requireReview
-  if (
-    criteria.requireReview &&
-    !session.events.some(
-      (e) =>
-        e.event_type === "DECISION" && e.decision_type === "review_completed",
-    )
-  ) {
-    return false;
-  }
-
-  return true;
-}
-
-/**
- * Load captured coordinator sessions from ~/.config/swarm-tools/sessions/
- *
- * Reads all JSONL session files and returns CoordinatorSession objects.
- *
- * Quality filters are applied to focus on high-signal coordinator sessions:
- * - minEvents: Filter out incomplete/aborted sessions (default: 3)
- * - requireWorkerSpawn: Ensure session delegated to workers (default: true)
- * - requireReview: Ensure coordinator reviewed work (default: true)
- *
- * Filters are applied BEFORE the limit for accurate sampling.
- *
- * @param options - Filter options
- * @returns Array of coordinator sessions that meet quality criteria
- */
-export async function loadCapturedSessions(options?: {
-  sessionIds?: string[];
-  limit?: number;
-  /** Minimum number of events required (default: 3) */
-  minEvents?: number;
-  /** Require at least one worker_spawned event (default: true) */
-  requireWorkerSpawn?: boolean;
-  /** Require at least one review_completed event (default: true) */
-  requireReview?: boolean;
-  /** Override session directory for testing */
-  sessionDir?: string;
-}): Promise<
-  Array<{ session: import("../../src/eval-capture.js").CoordinatorSession }>
-> {
-  const { getSessionDir, readSessionEvents, saveSession } = await import(
-    "../../src/eval-capture.js"
-  );
-  const sessionDir = options?.sessionDir ?? getSessionDir();
-
-  // Default quality filters
-  const qualityCriteria = {
-    minEvents: options?.minEvents ?? 3,
-    requireWorkerSpawn: options?.requireWorkerSpawn ?? true,
-    requireReview: options?.requireReview ?? true,
-  };
-
-  // If session dir doesn't exist, return empty
-  if (!fs.existsSync(sessionDir)) {
-    return [];
-  }
-
-  // Read all .jsonl files in session directory
-  const files = fs
-    .readdirSync(sessionDir)
-    .filter((f) => f.endsWith(".jsonl"));
-
-  // Filter by sessionIds if provided
-  const targetFiles = options?.sessionIds
-    ? files.filter((f) => options.sessionIds?.includes(f.replace(".jsonl", "")))
-    : files;
-
-  // Load each session
-  const sessions: Array<{
-    session: import("../../src/eval-capture.js").CoordinatorSession;
-  }> = [];
-  let filteredOutCount = 0;
-
-  for (const file of targetFiles) {
-    const sessionId = file.replace(".jsonl", "");
-
-    try {
-      let events: import("../../src/eval-capture.js").CoordinatorEvent[];
-
-      // If custom sessionDir, read directly; otherwise use eval-capture functions
-      if (options?.sessionDir) {
-        const sessionPath = `${sessionDir}/${sessionId}.jsonl`;
-        if (!fs.existsSync(sessionPath)) continue;
-
-        const content = fs.readFileSync(sessionPath, "utf-8");
-        const lines = content.trim().split("\n").filter(Boolean);
-        const { CoordinatorEventSchema } = await import(
-          "../../src/eval-capture.js"
-        );
-        events = lines.map((line) => {
-          const parsed = JSON.parse(line);
-          return CoordinatorEventSchema.parse(parsed);
-        });
-      } else {
-        events = readSessionEvents(sessionId);
-      }
-
-      if (events.length === 0) continue;
-
-      // Find epic_id from first event
-      const epicId = events[0]?.epic_id;
-      if (!epicId) continue;
-
-      // Build session object
-      const session: import("../../src/eval-capture.js").CoordinatorSession = {
-        session_id: sessionId,
-        epic_id: epicId,
-        start_time: events[0]?.timestamp ?? new Date().toISOString(),
-        end_time: events[events.length - 1]?.timestamp,
-        events,
-      };
-      if (!session) continue;
-
-      // Apply quality filters BEFORE limit
-      if (meetsQualityCriteria(session, qualityCriteria)) {
-        sessions.push({ session });
-      } else {
-        filteredOutCount++;
-      }
-    } catch (error) {
-      // Skip invalid sessions
-      console.warn(`Failed to load session ${sessionId}:`, error);
-    }
-
-    // Apply limit AFTER filtering
-    if (options?.limit && sessions.length >= options.limit) {
-      break;
-    }
-  }
-
-  // Log filtering stats for visibility
-  if (filteredOutCount > 0) {
-    console.log(
-      `Filtered out ${filteredOutCount} sessions (minEvents=${qualityCriteria.minEvents}, requireWorkerSpawn=${qualityCriteria.requireWorkerSpawn}, requireReview=${qualityCriteria.requireReview})`,
-    );
-  }
-
-  return sessions;
-}