@cursor/bdk 0.2.1

package/src/internal/eval-run-store.ts

@@ -0,0 +1,585 @@
+/**
+ * Store for playground-triggered eval batches.
+ *
+ * Default: process memory only, capped by `maxPlaygroundRuns` (see
+ * `defineEvalConfig`). When `agent/storage.ts` (`defineStorage`) declares an
+ * `evals` table, runs are mirrored there and survive restart.
+ * Runs against the live mount's loopback URL so case sessions appear in
+ * the playground session list.
+ */
+
+import { randomUUID } from "node:crypto";
+import {
+  type EvalCaseSnapshot,
+  type EvalConfigFile,
+  type EvalRunSnapshot,
+  resolveEvalMaxPlaygroundRuns,
+} from "../evals.js";
+import {
+  type DiscoveredEval,
+  discoverEvals,
+  type EvalRunResult,
+  filterDiscoveredEvals,
+  runDiscoveredEvals,
+} from "./eval-runner.js";
+
+/**
+ * Durable mirror for playground eval batches — the `StorageCoordinator`
+ * adapter over the `defineStorage` `evals` table. Writes are
+ * fire-and-forget (the coordinator serializes and isolates failures);
+ * `list` is the serve-start hydrate read.
+ */
+export interface EvalRunStorage {
+  save(run: EvalRunSnapshot): void;
+  delete(runId: string): void;
+  list(): Promise<EvalRunSnapshot[]>;
+}
+
+export interface StartEvalRunInput {
+  filterIds?: string[];
+  tags?: string[];
+  bearerToken?: string;
+  verbose?: boolean;
+  /** Per-case timeout override (same semantics as CLI `--timeout-ms`). */
+  timeoutMs?: number;
+}
+
+export type EvalListConfig = {
+  maxConcurrency?: number;
+  timeoutMs?: number;
+  maxPlaygroundRuns: number;
+  /** True when `agent/storage.ts` declares an `evals` table. */
+  durableRuns: boolean;
+};
+
+const INTERRUPTED_BATCH_ERROR = "Eval run interrupted (serve process exited before completion)";
+const INTERRUPTED_CASE_ERROR = "Eval case interrupted (serve process exited before completion)";
+const CANCELLED_BATCH_ERROR = "Eval run cancelled";
+const CANCELLED_CASE_ERROR = "Eval case cancelled";
+
+export class EvalRunStore {
+  private readonly runs = new Map<string, EvalRunSnapshot>();
+  private activeRunId: string | undefined;
+  private readonly abortByRunId = new Map<string, AbortController>();
+  private hydratePromise: Promise<void> | undefined;
+  private hydrated = false;
+  /** Filled after listen — e.g. `http://127.0.0.1:3000/weather-agent`. */
+  loopbackUrl: string | undefined;
+
+  constructor(
+    private readonly projectRoot: string,
+    private readonly logger: (line: string) => void,
+    /** `defineStorage` evals-table adapter; undefined ⇒ memory only. */
+    private readonly storage?: EvalRunStorage | undefined,
+  ) {}
+
+  async list(): Promise<{
+    evals: Array<{
+      id: string;
+      fileId: string;
+      description?: string;
+      tags?: string[];
+    }>;
+    config: EvalListConfig;
+  }> {
+    await this.hydrate();
+    const { evals, config } = await discoverEvals(this.projectRoot);
+    return {
+      evals: evals.map(e => ({
+        id: e.id,
+        fileId: e.fileId,
+        ...(e.definition.description === undefined
+          ? {}
+          : { description: e.definition.description }),
+        ...(e.definition.tags === undefined ? {} : { tags: e.definition.tags }),
+      })),
+      config: snapshotListConfig(config, this.storage !== undefined),
+    };
+  }
+
+  async get(runId: string): Promise<EvalRunSnapshot | undefined> {
+    await this.hydrate();
+    return this.runs.get(runId);
+  }
+
+  /** Newest-first snapshots for playground rehydrate after navigation/reload. */
+  async listRuns(): Promise<EvalRunSnapshot[]> {
+    await this.hydrate();
+    return this.sortedRuns();
+  }
+
+  /**
+   * Atomic snapshot of newest-first runs + the in-flight batch id (if any).
+   * Prefer this over pairing {@link listRuns} with {@link getActiveRunId}
+   * across an await boundary.
+   */
+  async listRunsWithActive(): Promise<{
+    runs: EvalRunSnapshot[];
+    activeRunId?: string;
+  }> {
+    await this.hydrate();
+    const activeRunId = this.activeRunId;
+    return {
+      runs: this.sortedRuns(),
+      ...(activeRunId === undefined ? {} : { activeRunId }),
+    };
+  }
+
+  getActiveRunId(): string | undefined {
+    return this.activeRunId;
+  }
+
+  async start(input: StartEvalRunInput = {}): Promise<EvalRunSnapshot> {
+    await this.hydrate();
+    if (this.activeRunId !== undefined) {
+      const active = this.runs.get(this.activeRunId);
+      if (active !== undefined && active.status === "running") {
+        throw new EvalRunConflictError(this.activeRunId);
+      }
+    }
+    const loopbackUrl = this.loopbackUrl;
+    if (loopbackUrl === undefined) {
+      throw new Error("Eval run store has no loopback URL yet (server not listening)");
+    }
+
+    const { evals, config } = await discoverEvals(this.projectRoot);
+    const selected = filterDiscoveredEvals(evals, {
+      filterIds: input.filterIds,
+      tags: input.tags,
+    });
+    if (selected.length === 0) {
+      throw new EvalRunEmptyError();
+    }
+
+    const runId = `evalrun_${randomUUID().replace(/-/g, "").slice(0, 16)}`;
+    const cases: EvalCaseSnapshot[] = selected.map(e => discoveredToPending(e));
+    const snapshot: EvalRunSnapshot = {
+      runId,
+      status: "running",
+      startedAt: new Date().toISOString(),
+      summary: {
+        passed: 0,
+        failed: 0,
+        scored: 0,
+        skipped: 0,
+        total: cases.length,
+        done: 0,
+      },
+      cases,
+      config: snapshotRunConfig(config, this.storage !== undefined),
+    };
+    this.runs.set(runId, snapshot);
+    this.activeRunId = runId;
+    const abort = new AbortController();
+    this.abortByRunId.set(runId, abort);
+    this.pruneOldRuns(config);
+    this.persist(snapshot);
+    this.logger(`[agent-sdk] eval run ${runId}: starting ${cases.length} case(s)`);
+
+    void this.execute(runId, {
+      baseUrl: loopbackUrl,
+      evals: selected,
+      config,
+      signal: abort.signal,
+      ...(input.bearerToken === undefined ? {} : { bearerToken: input.bearerToken }),
+      ...(input.verbose === undefined ? {} : { verbose: input.verbose }),
+      ...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }),
+    });
+
+    return snapshot;
+  }
+
+  /**
+   * Cancel a running batch by Eval ID (`runId`). Stops starting new cases and
+   * aborts in-flight turns. Idempotent when already cancelled.
+   */
+  async cancel(runId: string): Promise<EvalRunSnapshot> {
+    await this.hydrate();
+    const run = this.runs.get(runId);
+    if (run === undefined) {
+      throw new EvalRunUnknownError(runId);
+    }
+    if (run.status === "cancelled") {
+      return run;
+    }
+    if (run.status !== "running") {
+      throw new EvalRunNotRunningError(runId, run.status);
+    }
+    const abort = this.abortByRunId.get(runId);
+    abort?.abort();
+    markCancelledRun(run);
+    this.persist(run);
+    if (this.activeRunId === runId) {
+      this.activeRunId = undefined;
+    }
+    this.logger(`[agent-sdk] eval run ${runId}: cancelled`);
+    return run;
+  }
+
+  private sortedRuns(): EvalRunSnapshot[] {
+    return [...this.runs.values()].sort((a, b) => b.startedAt.localeCompare(a.startedAt));
+  }
+
+  /**
+   * Hydrate playground eval history from the `defineStorage` evals table.
+   * Called eagerly from the serve startup restore step — alongside session
+   * restore — and lazily as a safety net by every read. Idempotent;
+   * concurrent callers share one pass.
+   */
+  async hydrate(): Promise<void> {
+    if (this.hydrated) {
+      return;
+    }
+    if (this.hydratePromise === undefined) {
+      this.hydratePromise = this.hydrateFromStorage();
+    }
+    await this.hydratePromise;
+  }
+
+  private async hydrateFromStorage(): Promise<void> {
+    try {
+      if (this.storage === undefined) {
+        this.hydrated = true;
+        return;
+      }
+      const loaded = await this.storage.list();
+      for (const run of loaded) {
+        if (
+          run === null ||
+          typeof run !== "object" ||
+          typeof run.runId !== "string" ||
+          run.runId === ""
+        ) {
+          continue;
+        }
+        // Incomplete batches from a killed process should not block a new run.
+        if (run.status === "running") {
+          markInterruptedRun(run);
+          this.persist(run);
+        }
+        this.runs.set(run.runId, run);
+      }
+      // Prune with the defaults even when eval discovery fails — loaded
+      // history must not stay unpruned for the process lifetime just
+      // because an eval file has a syntax error at boot.
+      let config: Partial<EvalConfigFile> = {};
+      try {
+        config = (await discoverEvals(this.projectRoot)).config;
+      } catch (error) {
+        this.logger(
+          `[agent-sdk] eval discovery failed during hydrate (pruning with defaults): ${
+            error instanceof Error ? error.message : String(error)
+          }`,
+        );
+      }
+      this.pruneOldRuns(config);
+      this.logger(`[agent-sdk] hydrated ${this.runs.size} playground eval run(s) from storage`);
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.logger(`[agent-sdk] failed to hydrate playground eval runs: ${message}`);
+    } finally {
+      this.hydrated = true;
+    }
+  }
+
+  private pruneOldRuns(config: Partial<EvalConfigFile>): void {
+    const limit = resolveEvalMaxPlaygroundRuns(config.maxPlaygroundRuns);
+    const ordered = this.sortedRuns();
+    if (ordered.length <= limit) {
+      return;
+    }
+    for (const stale of ordered.slice(limit)) {
+      if (stale.runId === this.activeRunId) {
+        continue;
+      }
+      this.runs.delete(stale.runId);
+      this.storage?.delete(stale.runId);
+    }
+  }
+
+  /** Mirror the run onto durable storage (coordinator clones + serializes). */
+  private persist(run: EvalRunSnapshot): void {
+    this.storage?.save(run);
+  }
+
+  /**
+   * Serve is draining: durably mark any in-flight batch as interrupted
+   * while the storage queue can still deliver. Late `execute` writes are
+   * dropped by the closed coordinator, so without this a batch that was
+   * running at shutdown would look "running" (or silently lose its final
+   * completion write) until the next hydrate repairs it.
+   */
+  close(): void {
+    const activeRunId = this.activeRunId;
+    if (activeRunId === undefined) {
+      return;
+    }
+    const run = this.runs.get(activeRunId);
+    if (run === undefined || run.status !== "running") {
+      return;
+    }
+    this.abortByRunId.get(activeRunId)?.abort();
+    markInterruptedRun(run);
+    this.persist(run);
+    this.activeRunId = undefined;
+    this.abortByRunId.delete(activeRunId);
+  }
+
+  private async execute(
+    runId: string,
+    opts: {
+      baseUrl: string;
+      evals: DiscoveredEval[];
+      config: Partial<EvalConfigFile>;
+      bearerToken?: string;
+      verbose?: boolean;
+      timeoutMs?: number;
+      signal: AbortSignal;
+    },
+  ): Promise<void> {
+    if (this.runs.get(runId) === undefined) {
+      return;
+    }
+    try {
+      await runDiscoveredEvals({
+        projectRoot: this.projectRoot,
+        baseUrl: opts.baseUrl,
+        discovered: { evals: opts.evals, config: opts.config },
+        signal: opts.signal,
+        ...(opts.bearerToken === undefined ? {} : { bearerToken: opts.bearerToken }),
+        ...(opts.verbose === undefined ? {} : { verbose: opts.verbose }),
+        ...(opts.timeoutMs === undefined ? {} : { timeoutMs: opts.timeoutMs }),
+        onCaseStart: id => {
+          const run = this.runs.get(runId);
+          if (run === undefined || run.status === "cancelled") {
+            return;
+          }
+          const c = run.cases.find(x => x.id === id);
+          if (c !== undefined) {
+            c.status = "running";
+            c.inputs = [];
+            c.toolCalls = [];
+            c.finalText = undefined;
+            c.metrics = undefined;
+            c.logs = undefined;
+            c.assertions = undefined;
+            c.error = undefined;
+          }
+        },
+        onCaseEvent: (id, event) => {
+          const run = this.runs.get(runId);
+          if (run === undefined) {
+            return;
+          }
+          const c = run.cases.find(x => x.id === id);
+          if (c === undefined) {
+            return;
+          }
+          const sid =
+            typeof event.sessionId === "string" && event.sessionId !== ""
+              ? event.sessionId
+              : undefined;
+          if (sid !== undefined && c.sessionId === undefined) {
+            c.sessionId = sid;
+          }
+          if (event.type === "message.received" && typeof event.data?.text === "string") {
+            c.inputs = [...(c.inputs ?? []), event.data.text];
+          }
+          if (
+            event.type === "message.completed" &&
+            typeof event.data?.text === "string" &&
+            event.data.parentCallId === undefined
+          ) {
+            c.finalText = event.data.text;
+          }
+        },
+        onCaseDone: result => {
+          this.applyCaseResult(runId, result);
+          const run = this.runs.get(runId);
+          if (run !== undefined) {
+            this.persist(run);
+          }
+        },
+      });
+      const run = this.runs.get(runId);
+      if (run === undefined || run.status === "cancelled") {
+        return;
+      }
+      run.status = "completed";
+      run.finishedAt = new Date().toISOString();
+      this.persist(run);
+      this.logger(
+        `[agent-sdk] eval run ${runId}: done — ${run.summary.passed} passed, ${run.summary.failed} failed`,
+      );
+    } catch (error) {
+      const run = this.runs.get(runId);
+      if (run === undefined || run.status === "cancelled") {
+        return;
+      }
+      const message = error instanceof Error ? error.message : String(error);
+      if (opts.signal.aborted || message === "cancelled") {
+        markCancelledRun(run);
+        this.persist(run);
+        this.logger(`[agent-sdk] eval run ${runId}: cancelled`);
+        return;
+      }
+      run.status = "failed";
+      run.finishedAt = new Date().toISOString();
+      run.error = message;
+      this.persist(run);
+      this.logger(`[agent-sdk] eval run ${runId}: failed — ${run.error}`);
+    } finally {
+      this.abortByRunId.delete(runId);
+      if (this.activeRunId === runId) {
+        this.activeRunId = undefined;
+      }
+    }
+  }
+
+  private applyCaseResult(runId: string, result: EvalRunResult): void {
+    const run = this.runs.get(runId);
+    if (run === undefined) {
+      return;
+    }
+    const c = run.cases.find(x => x.id === result.id);
+    if (c === undefined) {
+      return;
+    }
+    const alreadyCounted = c.status === "done";
+    c.status = "done";
+    c.ok = result.ok;
+    c.verdict = result.verdict;
+    c.skipReason = result.skipReason;
+    c.metadata = result.metadata;
+    c.assertions = result.assertions;
+    c.durationMs = result.durationMs;
+    c.logs = result.logs;
+    c.metrics = Object.keys(result.metrics).length > 0 ? { ...result.metrics } : undefined;
+    c.inputs = result.inputs;
+    if (result.toolCalls !== undefined) {
+      c.toolCalls = result.toolCalls;
+    }
+    if (result.sessionId !== undefined) {
+      c.sessionId = result.sessionId;
+    }
+    if (result.error !== undefined) {
+      c.error = result.error;
+    }
+    if (result.finalText !== undefined) {
+      c.finalText = result.finalText;
+    }
+    if (result.tools !== undefined) {
+      c.tools = result.tools;
+    }
+    // Batch already finalized by cancel — keep case details, skip recount.
+    if (run.status === "cancelled" || alreadyCounted) {
+      return;
+    }
+    run.summary.done += 1;
+    const verdict = result.verdict ?? (result.ok ? "passed" : "failed");
+    run.summary[verdict] = (run.summary[verdict] ?? 0) + 1;
+  }
+}
+
+function markInterruptedRun(run: EvalRunSnapshot): void {
+  run.status = "failed";
+  run.finishedAt = run.finishedAt ?? new Date().toISOString();
+  run.error = run.error ?? INTERRUPTED_BATCH_ERROR;
+  for (const c of run.cases) {
+    if (c.status === "pending" || c.status === "running") {
+      c.status = "done";
+      c.ok = false;
+      c.error = c.error ?? INTERRUPTED_CASE_ERROR;
+    }
+  }
+  reconcileRunSummary(run);
+}
+
+function markCancelledRun(run: EvalRunSnapshot): void {
+  run.status = "cancelled";
+  run.finishedAt = run.finishedAt ?? new Date().toISOString();
+  run.error = CANCELLED_BATCH_ERROR;
+  for (const c of run.cases) {
+    if (c.status === "pending" || c.status === "running") {
+      c.status = "done";
+      c.ok = false;
+      c.error = CANCELLED_CASE_ERROR;
+    }
+  }
+  reconcileRunSummary(run);
+}
+
+function reconcileRunSummary(run: EvalRunSnapshot): void {
+  const counts = { passed: 0, failed: 0, scored: 0, skipped: 0 };
+  for (const c of run.cases) {
+    counts[c.verdict ?? (c.ok === true ? "passed" : "failed")] += 1;
+  }
+  run.summary = {
+    ...counts,
+    total: run.cases.length,
+    done: run.cases.length,
+  };
+}
+
+function snapshotListConfig(config: Partial<EvalConfigFile>, durableRuns: boolean): EvalListConfig {
+  return {
+    ...(config.maxConcurrency === undefined ? {} : { maxConcurrency: config.maxConcurrency }),
+    ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
+    maxPlaygroundRuns: resolveEvalMaxPlaygroundRuns(config.maxPlaygroundRuns),
+    durableRuns,
+  };
+}
+
+function snapshotRunConfig(
+  config: Partial<EvalConfigFile>,
+  durableRuns: boolean,
+): EvalRunSnapshot["config"] {
+  const list = snapshotListConfig(config, durableRuns);
+  return {
+    ...(list.maxConcurrency === undefined ? {} : { maxConcurrency: list.maxConcurrency }),
+    ...(list.timeoutMs === undefined ? {} : { timeoutMs: list.timeoutMs }),
+    maxPlaygroundRuns: list.maxPlaygroundRuns,
+    durableRuns: list.durableRuns,
+  };
+}
+
+function discoveredToPending(e: DiscoveredEval): EvalCaseSnapshot {
+  return {
+    id: e.id,
+    fileId: e.fileId,
+    status: "pending",
+    ...(e.definition.description === undefined ? {} : { description: e.definition.description }),
+    ...(e.definition.tags === undefined ? {} : { tags: e.definition.tags }),
+  };
+}
+
+export class EvalRunConflictError extends Error {
+  constructor(readonly activeRunId: string) {
+    super(`An eval run is already in progress (${activeRunId})`);
+    this.name = "EvalRunConflictError";
+  }
+}
+
+export class EvalRunEmptyError extends Error {
+  constructor() {
+    super("No matching evals found");
+    this.name = "EvalRunEmptyError";
+  }
+}
+
+export class EvalRunUnknownError extends Error {
+  constructor(readonly runId: string) {
+    super(`Unknown eval run ${runId}`);
+    this.name = "EvalRunUnknownError";
+  }
+}
+
+export class EvalRunNotRunningError extends Error {
+  constructor(
+    readonly runId: string,
+    readonly status: EvalRunSnapshot["status"],
+  ) {
+    super(`Eval run ${runId} is not running (status: ${status})`);
+    this.name = "EvalRunNotRunningError";
+  }
+}