@cursor/bdk 0.2.3 → 0.2.5

package/src/internal/eval-run-store.ts

@@ -1,585 +0,0 @@
-/**
- * Store for playground-triggered eval batches.
- *
- * Default: process memory only, capped by `maxPlaygroundRuns` (see
- * `defineEvalConfig`). When `agent/storage.ts` (`defineStorage`) declares an
- * `evals` table, runs are mirrored there and survive restart.
- * Runs against the live mount's loopback URL so case sessions appear in
- * the playground session list.
- */
-
-import { randomUUID } from "node:crypto";
-import {
-  type EvalCaseSnapshot,
-  type EvalConfigFile,
-  type EvalRunSnapshot,
-  resolveEvalMaxPlaygroundRuns,
-} from "../evals.js";
-import {
-  type DiscoveredEval,
-  discoverEvals,
-  type EvalRunResult,
-  filterDiscoveredEvals,
-  runDiscoveredEvals,
-} from "./eval-runner.js";
-
-/**
- * Durable mirror for playground eval batches — the `StorageCoordinator`
- * adapter over the `defineStorage` `evals` table. Writes are
- * fire-and-forget (the coordinator serializes and isolates failures);
- * `list` is the serve-start hydrate read.
- */
-export interface EvalRunStorage {
-  save(run: EvalRunSnapshot): void;
-  delete(runId: string): void;
-  list(): Promise<EvalRunSnapshot[]>;
-}
-
-export interface StartEvalRunInput {
-  filterIds?: string[];
-  tags?: string[];
-  bearerToken?: string;
-  verbose?: boolean;
-  /** Per-case timeout override (same semantics as CLI `--timeout-ms`). */
-  timeoutMs?: number;
-}
-
-export type EvalListConfig = {
-  maxConcurrency?: number;
-  timeoutMs?: number;
-  maxPlaygroundRuns: number;
-  /** True when `agent/storage.ts` declares an `evals` table. */
-  durableRuns: boolean;
-};
-
-const INTERRUPTED_BATCH_ERROR = "Eval run interrupted (serve process exited before completion)";
-const INTERRUPTED_CASE_ERROR = "Eval case interrupted (serve process exited before completion)";
-const CANCELLED_BATCH_ERROR = "Eval run cancelled";
-const CANCELLED_CASE_ERROR = "Eval case cancelled";
-
-export class EvalRunStore {
-  private readonly runs = new Map<string, EvalRunSnapshot>();
-  private activeRunId: string | undefined;
-  private readonly abortByRunId = new Map<string, AbortController>();
-  private hydratePromise: Promise<void> | undefined;
-  private hydrated = false;
-  /** Filled after listen — e.g. `http://127.0.0.1:3000/weather-agent`. */
-  loopbackUrl: string | undefined;
-
-  constructor(
-    private readonly projectRoot: string,
-    private readonly logger: (line: string) => void,
-    /** `defineStorage` evals-table adapter; undefined ⇒ memory only. */
-    private readonly storage?: EvalRunStorage | undefined,
-  ) {}
-
-  async list(): Promise<{
-    evals: Array<{
-      id: string;
-      fileId: string;
-      description?: string;
-      tags?: string[];
-    }>;
-    config: EvalListConfig;
-  }> {
-    await this.hydrate();
-    const { evals, config } = await discoverEvals(this.projectRoot);
-    return {
-      evals: evals.map(e => ({
-        id: e.id,
-        fileId: e.fileId,
-        ...(e.definition.description === undefined
-          ? {}
-          : { description: e.definition.description }),
-        ...(e.definition.tags === undefined ? {} : { tags: e.definition.tags }),
-      })),
-      config: snapshotListConfig(config, this.storage !== undefined),
-    };
-  }
-
-  async get(runId: string): Promise<EvalRunSnapshot | undefined> {
-    await this.hydrate();
-    return this.runs.get(runId);
-  }
-
-  /** Newest-first snapshots for playground rehydrate after navigation/reload. */
-  async listRuns(): Promise<EvalRunSnapshot[]> {
-    await this.hydrate();
-    return this.sortedRuns();
-  }
-
-  /**
-   * Atomic snapshot of newest-first runs + the in-flight batch id (if any).
-   * Prefer this over pairing {@link listRuns} with {@link getActiveRunId}
-   * across an await boundary.
-   */
-  async listRunsWithActive(): Promise<{
-    runs: EvalRunSnapshot[];
-    activeRunId?: string;
-  }> {
-    await this.hydrate();
-    const activeRunId = this.activeRunId;
-    return {
-      runs: this.sortedRuns(),
-      ...(activeRunId === undefined ? {} : { activeRunId }),
-    };
-  }
-
-  getActiveRunId(): string | undefined {
-    return this.activeRunId;
-  }
-
-  async start(input: StartEvalRunInput = {}): Promise<EvalRunSnapshot> {
-    await this.hydrate();
-    if (this.activeRunId !== undefined) {
-      const active = this.runs.get(this.activeRunId);
-      if (active !== undefined && active.status === "running") {
-        throw new EvalRunConflictError(this.activeRunId);
-      }
-    }
-    const loopbackUrl = this.loopbackUrl;
-    if (loopbackUrl === undefined) {
-      throw new Error("Eval run store has no loopback URL yet (server not listening)");
-    }
-
-    const { evals, config } = await discoverEvals(this.projectRoot);
-    const selected = filterDiscoveredEvals(evals, {
-      filterIds: input.filterIds,
-      tags: input.tags,
-    });
-    if (selected.length === 0) {
-      throw new EvalRunEmptyError();
-    }
-
-    const runId = `evalrun_${randomUUID().replace(/-/g, "").slice(0, 16)}`;
-    const cases: EvalCaseSnapshot[] = selected.map(e => discoveredToPending(e));
-    const snapshot: EvalRunSnapshot = {
-      runId,
-      status: "running",
-      startedAt: new Date().toISOString(),
-      summary: {
-        passed: 0,
-        failed: 0,
-        scored: 0,
-        skipped: 0,
-        total: cases.length,
-        done: 0,
-      },
-      cases,
-      config: snapshotRunConfig(config, this.storage !== undefined),
-    };
-    this.runs.set(runId, snapshot);
-    this.activeRunId = runId;
-    const abort = new AbortController();
-    this.abortByRunId.set(runId, abort);
-    this.pruneOldRuns(config);
-    this.persist(snapshot);
-    this.logger(`[agent-sdk] eval run ${runId}: starting ${cases.length} case(s)`);
-
-    void this.execute(runId, {
-      baseUrl: loopbackUrl,
-      evals: selected,
-      config,
-      signal: abort.signal,
-      ...(input.bearerToken === undefined ? {} : { bearerToken: input.bearerToken }),
-      ...(input.verbose === undefined ? {} : { verbose: input.verbose }),
-      ...(input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs }),
-    });
-
-    return snapshot;
-  }
-
-  /**
-   * Cancel a running batch by Eval ID (`runId`). Stops starting new cases and
-   * aborts in-flight turns. Idempotent when already cancelled.
-   */
-  async cancel(runId: string): Promise<EvalRunSnapshot> {
-    await this.hydrate();
-    const run = this.runs.get(runId);
-    if (run === undefined) {
-      throw new EvalRunUnknownError(runId);
-    }
-    if (run.status === "cancelled") {
-      return run;
-    }
-    if (run.status !== "running") {
-      throw new EvalRunNotRunningError(runId, run.status);
-    }
-    const abort = this.abortByRunId.get(runId);
-    abort?.abort();
-    markCancelledRun(run);
-    this.persist(run);
-    if (this.activeRunId === runId) {
-      this.activeRunId = undefined;
-    }
-    this.logger(`[agent-sdk] eval run ${runId}: cancelled`);
-    return run;
-  }
-
-  private sortedRuns(): EvalRunSnapshot[] {
-    return [...this.runs.values()].sort((a, b) => b.startedAt.localeCompare(a.startedAt));
-  }
-
-  /**
-   * Hydrate playground eval history from the `defineStorage` evals table.
-   * Called eagerly from the serve startup restore step — alongside session
-   * restore — and lazily as a safety net by every read. Idempotent;
-   * concurrent callers share one pass.
-   */
-  async hydrate(): Promise<void> {
-    if (this.hydrated) {
-      return;
-    }
-    if (this.hydratePromise === undefined) {
-      this.hydratePromise = this.hydrateFromStorage();
-    }
-    await this.hydratePromise;
-  }
-
-  private async hydrateFromStorage(): Promise<void> {
-    try {
-      if (this.storage === undefined) {
-        this.hydrated = true;
-        return;
-      }
-      const loaded = await this.storage.list();
-      for (const run of loaded) {
-        if (
-          run === null ||
-          typeof run !== "object" ||
-          typeof run.runId !== "string" ||
-          run.runId === ""
-        ) {
-          continue;
-        }
-        // Incomplete batches from a killed process should not block a new run.
-        if (run.status === "running") {
-          markInterruptedRun(run);
-          this.persist(run);
-        }
-        this.runs.set(run.runId, run);
-      }
-      // Prune with the defaults even when eval discovery fails — loaded
-      // history must not stay unpruned for the process lifetime just
-      // because an eval file has a syntax error at boot.
-      let config: Partial<EvalConfigFile> = {};
-      try {
-        config = (await discoverEvals(this.projectRoot)).config;
-      } catch (error) {
-        this.logger(
-          `[agent-sdk] eval discovery failed during hydrate (pruning with defaults): ${
-            error instanceof Error ? error.message : String(error)
-          }`,
-        );
-      }
-      this.pruneOldRuns(config);
-      this.logger(`[agent-sdk] hydrated ${this.runs.size} playground eval run(s) from storage`);
-    } catch (error) {
-      const message = error instanceof Error ? error.message : String(error);
-      this.logger(`[agent-sdk] failed to hydrate playground eval runs: ${message}`);
-    } finally {
-      this.hydrated = true;
-    }
-  }
-
-  private pruneOldRuns(config: Partial<EvalConfigFile>): void {
-    const limit = resolveEvalMaxPlaygroundRuns(config.maxPlaygroundRuns);
-    const ordered = this.sortedRuns();
-    if (ordered.length <= limit) {
-      return;
-    }
-    for (const stale of ordered.slice(limit)) {
-      if (stale.runId === this.activeRunId) {
-        continue;
-      }
-      this.runs.delete(stale.runId);
-      this.storage?.delete(stale.runId);
-    }
-  }
-
-  /** Mirror the run onto durable storage (coordinator clones + serializes). */
-  private persist(run: EvalRunSnapshot): void {
-    this.storage?.save(run);
-  }
-
-  /**
-   * Serve is draining: durably mark any in-flight batch as interrupted
-   * while the storage queue can still deliver. Late `execute` writes are
-   * dropped by the closed coordinator, so without this a batch that was
-   * running at shutdown would look "running" (or silently lose its final
-   * completion write) until the next hydrate repairs it.
-   */
-  close(): void {
-    const activeRunId = this.activeRunId;
-    if (activeRunId === undefined) {
-      return;
-    }
-    const run = this.runs.get(activeRunId);
-    if (run === undefined || run.status !== "running") {
-      return;
-    }
-    this.abortByRunId.get(activeRunId)?.abort();
-    markInterruptedRun(run);
-    this.persist(run);
-    this.activeRunId = undefined;
-    this.abortByRunId.delete(activeRunId);
-  }
-
-  private async execute(
-    runId: string,
-    opts: {
-      baseUrl: string;
-      evals: DiscoveredEval[];
-      config: Partial<EvalConfigFile>;
-      bearerToken?: string;
-      verbose?: boolean;
-      timeoutMs?: number;
-      signal: AbortSignal;
-    },
-  ): Promise<void> {
-    if (this.runs.get(runId) === undefined) {
-      return;
-    }
-    try {
-      await runDiscoveredEvals({
-        projectRoot: this.projectRoot,
-        baseUrl: opts.baseUrl,
-        discovered: { evals: opts.evals, config: opts.config },
-        signal: opts.signal,
-        ...(opts.bearerToken === undefined ? {} : { bearerToken: opts.bearerToken }),
-        ...(opts.verbose === undefined ? {} : { verbose: opts.verbose }),
-        ...(opts.timeoutMs === undefined ? {} : { timeoutMs: opts.timeoutMs }),
-        onCaseStart: id => {
-          const run = this.runs.get(runId);
-          if (run === undefined || run.status === "cancelled") {
-            return;
-          }
-          const c = run.cases.find(x => x.id === id);
-          if (c !== undefined) {
-            c.status = "running";
-            c.inputs = [];
-            c.toolCalls = [];
-            c.finalText = undefined;
-            c.metrics = undefined;
-            c.logs = undefined;
-            c.assertions = undefined;
-            c.error = undefined;
-          }
-        },
-        onCaseEvent: (id, event) => {
-          const run = this.runs.get(runId);
-          if (run === undefined) {
-            return;
-          }
-          const c = run.cases.find(x => x.id === id);
-          if (c === undefined) {
-            return;
-          }
-          const sid =
-            typeof event.sessionId === "string" && event.sessionId !== ""
-              ? event.sessionId
-              : undefined;
-          if (sid !== undefined && c.sessionId === undefined) {
-            c.sessionId = sid;
-          }
-          if (event.type === "message.received" && typeof event.data?.text === "string") {
-            c.inputs = [...(c.inputs ?? []), event.data.text];
-          }
-          if (
-            event.type === "message.completed" &&
-            typeof event.data?.text === "string" &&
-            event.data.parentCallId === undefined
-          ) {
-            c.finalText = event.data.text;
-          }
-        },
-        onCaseDone: result => {
-          this.applyCaseResult(runId, result);
-          const run = this.runs.get(runId);
-          if (run !== undefined) {
-            this.persist(run);
-          }
-        },
-      });
-      const run = this.runs.get(runId);
-      if (run === undefined || run.status === "cancelled") {
-        return;
-      }
-      run.status = "completed";
-      run.finishedAt = new Date().toISOString();
-      this.persist(run);
-      this.logger(
-        `[agent-sdk] eval run ${runId}: done — ${run.summary.passed} passed, ${run.summary.failed} failed`,
-      );
-    } catch (error) {
-      const run = this.runs.get(runId);
-      if (run === undefined || run.status === "cancelled") {
-        return;
-      }
-      const message = error instanceof Error ? error.message : String(error);
-      if (opts.signal.aborted || message === "cancelled") {
-        markCancelledRun(run);
-        this.persist(run);
-        this.logger(`[agent-sdk] eval run ${runId}: cancelled`);
-        return;
-      }
-      run.status = "failed";
-      run.finishedAt = new Date().toISOString();
-      run.error = message;
-      this.persist(run);
-      this.logger(`[agent-sdk] eval run ${runId}: failed — ${run.error}`);
-    } finally {
-      this.abortByRunId.delete(runId);
-      if (this.activeRunId === runId) {
-        this.activeRunId = undefined;
-      }
-    }
-  }
-
-  private applyCaseResult(runId: string, result: EvalRunResult): void {
-    const run = this.runs.get(runId);
-    if (run === undefined) {
-      return;
-    }
-    const c = run.cases.find(x => x.id === result.id);
-    if (c === undefined) {
-      return;
-    }
-    const alreadyCounted = c.status === "done";
-    c.status = "done";
-    c.ok = result.ok;
-    c.verdict = result.verdict;
-    c.skipReason = result.skipReason;
-    c.metadata = result.metadata;
-    c.assertions = result.assertions;
-    c.durationMs = result.durationMs;
-    c.logs = result.logs;
-    c.metrics = Object.keys(result.metrics).length > 0 ? { ...result.metrics } : undefined;
-    c.inputs = result.inputs;
-    if (result.toolCalls !== undefined) {
-      c.toolCalls = result.toolCalls;
-    }
-    if (result.sessionId !== undefined) {
-      c.sessionId = result.sessionId;
-    }
-    if (result.error !== undefined) {
-      c.error = result.error;
-    }
-    if (result.finalText !== undefined) {
-      c.finalText = result.finalText;
-    }
-    if (result.tools !== undefined) {
-      c.tools = result.tools;
-    }
-    // Batch already finalized by cancel — keep case details, skip recount.
-    if (run.status === "cancelled" || alreadyCounted) {
-      return;
-    }
-    run.summary.done += 1;
-    const verdict = result.verdict ?? (result.ok ? "passed" : "failed");
-    run.summary[verdict] = (run.summary[verdict] ?? 0) + 1;
-  }
-}
-
-function markInterruptedRun(run: EvalRunSnapshot): void {
-  run.status = "failed";
-  run.finishedAt = run.finishedAt ?? new Date().toISOString();
-  run.error = run.error ?? INTERRUPTED_BATCH_ERROR;
-  for (const c of run.cases) {
-    if (c.status === "pending" || c.status === "running") {
-      c.status = "done";
-      c.ok = false;
-      c.error = c.error ?? INTERRUPTED_CASE_ERROR;
-    }
-  }
-  reconcileRunSummary(run);
-}
-
-function markCancelledRun(run: EvalRunSnapshot): void {
-  run.status = "cancelled";
-  run.finishedAt = run.finishedAt ?? new Date().toISOString();
-  run.error = CANCELLED_BATCH_ERROR;
-  for (const c of run.cases) {
-    if (c.status === "pending" || c.status === "running") {
-      c.status = "done";
-      c.ok = false;
-      c.error = CANCELLED_CASE_ERROR;
-    }
-  }
-  reconcileRunSummary(run);
-}
-
-function reconcileRunSummary(run: EvalRunSnapshot): void {
-  const counts = { passed: 0, failed: 0, scored: 0, skipped: 0 };
-  for (const c of run.cases) {
-    counts[c.verdict ?? (c.ok === true ? "passed" : "failed")] += 1;
-  }
-  run.summary = {
-    ...counts,
-    total: run.cases.length,
-    done: run.cases.length,
-  };
-}
-
-function snapshotListConfig(config: Partial<EvalConfigFile>, durableRuns: boolean): EvalListConfig {
-  return {
-    ...(config.maxConcurrency === undefined ? {} : { maxConcurrency: config.maxConcurrency }),
-    ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
-    maxPlaygroundRuns: resolveEvalMaxPlaygroundRuns(config.maxPlaygroundRuns),
-    durableRuns,
-  };
-}
-
-function snapshotRunConfig(
-  config: Partial<EvalConfigFile>,
-  durableRuns: boolean,
-): EvalRunSnapshot["config"] {
-  const list = snapshotListConfig(config, durableRuns);
-  return {
-    ...(list.maxConcurrency === undefined ? {} : { maxConcurrency: list.maxConcurrency }),
-    ...(list.timeoutMs === undefined ? {} : { timeoutMs: list.timeoutMs }),
-    maxPlaygroundRuns: list.maxPlaygroundRuns,
-    durableRuns: list.durableRuns,
-  };
-}
-
-function discoveredToPending(e: DiscoveredEval): EvalCaseSnapshot {
-  return {
-    id: e.id,
-    fileId: e.fileId,
-    status: "pending",
-    ...(e.definition.description === undefined ? {} : { description: e.definition.description }),
-    ...(e.definition.tags === undefined ? {} : { tags: e.definition.tags }),
-  };
-}
-
-export class EvalRunConflictError extends Error {
-  constructor(readonly activeRunId: string) {
-    super(`An eval run is already in progress (${activeRunId})`);
-    this.name = "EvalRunConflictError";
-  }
-}
-
-export class EvalRunEmptyError extends Error {
-  constructor() {
-    super("No matching evals found");
-    this.name = "EvalRunEmptyError";
-  }
-}
-
-export class EvalRunUnknownError extends Error {
-  constructor(readonly runId: string) {
-    super(`Unknown eval run ${runId}`);
-    this.name = "EvalRunUnknownError";
-  }
-}
-
-export class EvalRunNotRunningError extends Error {
-  constructor(
-    readonly runId: string,
-    readonly status: EvalRunSnapshot["status"],
-  ) {
-    super(`Eval run ${runId} is not running (status: ${status})`);
-    this.name = "EvalRunNotRunningError";
-  }
-}