@cursor/july 0.1.1

package/src/internal/eval-run-store.ts

@@ -0,0 +1,519 @@
+/**
+ * Store for playground-triggered eval batches.
+ *
+ * Default: process memory only, capped by `maxPlaygroundRuns` (see `defineEvalConfig`).
+ * Optional durable `persistRuns` from `evals/evals.config.ts` survives restart.
+ * Runs against the live mount's loopback URL so case sessions appear in
+ * the playground session list.
+ */
+
+import { randomUUID } from "node:crypto";
+import {
+  type EvalCaseSnapshot,
+  type EvalConfigFile,
+  type EvalRunPersistence,
+  type EvalRunSnapshot,
+  resolveEvalMaxPlaygroundRuns,
+} from "../evals.js";
+import {
+  type DiscoveredEval,
+  discoverEvals,
+  type EvalRunResult,
+  filterDiscoveredEvals,
+  runDiscoveredEvals,
+} from "./eval-runner.js";
+
+export type {
+  EvalBatchStatus,
+  EvalCaseRunStatus as EvalCaseStatus,
+  EvalCaseSnapshot,
+  EvalRunSnapshot,
+} from "../evals.js";
+
+export interface StartEvalRunInput {
+  filterIds?: string[];
+  tags?: string[];
+  bearerToken?: string;
+  verbose?: boolean;
+}
+
+export type EvalListConfig = {
+  maxConcurrency?: number;
+  timeoutMs?: number;
+  maxPlaygroundRuns: number;
+  /** True when `evals.config.ts` set `persistRuns`. */
+  durableRuns: boolean;
+};
+
+const INTERRUPTED_BATCH_ERROR =
+  "Eval run interrupted (serve process exited before completion)";
+const INTERRUPTED_CASE_ERROR =
+  "Eval case interrupted (serve process exited before completion)";
+
+export class EvalRunStore {
+  private readonly runs = new Map<string, EvalRunSnapshot>();
+  private activeRunId: string | undefined;
+  private hydratePromise: Promise<void> | undefined;
+  private hydrated = false;
+  /** Per-run persist chain so intermediate saves cannot overwrite a later final write. */
+  private readonly persistChains = new Map<string, Promise<void>>();
+  /** Filled after listen — e.g. `http://127.0.0.1:3000/weather-agent`. */
+  loopbackUrl: string | undefined;
+
+  constructor(
+    private readonly projectRoot: string,
+    private readonly logger: (line: string) => void
+  ) {}
+
+  async list(): Promise<{
+    evals: Array<{
+      id: string;
+      fileId: string;
+      description?: string;
+      tags?: string[];
+    }>;
+    config: EvalListConfig;
+  }> {
+    await this.ensureHydrated();
+    const { evals, config } = await discoverEvals(this.projectRoot);
+    return {
+      evals: evals.map((e) => ({
+        id: e.id,
+        fileId: e.fileId,
+        ...(e.definition.description === undefined
+          ? {}
+          : { description: e.definition.description }),
+        ...(e.definition.tags === undefined ? {} : { tags: e.definition.tags }),
+      })),
+      config: snapshotListConfig(config),
+    };
+  }
+
+  async get(runId: string): Promise<EvalRunSnapshot | undefined> {
+    await this.ensureHydrated();
+    return this.runs.get(runId);
+  }
+
+  /** Newest-first snapshots for playground rehydrate after navigation/reload. */
+  async listRuns(): Promise<EvalRunSnapshot[]> {
+    await this.ensureHydrated();
+    return this.sortedRuns();
+  }
+
+  /**
+   * Atomic snapshot of newest-first runs + the in-flight batch id (if any).
+   * Prefer this over pairing {@link listRuns} with {@link getActiveRunId}
+   * across an await boundary.
+   */
+  async listRunsWithActive(): Promise<{
+    runs: EvalRunSnapshot[];
+    activeRunId?: string;
+  }> {
+    await this.ensureHydrated();
+    const activeRunId = this.activeRunId;
+    return {
+      runs: this.sortedRuns(),
+      ...(activeRunId === undefined ? {} : { activeRunId }),
+    };
+  }
+
+  getActiveRunId(): string | undefined {
+    return this.activeRunId;
+  }
+
+  async start(input: StartEvalRunInput = {}): Promise<EvalRunSnapshot> {
+    await this.ensureHydrated();
+    if (this.activeRunId !== undefined) {
+      const active = this.runs.get(this.activeRunId);
+      if (active !== undefined && active.status === "running") {
+        throw new EvalRunConflictError(this.activeRunId);
+      }
+    }
+    const loopbackUrl = this.loopbackUrl;
+    if (loopbackUrl === undefined) {
+      throw new Error(
+        "Eval run store has no loopback URL yet (server not listening)"
+      );
+    }
+
+    const { evals, config } = await discoverEvals(this.projectRoot);
+    const selected = filterDiscoveredEvals(evals, {
+      filterIds: input.filterIds,
+      tags: input.tags,
+    });
+    if (selected.length === 0) {
+      throw new EvalRunEmptyError();
+    }
+
+    const runId = `evalrun_${randomUUID().replace(/-/g, "").slice(0, 16)}`;
+    const cases: EvalCaseSnapshot[] = selected.map((e) =>
+      discoveredToPending(e)
+    );
+    const snapshot: EvalRunSnapshot = {
+      runId,
+      status: "running",
+      startedAt: new Date().toISOString(),
+      summary: {
+        passed: 0,
+        failed: 0,
+        total: cases.length,
+        done: 0,
+      },
+      cases,
+      config: snapshotRunConfig(config),
+    };
+    this.runs.set(runId, snapshot);
+    this.activeRunId = runId;
+    await this.pruneOldRuns(config);
+    await this.enqueuePersist(snapshot, config.persistRuns);
+    this.logger(
+      `[agent-serve] eval run ${runId}: starting ${cases.length} case(s)`
+    );
+
+    void this.execute(runId, {
+      baseUrl: loopbackUrl,
+      evals: selected,
+      config,
+      ...(input.bearerToken === undefined
+        ? {}
+        : { bearerToken: input.bearerToken }),
+      ...(input.verbose === undefined ? {} : { verbose: input.verbose }),
+    });
+
+    return snapshot;
+  }
+
+  private sortedRuns(): EvalRunSnapshot[] {
+    return [...this.runs.values()].sort((a, b) =>
+      b.startedAt.localeCompare(a.startedAt)
+    );
+  }
+
+  private async ensureHydrated(): Promise<void> {
+    if (this.hydrated) {
+      return;
+    }
+    if (this.hydratePromise === undefined) {
+      this.hydratePromise = this.hydrateFromPersistence();
+    }
+    await this.hydratePromise;
+  }
+
+  private async hydrateFromPersistence(): Promise<void> {
+    try {
+      const { config } = await discoverEvals(this.projectRoot);
+      const persist = config.persistRuns;
+      if (persist === undefined) {
+        this.hydrated = true;
+        return;
+      }
+      const loaded = await persist.load({ projectRoot: this.projectRoot });
+      if (!Array.isArray(loaded)) {
+        this.logger(
+          "[agent-serve] eval persistRuns.load() did not return an array; ignoring"
+        );
+        this.hydrated = true;
+        return;
+      }
+      const repaired: EvalRunSnapshot[] = [];
+      for (const run of loaded) {
+        if (
+          run === null ||
+          typeof run !== "object" ||
+          typeof run.runId !== "string" ||
+          run.runId === ""
+        ) {
+          continue;
+        }
+        // Incomplete batches from a killed process should not block a new run.
+        if (run.status === "running") {
+          markInterruptedRun(run);
+          repaired.push(run);
+        }
+        this.runs.set(run.runId, run);
+      }
+      for (const run of repaired) {
+        await this.enqueuePersist(run, persist);
+      }
+      await this.pruneOldRuns(config);
+      this.logger(
+        `[agent-serve] hydrated ${this.runs.size} playground eval run(s) from persistRuns`
+      );
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.logger(
+        `[agent-serve] failed to hydrate playground eval runs: ${message}`
+      );
+    } finally {
+      this.hydrated = true;
+    }
+  }
+
+  private async pruneOldRuns(config: Partial<EvalConfigFile>): Promise<void> {
+    const limit = resolveEvalMaxPlaygroundRuns(config.maxPlaygroundRuns);
+    const ordered = this.sortedRuns();
+    if (ordered.length <= limit) {
+      return;
+    }
+    for (const stale of ordered.slice(limit)) {
+      if (stale.runId === this.activeRunId) {
+        continue;
+      }
+      this.runs.delete(stale.runId);
+      await this.persistDelete(stale.runId, config.persistRuns);
+    }
+  }
+
+  /**
+   * Queue a durable save for `run`. Snapshots are cloned at enqueue time and
+   * writes for the same runId run strictly in order so an intermediate
+   * fire-and-forget save cannot finish after (and overwrite) a later final save.
+   */
+  private enqueuePersist(
+    run: EvalRunSnapshot,
+    persist: EvalRunPersistence | undefined
+  ): Promise<void> {
+    if (persist === undefined) {
+      return Promise.resolve();
+    }
+    const snapshot = structuredClone(run);
+    const prev = this.persistChains.get(run.runId) ?? Promise.resolve();
+    const next = prev
+      .catch(() => undefined)
+      .then(() => this.persistSave(snapshot, persist));
+    this.persistChains.set(run.runId, next);
+    return next;
+  }
+
+  private async persistSave(
+    run: EvalRunSnapshot,
+    persist: EvalRunPersistence
+  ): Promise<void> {
+    try {
+      await persist.save(run, { projectRoot: this.projectRoot });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.logger(
+        `[agent-serve] persistRuns.save(${run.runId}) failed: ${message}`
+      );
+    }
+  }
+
+  private async persistDelete(
+    runId: string,
+    persist: EvalRunPersistence | undefined
+  ): Promise<void> {
+    if (persist === undefined) {
+      return;
+    }
+    try {
+      await persist.delete(runId, { projectRoot: this.projectRoot });
+    } catch (error) {
+      const message = error instanceof Error ? error.message : String(error);
+      this.logger(
+        `[agent-serve] persistRuns.delete(${runId}) failed: ${message}`
+      );
+    }
+  }
+
+  private async execute(
+    runId: string,
+    opts: {
+      baseUrl: string;
+      evals: DiscoveredEval[];
+      config: Partial<EvalConfigFile>;
+      bearerToken?: string;
+      verbose?: boolean;
+    }
+  ): Promise<void> {
+    if (this.runs.get(runId) === undefined) {
+      return;
+    }
+    try {
+      await runDiscoveredEvals({
+        projectRoot: this.projectRoot,
+        baseUrl: opts.baseUrl,
+        discovered: { evals: opts.evals, config: opts.config },
+        ...(opts.bearerToken === undefined
+          ? {}
+          : { bearerToken: opts.bearerToken }),
+        ...(opts.verbose === undefined ? {} : { verbose: opts.verbose }),
+        onCaseStart: (id) => {
+          const run = this.runs.get(runId);
+          if (run === undefined) {
+            return;
+          }
+          const c = run.cases.find((x) => x.id === id);
+          if (c !== undefined) {
+            c.status = "running";
+            c.inputs = [];
+            c.toolCalls = [];
+            c.finalText = undefined;
+          }
+        },
+        onCaseEvent: (id, event) => {
+          const run = this.runs.get(runId);
+          if (run === undefined) {
+            return;
+          }
+          const c = run.cases.find((x) => x.id === id);
+          if (c === undefined) {
+            return;
+          }
+          const sid =
+            typeof event.sessionId === "string" && event.sessionId !== ""
+              ? event.sessionId
+              : undefined;
+          if (sid !== undefined && c.sessionId === undefined) {
+            c.sessionId = sid;
+          }
+          if (
+            event.type === "message.received" &&
+            typeof event.data?.text === "string"
+          ) {
+            c.inputs = [...(c.inputs ?? []), event.data.text];
+          }
+          if (
+            event.type === "message.completed" &&
+            typeof event.data?.text === "string"
+          ) {
+            c.finalText = event.data.text;
+          }
+        },
+        onCaseDone: (result) => {
+          this.applyCaseResult(runId, result);
+          const run = this.runs.get(runId);
+          if (run !== undefined) {
+            void this.enqueuePersist(run, opts.config.persistRuns);
+          }
+        },
+      });
+      const run = this.runs.get(runId);
+      if (run === undefined) {
+        return;
+      }
+      run.status = "completed";
+      run.finishedAt = new Date().toISOString();
+      await this.enqueuePersist(run, opts.config.persistRuns);
+      this.logger(
+        `[agent-serve] eval run ${runId}: done — ${run.summary.passed} passed, ${run.summary.failed} failed`
+      );
+    } catch (error) {
+      const run = this.runs.get(runId);
+      if (run === undefined) {
+        return;
+      }
+      run.status = "failed";
+      run.finishedAt = new Date().toISOString();
+      run.error = error instanceof Error ? error.message : String(error);
+      await this.enqueuePersist(run, opts.config.persistRuns);
+      this.logger(`[agent-serve] eval run ${runId}: failed — ${run.error}`);
+    } finally {
+      if (this.activeRunId === runId) {
+        this.activeRunId = undefined;
+      }
+    }
+  }
+
+  private applyCaseResult(runId: string, result: EvalRunResult): void {
+    const run = this.runs.get(runId);
+    if (run === undefined) {
+      return;
+    }
+    const c = run.cases.find((x) => x.id === result.id);
+    if (c === undefined) {
+      return;
+    }
+    c.status = "done";
+    c.ok = result.ok;
+    c.assertions = result.assertions;
+    c.durationMs = result.durationMs;
+    c.logs = result.logs;
+    c.inputs = result.inputs;
+    if (result.toolCalls !== undefined) {
+      c.toolCalls = result.toolCalls;
+    }
+    if (result.sessionId !== undefined) {
+      c.sessionId = result.sessionId;
+    }
+    if (result.error !== undefined) {
+      c.error = result.error;
+    }
+    if (result.finalText !== undefined) {
+      c.finalText = result.finalText;
+    }
+    if (result.tools !== undefined) {
+      c.tools = result.tools;
+    }
+    run.summary.done += 1;
+    if (result.ok) {
+      run.summary.passed += 1;
+    } else {
+      run.summary.failed += 1;
+    }
+  }
+}
+
+function markInterruptedRun(run: EvalRunSnapshot): void {
+  run.status = "failed";
+  run.finishedAt = run.finishedAt ?? new Date().toISOString();
+  run.error = run.error ?? INTERRUPTED_BATCH_ERROR;
+  for (const c of run.cases) {
+    if (c.status === "pending" || c.status === "running") {
+      c.status = "done";
+      c.ok = false;
+      c.error = c.error ?? INTERRUPTED_CASE_ERROR;
+    }
+  }
+}
+
+function snapshotListConfig(config: Partial<EvalConfigFile>): EvalListConfig {
+  return {
+    ...(config.maxConcurrency === undefined
+      ? {}
+      : { maxConcurrency: config.maxConcurrency }),
+    ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }),
+    maxPlaygroundRuns: resolveEvalMaxPlaygroundRuns(config.maxPlaygroundRuns),
+    durableRuns: config.persistRuns !== undefined,
+  };
+}
+
+function snapshotRunConfig(
+  config: Partial<EvalConfigFile>
+): EvalRunSnapshot["config"] {
+  const list = snapshotListConfig(config);
+  return {
+    ...(list.maxConcurrency === undefined
+      ? {}
+      : { maxConcurrency: list.maxConcurrency }),
+    ...(list.timeoutMs === undefined ? {} : { timeoutMs: list.timeoutMs }),
+    maxPlaygroundRuns: list.maxPlaygroundRuns,
+    durableRuns: list.durableRuns,
+  };
+}
+
+function discoveredToPending(e: DiscoveredEval): EvalCaseSnapshot {
+  return {
+    id: e.id,
+    fileId: e.fileId,
+    status: "pending",
+    ...(e.definition.description === undefined
+      ? {}
+      : { description: e.definition.description }),
+    ...(e.definition.tags === undefined ? {} : { tags: e.definition.tags }),
+  };
+}
+
+export class EvalRunConflictError extends Error {
+  constructor(readonly activeRunId: string) {
+    super(`An eval run is already in progress (${activeRunId})`);
+    this.name = "EvalRunConflictError";
+  }
+}
+
+export class EvalRunEmptyError extends Error {
+  constructor() {
+    super("No matching evals found");
+    this.name = "EvalRunEmptyError";
+  }
+}