@cursor/july 0.1.1

package/dist/internal/eval-run-store.js

@@ -0,0 +1,396 @@
+/**
+ * Store for playground-triggered eval batches.
+ *
+ * Default: process memory only, capped by `maxPlaygroundRuns` (see `defineEvalConfig`).
+ * Optional durable `persistRuns` from `evals/evals.config.ts` survives restart.
+ * Runs against the live mount's loopback URL so case sessions appear in
+ * the playground session list.
+ */
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { randomUUID } from "node:crypto";
+import { resolveEvalMaxPlaygroundRuns, } from "../evals.js";
+import { discoverEvals, filterDiscoveredEvals, runDiscoveredEvals, } from "./eval-runner.js";
+const INTERRUPTED_BATCH_ERROR = "Eval run interrupted (serve process exited before completion)";
+const INTERRUPTED_CASE_ERROR = "Eval case interrupted (serve process exited before completion)";
+export class EvalRunStore {
+    constructor(projectRoot, logger) {
+        this.projectRoot = projectRoot;
+        this.logger = logger;
+        this.runs = new Map();
+        this.hydrated = false;
+        /** Per-run persist chain so intermediate saves cannot overwrite a later final write. */
+        this.persistChains = new Map();
+    }
+    list() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.ensureHydrated();
+            const { evals, config } = yield discoverEvals(this.projectRoot);
+            return {
+                evals: evals.map((e) => (Object.assign(Object.assign({ id: e.id, fileId: e.fileId }, (e.definition.description === undefined
+                    ? {}
+                    : { description: e.definition.description })), (e.definition.tags === undefined ? {} : { tags: e.definition.tags })))),
+                config: snapshotListConfig(config),
+            };
+        });
+    }
+    get(runId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.ensureHydrated();
+            return this.runs.get(runId);
+        });
+    }
+    /** Newest-first snapshots for playground rehydrate after navigation/reload. */
+    listRuns() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.ensureHydrated();
+            return this.sortedRuns();
+        });
+    }
+    /**
+     * Atomic snapshot of newest-first runs + the in-flight batch id (if any).
+     * Prefer this over pairing {@link listRuns} with {@link getActiveRunId}
+     * across an await boundary.
+     */
+    listRunsWithActive() {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.ensureHydrated();
+            const activeRunId = this.activeRunId;
+            return Object.assign({ runs: this.sortedRuns() }, (activeRunId === undefined ? {} : { activeRunId }));
+        });
+    }
+    getActiveRunId() {
+        return this.activeRunId;
+    }
+    start() {
+        return __awaiter(this, arguments, void 0, function* (input = {}) {
+            yield this.ensureHydrated();
+            if (this.activeRunId !== undefined) {
+                const active = this.runs.get(this.activeRunId);
+                if (active !== undefined && active.status === "running") {
+                    throw new EvalRunConflictError(this.activeRunId);
+                }
+            }
+            const loopbackUrl = this.loopbackUrl;
+            if (loopbackUrl === undefined) {
+                throw new Error("Eval run store has no loopback URL yet (server not listening)");
+            }
+            const { evals, config } = yield discoverEvals(this.projectRoot);
+            const selected = filterDiscoveredEvals(evals, {
+                filterIds: input.filterIds,
+                tags: input.tags,
+            });
+            if (selected.length === 0) {
+                throw new EvalRunEmptyError();
+            }
+            const runId = `evalrun_${randomUUID().replace(/-/g, "").slice(0, 16)}`;
+            const cases = selected.map((e) => discoveredToPending(e));
+            const snapshot = {
+                runId,
+                status: "running",
+                startedAt: new Date().toISOString(),
+                summary: {
+                    passed: 0,
+                    failed: 0,
+                    total: cases.length,
+                    done: 0,
+                },
+                cases,
+                config: snapshotRunConfig(config),
+            };
+            this.runs.set(runId, snapshot);
+            this.activeRunId = runId;
+            yield this.pruneOldRuns(config);
+            yield this.enqueuePersist(snapshot, config.persistRuns);
+            this.logger(`[agent-serve] eval run ${runId}: starting ${cases.length} case(s)`);
+            void this.execute(runId, Object.assign(Object.assign({ baseUrl: loopbackUrl, evals: selected, config }, (input.bearerToken === undefined
+                ? {}
+                : { bearerToken: input.bearerToken })), (input.verbose === undefined ? {} : { verbose: input.verbose })));
+            return snapshot;
+        });
+    }
+    sortedRuns() {
+        return [...this.runs.values()].sort((a, b) => b.startedAt.localeCompare(a.startedAt));
+    }
+    ensureHydrated() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.hydrated) {
+                return;
+            }
+            if (this.hydratePromise === undefined) {
+                this.hydratePromise = this.hydrateFromPersistence();
+            }
+            yield this.hydratePromise;
+        });
+    }
+    hydrateFromPersistence() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                const { config } = yield discoverEvals(this.projectRoot);
+                const persist = config.persistRuns;
+                if (persist === undefined) {
+                    this.hydrated = true;
+                    return;
+                }
+                const loaded = yield persist.load({ projectRoot: this.projectRoot });
+                if (!Array.isArray(loaded)) {
+                    this.logger("[agent-serve] eval persistRuns.load() did not return an array; ignoring");
+                    this.hydrated = true;
+                    return;
+                }
+                const repaired = [];
+                for (const run of loaded) {
+                    if (run === null ||
+                        typeof run !== "object" ||
+                        typeof run.runId !== "string" ||
+                        run.runId === "") {
+                        continue;
+                    }
+                    // Incomplete batches from a killed process should not block a new run.
+                    if (run.status === "running") {
+                        markInterruptedRun(run);
+                        repaired.push(run);
+                    }
+                    this.runs.set(run.runId, run);
+                }
+                for (const run of repaired) {
+                    yield this.enqueuePersist(run, persist);
+                }
+                yield this.pruneOldRuns(config);
+                this.logger(`[agent-serve] hydrated ${this.runs.size} playground eval run(s) from persistRuns`);
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                this.logger(`[agent-serve] failed to hydrate playground eval runs: ${message}`);
+            }
+            finally {
+                this.hydrated = true;
+            }
+        });
+    }
+    pruneOldRuns(config) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const limit = resolveEvalMaxPlaygroundRuns(config.maxPlaygroundRuns);
+            const ordered = this.sortedRuns();
+            if (ordered.length <= limit) {
+                return;
+            }
+            for (const stale of ordered.slice(limit)) {
+                if (stale.runId === this.activeRunId) {
+                    continue;
+                }
+                this.runs.delete(stale.runId);
+                yield this.persistDelete(stale.runId, config.persistRuns);
+            }
+        });
+    }
+    /**
+     * Queue a durable save for `run`. Snapshots are cloned at enqueue time and
+     * writes for the same runId run strictly in order so an intermediate
+     * fire-and-forget save cannot finish after (and overwrite) a later final save.
+     */
+    enqueuePersist(run, persist) {
+        var _a;
+        if (persist === undefined) {
+            return Promise.resolve();
+        }
+        const snapshot = structuredClone(run);
+        const prev = (_a = this.persistChains.get(run.runId)) !== null && _a !== void 0 ? _a : Promise.resolve();
+        const next = prev
+            .catch(() => undefined)
+            .then(() => this.persistSave(snapshot, persist));
+        this.persistChains.set(run.runId, next);
+        return next;
+    }
+    persistSave(run, persist) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield persist.save(run, { projectRoot: this.projectRoot });
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                this.logger(`[agent-serve] persistRuns.save(${run.runId}) failed: ${message}`);
+            }
+        });
+    }
+    persistDelete(runId, persist) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (persist === undefined) {
+                return;
+            }
+            try {
+                yield persist.delete(runId, { projectRoot: this.projectRoot });
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                this.logger(`[agent-serve] persistRuns.delete(${runId}) failed: ${message}`);
+            }
+        });
+    }
+    execute(runId, opts) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.runs.get(runId) === undefined) {
+                return;
+            }
+            try {
+                yield runDiscoveredEvals(Object.assign(Object.assign(Object.assign({ projectRoot: this.projectRoot, baseUrl: opts.baseUrl, discovered: { evals: opts.evals, config: opts.config } }, (opts.bearerToken === undefined
+                    ? {}
+                    : { bearerToken: opts.bearerToken })), (opts.verbose === undefined ? {} : { verbose: opts.verbose })), { onCaseStart: (id) => {
+                        const run = this.runs.get(runId);
+                        if (run === undefined) {
+                            return;
+                        }
+                        const c = run.cases.find((x) => x.id === id);
+                        if (c !== undefined) {
+                            c.status = "running";
+                            c.inputs = [];
+                            c.toolCalls = [];
+                            c.finalText = undefined;
+                        }
+                    }, onCaseEvent: (id, event) => {
+                        var _a, _b;
+                        var _c;
+                        const run = this.runs.get(runId);
+                        if (run === undefined) {
+                            return;
+                        }
+                        const c = run.cases.find((x) => x.id === id);
+                        if (c === undefined) {
+                            return;
+                        }
+                        const sid = typeof event.sessionId === "string" && event.sessionId !== ""
+                            ? event.sessionId
+                            : undefined;
+                        if (sid !== undefined && c.sessionId === undefined) {
+                            c.sessionId = sid;
+                        }
+                        if (event.type === "message.received" &&
+                            typeof ((_a = event.data) === null || _a === void 0 ? void 0 : _a.text) === "string") {
+                            c.inputs = [...((_c = c.inputs) !== null && _c !== void 0 ? _c : []), event.data.text];
+                        }
+                        if (event.type === "message.completed" &&
+                            typeof ((_b = event.data) === null || _b === void 0 ? void 0 : _b.text) === "string") {
+                            c.finalText = event.data.text;
+                        }
+                    }, onCaseDone: (result) => {
+                        this.applyCaseResult(runId, result);
+                        const run = this.runs.get(runId);
+                        if (run !== undefined) {
+                            void this.enqueuePersist(run, opts.config.persistRuns);
+                        }
+                    } }));
+                const run = this.runs.get(runId);
+                if (run === undefined) {
+                    return;
+                }
+                run.status = "completed";
+                run.finishedAt = new Date().toISOString();
+                yield this.enqueuePersist(run, opts.config.persistRuns);
+                this.logger(`[agent-serve] eval run ${runId}: done — ${run.summary.passed} passed, ${run.summary.failed} failed`);
+            }
+            catch (error) {
+                const run = this.runs.get(runId);
+                if (run === undefined) {
+                    return;
+                }
+                run.status = "failed";
+                run.finishedAt = new Date().toISOString();
+                run.error = error instanceof Error ? error.message : String(error);
+                yield this.enqueuePersist(run, opts.config.persistRuns);
+                this.logger(`[agent-serve] eval run ${runId}: failed — ${run.error}`);
+            }
+            finally {
+                if (this.activeRunId === runId) {
+                    this.activeRunId = undefined;
+                }
+            }
+        });
+    }
+    applyCaseResult(runId, result) {
+        const run = this.runs.get(runId);
+        if (run === undefined) {
+            return;
+        }
+        const c = run.cases.find((x) => x.id === result.id);
+        if (c === undefined) {
+            return;
+        }
+        c.status = "done";
+        c.ok = result.ok;
+        c.assertions = result.assertions;
+        c.durationMs = result.durationMs;
+        c.logs = result.logs;
+        c.inputs = result.inputs;
+        if (result.toolCalls !== undefined) {
+            c.toolCalls = result.toolCalls;
+        }
+        if (result.sessionId !== undefined) {
+            c.sessionId = result.sessionId;
+        }
+        if (result.error !== undefined) {
+            c.error = result.error;
+        }
+        if (result.finalText !== undefined) {
+            c.finalText = result.finalText;
+        }
+        if (result.tools !== undefined) {
+            c.tools = result.tools;
+        }
+        run.summary.done += 1;
+        if (result.ok) {
+            run.summary.passed += 1;
+        }
+        else {
+            run.summary.failed += 1;
+        }
+    }
+}
+function markInterruptedRun(run) {
+    var _a, _b, _c;
+    run.status = "failed";
+    run.finishedAt = (_a = run.finishedAt) !== null && _a !== void 0 ? _a : new Date().toISOString();
+    run.error = (_b = run.error) !== null && _b !== void 0 ? _b : INTERRUPTED_BATCH_ERROR;
+    for (const c of run.cases) {
+        if (c.status === "pending" || c.status === "running") {
+            c.status = "done";
+            c.ok = false;
+            c.error = (_c = c.error) !== null && _c !== void 0 ? _c : INTERRUPTED_CASE_ERROR;
+        }
+    }
+}
+function snapshotListConfig(config) {
+    return Object.assign(Object.assign(Object.assign({}, (config.maxConcurrency === undefined
+        ? {}
+        : { maxConcurrency: config.maxConcurrency })), (config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs })), { maxPlaygroundRuns: resolveEvalMaxPlaygroundRuns(config.maxPlaygroundRuns), durableRuns: config.persistRuns !== undefined });
+}
+function snapshotRunConfig(config) {
+    const list = snapshotListConfig(config);
+    return Object.assign(Object.assign(Object.assign({}, (list.maxConcurrency === undefined
+        ? {}
+        : { maxConcurrency: list.maxConcurrency })), (list.timeoutMs === undefined ? {} : { timeoutMs: list.timeoutMs })), { maxPlaygroundRuns: list.maxPlaygroundRuns, durableRuns: list.durableRuns });
+}
+function discoveredToPending(e) {
+    return Object.assign(Object.assign({ id: e.id, fileId: e.fileId, status: "pending" }, (e.definition.description === undefined
+        ? {}
+        : { description: e.definition.description })), (e.definition.tags === undefined ? {} : { tags: e.definition.tags }));
+}
+export class EvalRunConflictError extends Error {
+    constructor(activeRunId) {
+        super(`An eval run is already in progress (${activeRunId})`);
+        this.activeRunId = activeRunId;
+        this.name = "EvalRunConflictError";
+    }
+}
+export class EvalRunEmptyError extends Error {
+    constructor() {
+        super("No matching evals found");
+        this.name = "EvalRunEmptyError";
+    }
+}

package/dist/internal/eval-runner.d.ts

@@ -0,0 +1,76 @@
+/**
+ * Discover and run filesystem evals under `<project>/evals/`.
+ *
+ * Case identity: single-`test` files use the path
+ * (`evals/weather/nyc.eval.ts` → `weather/nyc`); multi-`cases` files use
+ * `<fileId>/<case.id>` (`evals/weather.eval.ts` + `nyc` → `weather/nyc`).
+ * Drive with `t.send`, assert with `t.succeeded` / `t.calledTool` /
+ * `t.messageIncludes` / `t.check`.
+ */
+import type { EvalAssertionResult, EvalCaseDefinition, EvalConfigFile } from "../evals.js";
+import type { SessionEvent } from "../types.js";
+export interface DiscoveredEval {
+    /** Runnable datapoint id (`weather/nyc` or file id for single-test files). */
+    id: string;
+    /** File id under `evals/` (`weather` for `evals/weather.eval.ts`). */
+    fileId: string;
+    path: string;
+    definition: EvalCaseDefinition;
+}
+export interface EvalRunResult {
+    id: string;
+    path: string;
+    description?: string;
+    ok: boolean;
+    assertions: EvalAssertionResult[];
+    sessionId?: string;
+    /** Messages passed to `t.send` (eval inputs). */
+    inputs: string[];
+    finalText?: string;
+    tools?: string[];
+    /** Tool calls with args/results from the trajectory. */
+    toolCalls?: Array<{
+        toolName: string;
+        args?: unknown;
+        output?: unknown;
+        isError: boolean;
+    }>;
+    error?: string;
+    logs: string[];
+    durationMs: number;
+}
+export interface EvalSelectionOptions {
+    filterIds?: string[];
+    tags?: string[];
+}
+export interface RunEvalsOptions extends EvalSelectionOptions {
+    projectRoot: string;
+    baseUrl: string;
+    timeoutMs?: number;
+    bearerToken?: string;
+    verbose?: boolean;
+    onEvent?: (event: SessionEvent) => void;
+    /** Fired when a case begins executing (after concurrency slot acquired). */
+    onCaseStart?: (id: string) => void;
+    /** Fired when a case finishes (pass or fail). */
+    onCaseDone?: (result: EvalRunResult) => void;
+    /** Fired for each session event during a case (for live progress / sessionId). */
+    onCaseEvent?: (id: string, event: SessionEvent) => void;
+    importModule?: (absolutePath: string) => Promise<Record<string, unknown>>;
+    /**
+     * Pre-discovered evals + config. When set, skips filesystem discovery and
+     * selection filters (caller already chose the case list).
+     */
+    discovered?: {
+        evals: readonly DiscoveredEval[];
+        config: Partial<EvalConfigFile>;
+    };
+}
+export declare function discoverEvals(projectRoot: string, importModule?: (absolutePath: string) => Promise<Record<string, unknown>>): Promise<{
+    evals: DiscoveredEval[];
+    config: Partial<EvalConfigFile>;
+}>;
+/** Apply id/tag filters used by CLI, playground, and `runDiscoveredEvals`. */
+export declare function filterDiscoveredEvals(evals: readonly DiscoveredEval[], options?: EvalSelectionOptions): DiscoveredEval[];
+export declare function runDiscoveredEvals(options: RunEvalsOptions): Promise<EvalRunResult[]>;
+//# sourceMappingURL=eval-runner.d.ts.map

package/dist/internal/eval-runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-runner.d.ts","sourceRoot":"","sources":["../../src/internal/eval-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAMH,OAAO,KAAK,EACV,mBAAmB,EACnB,kBAAkB,EAClB,cAAc,EAMf,MAAM,aAAa,CAAC;AAOrB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAMhD,MAAM,WAAW,cAAc;IAC7B,8EAA8E;IAC9E,EAAE,EAAE,MAAM,CAAC;IACX,sEAAsE;IACtE,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,kBAAkB,CAAC;CAChC;AAED,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,EAAE,EAAE,OAAO,CAAC;IACZ,UAAU,EAAE,mBAAmB,EAAE,CAAC;IAClC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,wDAAwD;IACxD,SAAS,CAAC,EAAE,KAAK,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,IAAI,CAAC,EAAE,OAAO,CAAC;QACf,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC;KAClB,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,eAAgB,SAAQ,oBAAoB;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IACxC,4EAA4E;IAC5E,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,iDAAiD;IACjD,UAAU,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,KAAK,IAAI,CAAC;IAC7C,kFAAkF;IAClF,WAAW,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,KAAK,IAAI,CAAC;IACxD,YAAY,CAAC,EAAE,CAAC,YAAY,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;IAC1E;;;OAGG;IACH,UAAU,CAAC,EAAE;QACX,KAAK,EAAE,SAAS,cAAc,EAAE,CAAC;QACjC,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;KACjC,CAAC;CACH;AAuCD,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,EACnB,YAAY,GAAE,CACZ,YAAY,EAAE,MAAM,KACjB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAiB,GACpD,OAAO,CAAC;IACT,KAAK,EAAE,cAAc,EAAE,CAAC;IACxB,MAAM,EAAE,OAAO,CAAC,cAAc,CAAC,CAAC;CACjC,CAAC,CA4CD;AAED,8EAA8E;AAC9E,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,SAAS,cAAc,EAAE,EAChC,OAAO,GAAE,oBAAyB,GACjC,cAAc,EAAE,CAalB;AAED,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,eAAe,GACvB,OAAO,CAAC,aAAa,EAAE,CAAC,CA0C1B"}