@cursor/july 0.1.5 → 0.1.7

package/dist/internal/eval-run-store.d.ts

@@ -1,50 +1,54 @@
 /**
  * Store for playground-triggered eval batches.
  *
- * Default: process memory only, capped by `maxPlaygroundRuns` (see `defineEvalConfig`).
- * Optional durable `persistRuns` from `evals/evals.config.ts` survives restart.
+ * Default: process memory only, capped by `maxPlaygroundRuns` (see
+ * `defineEvalConfig`). When `agent/storage.ts` (`defineStorage`) declares an
+ * `evals` table, runs are mirrored there and survive restart.
  * Runs against the live mount's loopback URL so case sessions appear in
  * the playground session list.
  */
-import { type EvalRunPersistence, type EvalRunSnapshot } from "../evals.js";
+import { type EvalRunSnapshot } from "../evals.js";
 export type { EvalBatchStatus, EvalCaseRunStatus as EvalCaseStatus, EvalCaseSnapshot, EvalRunSnapshot, } from "../evals.js";
+/**
+ * Durable mirror for playground eval batches — the `StorageCoordinator`
+ * adapter over the `defineStorage` `evals` table. Writes are
+ * fire-and-forget (the coordinator serializes and isolates failures);
+ * `list` is the serve-start hydrate read.
+ */
+export interface EvalRunStorage {
+    save(run: EvalRunSnapshot): void;
+    delete(runId: string): void;
+    list(): Promise<EvalRunSnapshot[]>;
+}
 export interface StartEvalRunInput {
     filterIds?: string[];
     tags?: string[];
     bearerToken?: string;
     verbose?: boolean;
+    /** Per-case timeout override (same semantics as CLI `--timeout-ms`). */
+    timeoutMs?: number;
 }
 export type EvalListConfig = {
     maxConcurrency?: number;
     timeoutMs?: number;
     maxPlaygroundRuns: number;
-    /** True when `evals.config.ts` set `persistRuns`. */
+    /** True when `agent/storage.ts` declares an `evals` table. */
     durableRuns: boolean;
 };
 export declare class EvalRunStore {
     private readonly projectRoot;
     private readonly logger;
-    /**
-     * Project-wide sink from `agent/storage.ts` (`defineStorage`),
-     * used when `evals.config.ts` sets no `persistRuns` of its own.
-     */
-    private readonly fallbackPersistence?;
+    /** `defineStorage` evals-table adapter; undefined ⇒ memory only. */
+    private readonly storage?;
     private readonly runs;
     private activeRunId;
     private hydratePromise;
     private hydrated;
-    /** Per-run persist chain so intermediate saves cannot overwrite a later final write. */
-    private readonly persistChains;
     /** Filled after listen — e.g. `http://127.0.0.1:3000/weather-agent`. */
     loopbackUrl: string | undefined;
     constructor(projectRoot: string, logger: (line: string) => void, 
-    /**
-     * Project-wide sink from `agent/storage.ts` (`defineStorage`),
-     * used when `evals.config.ts` sets no `persistRuns` of its own.
-     */
-    fallbackPersistence?: EvalRunPersistence | undefined);
-    /** `evals.config.ts` `persistRuns` wins; `defineStorage` is fallback. */
-    private resolvePersist;
+    /** `defineStorage` evals-table adapter; undefined ⇒ memory only. */
+    storage?: EvalRunStorage | undefined);
     list(): Promise<{
         evals: Array<{
             id: string;
@@ -70,23 +74,24 @@ export declare class EvalRunStore {
     start(input?: StartEvalRunInput): Promise<EvalRunSnapshot>;
     private sortedRuns;
     /**
-     * Hydrate playground eval history from durable persistence
-     * (`evals.config.ts` `persistRuns`, else the `defineStorage`
-     * fallback). Called eagerly from the serve startup restore step —
-     * alongside session restore — and lazily as a safety net by every read.
-     * Idempotent; concurrent callers share one pass.
+     * Hydrate playground eval history from the `defineStorage` evals table.
+     * Called eagerly from the serve startup restore step — alongside session
+     * restore — and lazily as a safety net by every read. Idempotent;
+     * concurrent callers share one pass.
      */
     hydrate(): Promise<void>;
-    private hydrateFromPersistence;
+    private hydrateFromStorage;
     private pruneOldRuns;
+    /** Mirror the run onto durable storage (coordinator clones + serializes). */
+    private persist;
     /**
-     * Queue a durable save for `run`. Snapshots are cloned at enqueue time and
-     * writes for the same runId run strictly in order so an intermediate
-     * fire-and-forget save cannot finish after (and overwrite) a later final save.
+     * Serve is draining: durably mark any in-flight batch as interrupted
+     * while the storage queue can still deliver. Late `execute` writes are
+     * dropped by the closed coordinator, so without this a batch that was
+     * running at shutdown would look "running" (or silently lose its final
+     * completion write) until the next hydrate repairs it.
      */
-    private enqueuePersist;
-    private persistSave;
-    private persistDelete;
+    close(): void;
     private execute;
     private applyCaseResult;
 }

package/dist/internal/eval-run-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-run-store.d.ts","sourceRoot":"","sources":["../../src/internal/eval-run-store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EAGL,KAAK,kBAAkB,EACvB,KAAK,eAAe,EAErB,MAAM,aAAa,CAAC;AASrB,YAAY,EACV,eAAe,EACf,iBAAiB,IAAI,cAAc,EACnC,gBAAgB,EAChB,eAAe,GAChB,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,iBAAiB;IAChC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qDAAqD;IACrD,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAOF,qBAAa,YAAY;IAWrB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC;IAhBvC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsC;IAC3D,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,cAAc,CAA4B;IAClD,OAAO,CAAC,QAAQ,CAAS;IACzB,wFAAwF;IACxF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAoC;IAClE,wEAAwE;IACxE,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAEhC,YACmB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI;IAC/C;;;OAGG;IACc,mBAAmB,CAAC,EAAE,kBAAkB,YAAA,EACvD;IAEJ,yEAAyE;IACzE,OAAO,CAAC,cAAc;IAMhB,IAAI,IAAI,OAAO,CAAC;QACpB,KAAK,EAAE,KAAK,CAAC;YACX,EAAE,EAAE,MAAM,CAAC;YACX,MAAM,EAAE,MAAM,CAAC;YACf,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC,CAAC;QACH,MAAM,EAAE,cAAc,CAAC;KACxB,CAAC,CAiBD;IAEK,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAG7D;IAED,+EAA+E;IACzE,QAAQ,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAG3C;IAED;;;;OAIG;IACG,kBAAkB,IAAI,OAAO,CAAC;QAClC,IAAI,EAAE,eAAe,EAAE,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAOD;IAED,cAAc,IAAI,MAAM,GAAG,SAAS,CAEnC;IAEK,KAAK,CAAC,KAAK,GAAE,iBAAsB,GAAG,OAAO,CAAC,eAAe,CAAC,CA+DnE;IAED,OAAO,CAAC,UAAU;IAMlB;;;;;;OAMG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ7B;YAEa,sBAAsB;YAkDtB,YAAY;IAe1B;;;;OAIG;IACH,OAAO,CAAC,cAAc;YAgBR,WAAW;YAcX,aAAa;YAeb,OAAO;IAoGrB,OAAO,CAAC,eAAe;CAqCxB;AAwDD,qBAAa,oBAAqB,SAAQ,KAAK;IACjC,QAAQ,CAAC,WAAW,EAAE,MAAM;IAAxC,YAAqB,WAAW,EAAE,MAAM,EAGvC;CACF;AAED,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,cAGC;CACF"}
+{"version":3,"file":"eval-run-store.d.ts","sourceRoot":"","sources":["../../src/internal/eval-run-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAGL,KAAK,eAAe,EAErB,MAAM,aAAa,CAAC;AASrB,YAAY,EACV,eAAe,EACf,iBAAiB,IAAI,cAAc,EACnC,gBAAgB,EAChB,eAAe,GAChB,MAAM,aAAa,CAAC;AAErB;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,CAAC;IACjC,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAOF,qBAAa,YAAY;IASrB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;IAX3B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsC;IAC3D,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,cAAc,CAA4B;IAClD,OAAO,CAAC,QAAQ,CAAS;IACzB,wEAAwE;IACxE,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAEhC,YACmB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI;IAC/C,oEAAoE;IACnD,OAAO,CAAC,EAAE,cAAc,GAAG,SAAS,EACnD;IAEE,IAAI,IAAI,OAAO,CAAC;QACpB,KAAK,EAAE,KAAK,CAAC;YACX,EAAE,EAAE,MAAM,CAAC;YACX,MAAM,EAAE,MAAM,CAAC;YACf,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC,CAAC;QACH,MAAM,EAAE,cAAc,CAAC;KACxB,CAAC,CAcD;IAEK,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAG7D;IAED,+EAA+E;IACzE,QAAQ,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAG3C;IAED;;;;OAIG;IACG,kBAAkB,IAAI,OAAO,CAAC;QAClC,IAAI,EAAE,eAAe,EAAE,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAOD;IAED,cAAc,IAAI,MAAM,GAAG,SAAS,CAEnC;IAEK,KAAK,CAAC,KAAK,GAAE,iBAAsB,GAAG,OAAO,CAAC,eAAe,CAAC,CA6DnE;IAED,OAAO,CAAC,UAAU;IAMlB;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ7B;YAEa,kBAAkB;IAkDhC,OAAO,CAAC,YAAY;IAepB,6EAA6E;IAC7E,OAAO,CAAC,OAAO;IAIf;;;;;;OAMG;IACH,KAAK,IAAI,IAAI,CAYZ;YAEa,OAAO;IAsGrB,OAAO,CAAC,eAAe;CAqCxB;AAyED,qBAAa,oBAAqB,SAAQ,KAAK;IACjC,QAAQ,CAAC,WAAW,EAAE,MAAM;IAAxC,YAAqB,WAAW,EAAE,MAAM,EAGvC;CACF;AAED,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,cAGC;CACF"}

package/dist/internal/eval-run-store.js

@@ -1,8 +1,9 @@
 /**
  * Store for playground-triggered eval batches.
  *
- * Default: process memory only, capped by `maxPlaygroundRuns` (see `defineEvalConfig`).
- * Optional durable `persistRuns` from `evals/evals.config.ts` survives restart.
+ * Default: process memory only, capped by `maxPlaygroundRuns` (see
+ * `defineEvalConfig`). When `agent/storage.ts` (`defineStorage`) declares an
+ * `evals` table, runs are mirrored there and survive restart.
  * Runs against the live mount's loopback URL so case sessions appear in
  * the playground session list.
  */
@@ -22,23 +23,13 @@ const INTERRUPTED_BATCH_ERROR = "Eval run interrupted (serve process exited befo
 const INTERRUPTED_CASE_ERROR = "Eval case interrupted (serve process exited before completion)";
 export class EvalRunStore {
     constructor(projectRoot, logger, 
-    /**
-     * Project-wide sink from `agent/storage.ts` (`defineStorage`),
-     * used when `evals.config.ts` sets no `persistRuns` of its own.
-     */
-    fallbackPersistence) {
+    /** `defineStorage` evals-table adapter; undefined ⇒ memory only. */
+    storage) {
         this.projectRoot = projectRoot;
         this.logger = logger;
-        this.fallbackPersistence = fallbackPersistence;
+        this.storage = storage;
         this.runs = new Map();
         this.hydrated = false;
-        /** Per-run persist chain so intermediate saves cannot overwrite a later final write. */
-        this.persistChains = new Map();
-    }
-    /** `evals.config.ts` `persistRuns` wins; `defineStorage` is fallback. */
-    resolvePersist(config) {
-        var _a;
-        return (_a = config.persistRuns) !== null && _a !== void 0 ? _a : this.fallbackPersistence;
     }
     list() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -48,7 +39,7 @@ export class EvalRunStore {
                 evals: evals.map((e) => (Object.assign(Object.assign({ id: e.id, fileId: e.fileId }, (e.definition.description === undefined
                     ? {}
                     : { description: e.definition.description })), (e.definition.tags === undefined ? {} : { tags: e.definition.tags })))),
-                config: snapshotListConfig(config, this.resolvePersist(config) !== undefined),
+                config: snapshotListConfig(config, this.storage !== undefined),
             };
         });
     }
@@ -114,16 +105,16 @@ export class EvalRunStore {
                     done: 0,
                 },
                 cases,
-                config: snapshotRunConfig(config, this.resolvePersist(config) !== undefined),
+                config: snapshotRunConfig(config, this.storage !== undefined),
             };
             this.runs.set(runId, snapshot);
             this.activeRunId = runId;
-            yield this.pruneOldRuns(config);
-            yield this.enqueuePersist(snapshot, this.resolvePersist(config));
+            this.pruneOldRuns(config);
+            this.persist(snapshot);
             this.logger(`[agentkit] eval run ${runId}: starting ${cases.length} case(s)`);
-            void this.execute(runId, Object.assign(Object.assign({ baseUrl: loopbackUrl, evals: selected, config }, (input.bearerToken === undefined
+            void this.execute(runId, Object.assign(Object.assign(Object.assign({ baseUrl: loopbackUrl, evals: selected, config }, (input.bearerToken === undefined
                 ? {}
-                : { bearerToken: input.bearerToken })), (input.verbose === undefined ? {} : { verbose: input.verbose })));
+                : { bearerToken: input.bearerToken })), (input.verbose === undefined ? {} : { verbose: input.verbose })), (input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs })));
             return snapshot;
         });
     }
@@ -131,11 +122,10 @@ export class EvalRunStore {
         return [...this.runs.values()].sort((a, b) => b.startedAt.localeCompare(a.startedAt));
     }
     /**
-     * Hydrate playground eval history from durable persistence
-     * (`evals.config.ts` `persistRuns`, else the `defineStorage`
-     * fallback). Called eagerly from the serve startup restore step —
-     * alongside session restore — and lazily as a safety net by every read.
-     * Idempotent; concurrent callers share one pass.
+     * Hydrate playground eval history from the `defineStorage` evals table.
+     * Called eagerly from the serve startup restore step — alongside session
+     * restore — and lazily as a safety net by every read. Idempotent;
+     * concurrent callers share one pass.
      */
     hydrate() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -143,27 +133,19 @@ export class EvalRunStore {
                 return;
             }
             if (this.hydratePromise === undefined) {
-                this.hydratePromise = this.hydrateFromPersistence();
+                this.hydratePromise = this.hydrateFromStorage();
             }
             yield this.hydratePromise;
         });
     }
-    hydrateFromPersistence() {
+    hydrateFromStorage() {
         return __awaiter(this, void 0, void 0, function* () {
             try {
-                const { config } = yield discoverEvals(this.projectRoot);
-                const persist = this.resolvePersist(config);
-                if (persist === undefined) {
-                    this.hydrated = true;
-                    return;
-                }
-                const loaded = yield persist.load({ projectRoot: this.projectRoot });
-                if (!Array.isArray(loaded)) {
-                    this.logger("[agentkit] eval persistRuns.load() did not return an array; ignoring");
+                if (this.storage === undefined) {
                     this.hydrated = true;
                     return;
                 }
-                const repaired = [];
+                const loaded = yield this.storage.list();
                 for (const run of loaded) {
                     if (run === null ||
                         typeof run !== "object" ||
@@ -174,15 +156,22 @@ export class EvalRunStore {
                     // Incomplete batches from a killed process should not block a new run.
                     if (run.status === "running") {
                         markInterruptedRun(run);
-                        repaired.push(run);
+                        this.persist(run);
                     }
                     this.runs.set(run.runId, run);
                 }
-                for (const run of repaired) {
-                    yield this.enqueuePersist(run, persist);
+                // Prune with the defaults even when eval discovery fails — loaded
+                // history must not stay unpruned for the process lifetime just
+                // because an eval file has a syntax error at boot.
+                let config = {};
+                try {
+                    config = (yield discoverEvals(this.projectRoot)).config;
+                }
+                catch (error) {
+                    this.logger(`[agentkit] eval discovery failed during hydrate (pruning with defaults): ${error instanceof Error ? error.message : String(error)}`);
                 }
-                yield this.pruneOldRuns(config);
-                this.logger(`[agentkit] hydrated ${this.runs.size} playground eval run(s) from persistRuns`);
+                this.pruneOldRuns(config);
+                this.logger(`[agentkit] hydrated ${this.runs.size} playground eval run(s) from storage`);
             }
             catch (error) {
                 const message = error instanceof Error ? error.message : String(error);
@@ -194,63 +183,44 @@ export class EvalRunStore {
         });
     }
     pruneOldRuns(config) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const limit = resolveEvalMaxPlaygroundRuns(config.maxPlaygroundRuns);
-            const ordered = this.sortedRuns();
-            if (ordered.length <= limit) {
-                return;
-            }
-            for (const stale of ordered.slice(limit)) {
-                if (stale.runId === this.activeRunId) {
-                    continue;
-                }
-                this.runs.delete(stale.runId);
-                yield this.persistDelete(stale.runId, this.resolvePersist(config));
+        var _a;
+        const limit = resolveEvalMaxPlaygroundRuns(config.maxPlaygroundRuns);
+        const ordered = this.sortedRuns();
+        if (ordered.length <= limit) {
+            return;
+        }
+        for (const stale of ordered.slice(limit)) {
+            if (stale.runId === this.activeRunId) {
+                continue;
             }
-        });
+            this.runs.delete(stale.runId);
+            (_a = this.storage) === null || _a === void 0 ? void 0 : _a.delete(stale.runId);
+        }
+    }
+    /** Mirror the run onto durable storage (coordinator clones + serializes). */
+    persist(run) {
+        var _a;
+        (_a = this.storage) === null || _a === void 0 ? void 0 : _a.save(run);
     }
     /**
-     * Queue a durable save for `run`. Snapshots are cloned at enqueue time and
-     * writes for the same runId run strictly in order so an intermediate
-     * fire-and-forget save cannot finish after (and overwrite) a later final save.
+     * Serve is draining: durably mark any in-flight batch as interrupted
+     * while the storage queue can still deliver. Late `execute` writes are
+     * dropped by the closed coordinator, so without this a batch that was
+     * running at shutdown would look "running" (or silently lose its final
+     * completion write) until the next hydrate repairs it.
      */
-    enqueuePersist(run, persist) {
-        var _a;
-        if (persist === undefined) {
-            return Promise.resolve();
+    close() {
+        const activeRunId = this.activeRunId;
+        if (activeRunId === undefined) {
+            return;
         }
-        const snapshot = structuredClone(run);
-        const prev = (_a = this.persistChains.get(run.runId)) !== null && _a !== void 0 ? _a : Promise.resolve();
-        const next = prev
-            .catch(() => undefined)
-            .then(() => this.persistSave(snapshot, persist));
-        this.persistChains.set(run.runId, next);
-        return next;
-    }
-    persistSave(run, persist) {
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield persist.save(run, { projectRoot: this.projectRoot });
-            }
-            catch (error) {
-                const message = error instanceof Error ? error.message : String(error);
-                this.logger(`[agentkit] persistRuns.save(${run.runId}) failed: ${message}`);
-            }
-        });
-    }
-    persistDelete(runId, persist) {
-        return __awaiter(this, void 0, void 0, function* () {
-            if (persist === undefined) {
-                return;
-            }
-            try {
-                yield persist.delete(runId, { projectRoot: this.projectRoot });
-            }
-            catch (error) {
-                const message = error instanceof Error ? error.message : String(error);
-                this.logger(`[agentkit] persistRuns.delete(${runId}) failed: ${message}`);
-            }
-        });
+        const run = this.runs.get(activeRunId);
+        if (run === undefined || run.status !== "running") {
+            return;
+        }
+        markInterruptedRun(run);
+        this.persist(run);
+        this.activeRunId = undefined;
     }
     execute(runId, opts) {
         return __awaiter(this, void 0, void 0, function* () {
@@ -258,9 +228,9 @@ export class EvalRunStore {
                 return;
             }
             try {
-                yield runDiscoveredEvals(Object.assign(Object.assign(Object.assign({ projectRoot: this.projectRoot, baseUrl: opts.baseUrl, discovered: { evals: opts.evals, config: opts.config } }, (opts.bearerToken === undefined
+                yield runDiscoveredEvals(Object.assign(Object.assign(Object.assign(Object.assign({ projectRoot: this.projectRoot, baseUrl: opts.baseUrl, discovered: { evals: opts.evals, config: opts.config } }, (opts.bearerToken === undefined
                     ? {}
-                    : { bearerToken: opts.bearerToken })), (opts.verbose === undefined ? {} : { verbose: opts.verbose })), { onCaseStart: (id) => {
+                    : { bearerToken: opts.bearerToken })), (opts.verbose === undefined ? {} : { verbose: opts.verbose })), (opts.timeoutMs === undefined ? {} : { timeoutMs: opts.timeoutMs })), { onCaseStart: (id) => {
                         const run = this.runs.get(runId);
                         if (run === undefined) {
                             return;
@@ -302,7 +272,7 @@ export class EvalRunStore {
                         this.applyCaseResult(runId, result);
                         const run = this.runs.get(runId);
                         if (run !== undefined) {
-                            void this.enqueuePersist(run, this.resolvePersist(opts.config));
+                            this.persist(run);
                         }
                     } }));
                 const run = this.runs.get(runId);
@@ -311,7 +281,7 @@ export class EvalRunStore {
                 }
                 run.status = "completed";
                 run.finishedAt = new Date().toISOString();
-                yield this.enqueuePersist(run, this.resolvePersist(opts.config));
+                this.persist(run);
                 this.logger(`[agentkit] eval run ${runId}: done — ${run.summary.passed} passed, ${run.summary.failed} failed`);
             }
             catch (error) {
@@ -322,7 +292,7 @@ export class EvalRunStore {
                 run.status = "failed";
                 run.finishedAt = new Date().toISOString();
                 run.error = error instanceof Error ? error.message : String(error);
-                yield this.enqueuePersist(run, this.resolvePersist(opts.config));
+                this.persist(run);
                 this.logger(`[agentkit] eval run ${runId}: failed — ${run.error}`);
             }
             finally {
@@ -383,6 +353,24 @@ function markInterruptedRun(run) {
             c.error = (_c = c.error) !== null && _c !== void 0 ? _c : INTERRUPTED_CASE_ERROR;
         }
     }
+    // Reconcile the summary with the repaired case rows so a persisted
+    // interrupted batch never claims pre-interruption pass counts.
+    let passed = 0;
+    let failed = 0;
+    for (const c of run.cases) {
+        if (c.ok === true) {
+            passed += 1;
+        }
+        else {
+            failed += 1;
+        }
+    }
+    run.summary = {
+        passed,
+        failed,
+        total: run.cases.length,
+        done: run.cases.length,
+    };
 }
 function snapshotListConfig(config, durableRuns) {
     return Object.assign(Object.assign(Object.assign({}, (config.maxConcurrency === undefined

package/dist/internal/evals-client.d.ts

@@ -0,0 +1,96 @@
+/**
+ * HTTP client for playground eval batches (`/v1/dev/evals/runs`).
+ *
+ * Used by `agentkit eval --prod` (or `--url`) so the batch runs on the serve
+ * process and is mirrored to `defineStorage` when an `evals` table is
+ * configured. Without those flags, eval keeps the in-process ephemeral path
+ * in `cli-ax.ts`.
+ */
+import type { EvalRunSnapshot } from "../evals.js";
+import type { EvalRunResult } from "./eval-runner.js";
+import { type RequestAuth } from "./request-headers.js";
+export type RemoteEvalErrorCode = "no_matching_evals" | "eval_run_in_progress" | "unknown_eval_run" | "start_failed" | "poll_failed" | "aborted";
+export declare class RemoteEvalError extends Error {
+    readonly status?: number | undefined;
+    readonly code?: RemoteEvalErrorCode | undefined;
+    readonly activeRunId?: string | undefined;
+    constructor(message: string, status?: number | undefined, code?: RemoteEvalErrorCode | undefined, activeRunId?: string | undefined);
+}
+export interface RemoteEvalListItem {
+    id: string;
+    fileId: string;
+    description?: string;
+    tags?: string[];
+}
+export interface RunRemoteEvalsOptions extends RequestAuth {
+    baseUrl: string;
+    filterIds?: string[];
+    tags?: string[];
+    /** Per-case timeout override forwarded to the serve process. */
+    timeoutMs?: number;
+    /** Ask the serve harness to emit verbose case logs (server-side). */
+    verbose?: boolean;
+    fetchImpl?: typeof fetch;
+    /** Poll interval while the batch is `running`. */
+    pollMs?: number;
+    /** Abort the wait (not the remote batch). */
+    signal?: AbortSignal;
+    /** Called each time the polled snapshot changes. */
+    onSnapshot?: (snap: EvalRunSnapshot) => void;
+}
+/**
+ * Start a batch on the target serve via `POST /v1/dev/evals/runs`, poll until
+ * terminal, and return CLI-shaped case results.
+ */
+export declare function runRemoteEvals(options: RunRemoteEvalsOptions): Promise<{
+    snapshot: EvalRunSnapshot;
+    results: EvalRunResult[];
+}>;
+/** List datapoints from `GET /v1/dev/evals` (same surface a remote run uses). */
+export declare function listRemoteEvals(args: {
+    baseUrl: string;
+    auth?: RequestAuth;
+    fetchImpl?: typeof fetch;
+    filterIds?: string[];
+    tags?: string[];
+    signal?: AbortSignal;
+}): Promise<RemoteEvalListItem[]>;
+export declare function startRemoteEvalRun(args: {
+    baseUrl: string;
+    auth?: RequestAuth;
+    fetchImpl?: typeof fetch;
+    filterIds?: string[];
+    tags?: string[];
+    timeoutMs?: number;
+    verbose?: boolean;
+    signal?: AbortSignal;
+}): Promise<EvalRunSnapshot>;
+/** One-shot `GET /v1/dev/evals/runs/:runId` (no polling). */
+export declare function getRemoteEvalRun(args: {
+    baseUrl: string;
+    runId: string;
+    auth?: RequestAuth;
+    fetchImpl?: typeof fetch;
+    signal?: AbortSignal;
+}): Promise<EvalRunSnapshot>;
+/** Recent batches from `GET /v1/dev/evals/runs`. */
+export declare function listRemoteEvalRuns(args: {
+    baseUrl: string;
+    auth?: RequestAuth;
+    fetchImpl?: typeof fetch;
+    signal?: AbortSignal;
+}): Promise<{
+    runs: EvalRunSnapshot[];
+    activeRunId?: string;
+}>;
+export declare function pollRemoteEvalRun(args: {
+    baseUrl: string;
+    runId: string;
+    auth?: RequestAuth;
+    fetchImpl?: typeof fetch;
+    pollMs?: number;
+    signal?: AbortSignal;
+    onSnapshot?: (snap: EvalRunSnapshot) => void;
+}): Promise<EvalRunSnapshot>;
+export declare function snapshotToEvalRunResults(snapshot: EvalRunSnapshot): EvalRunResult[];
+//# sourceMappingURL=evals-client.d.ts.map

package/dist/internal/evals-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evals-client.d.ts","sourceRoot":"","sources":["../../src/internal/evals-client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAuB,KAAK,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAI7E,MAAM,MAAM,mBAAmB,GAC3B,mBAAmB,GACnB,sBAAsB,GACtB,kBAAkB,GAClB,cAAc,GACd,aAAa,GACb,SAAS,CAAC;AAEd,qBAAa,eAAgB,SAAQ,KAAK;IAGtC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM;IACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,mBAAmB;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM;IAJ/B,YACE,OAAO,EAAE,MAAM,EACN,MAAM,CAAC,EAAE,MAAM,YAAA,EACf,IAAI,CAAC,EAAE,mBAAmB,YAAA,EAC1B,WAAW,CAAC,EAAE,MAAM,YAAA,EAI9B;CACF;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,qBAAsB,SAAQ,WAAW;IACxD,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,kDAAkD;IAClD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;CAC9C;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC;IAAE,QAAQ,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE,CAAC,CA+BlE;AAED,iFAAiF;AACjF,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAgEhC;AAED,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,eAAe,CAAC,CAuF3B;AAED,6DAA6D;AAC7D,wBAAsB,gBAAgB,CAAC,IAAI,EAAE;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,eAAe,CAAC,CAmC3B;AAED,oDAAoD;AACpD,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,eAAe,EAAE,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAyC7D;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;CAC9C,GAAG,OAAO,CAAC,eAAe,CAAC,CAoB3B;AAED,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,eAAe,GACxB,aAAa,EAAE,CAgBjB"}