@cursor/july 0.1.6 → 0.1.7

package/dist/internal/cli-ax.js

@@ -25,6 +25,7 @@ import { resolveDeployAuth } from "./cli-deploy.js";
 import { resolveApiKey } from "./cursor/credentials.js";
 import { CLI_COMMAND_NAME } from "./distribution.js";
 import { discoverEvals, filterDiscoveredEvals, runDiscoveredEvals, } from "./eval-runner.js";
+import { getRemoteEvalRun, listRemoteEvalRuns, listRemoteEvals, RemoteEvalError, runRemoteEvals, startRemoteEvalRun, } from "./evals-client.js";
 import { formatInitNextSteps, formatInitScaffoldSummary, initNextCommands, initProject, installScaffoldDependencies, printInitGuide, } from "./init-project.js";
 import { offerInstallCursorSkills } from "./install-cursor-skills.js";
 import { runLogsTail, sanitizeCustomerError } from "./logs-client.js";
@@ -476,26 +477,25 @@ export function cmdTrajectory(options) {
     });
 }
 /**
- * Which surface `eval` should target. Deployment is the default; `--local`
- * opts into an ephemeral local server, `--url` into a running one.
+ * Which surface `eval` should target. Local (ephemeral) is the default;
+ * `--prod` uses the hosted deployment, `--url` a running server.
  * Exported for tests.
  */
 export function resolveEvalTargetMode(options) {
     const flags = [
-        options.local === true ? "--local" : undefined,
         options.prod === true ? "--prod" : undefined,
         options.url !== undefined ? "--url" : undefined,
     ].filter((flag) => flag !== undefined);
     if (flags.length > 1) {
         return { error: `${flags.join(" and ")} are mutually exclusive` };
     }
-    if (options.local === true) {
-        return "local";
-    }
     if (options.url !== undefined) {
         return "url";
     }
-    return "deployment";
+    if (options.prod === true) {
+        return "deployment";
+    }
+    return "local";
 }
 /** Resolve the eval target; prints a CLI-ready error and returns an exit code on failure. */
 function resolveEvalTarget(options) {
@@ -528,13 +528,12 @@ function resolveEvalTarget(options) {
                 // disabled feature gate, and the message distinguishes those from a
                 // genuinely missing deployment.
                 process.stderr.write(`${sanitizeCustomerError(error.message)}\n` +
-                    `Evals run on the hosted deployment by default.\n` +
                     `  deploy it first:  ${CLI_COMMAND_NAME} deploy --slug ${error.slug}\n` +
-                    `  or run locally:   ${CLI_COMMAND_NAME} eval --local --dir ${options.dir}\n`);
+                    `  or omit --prod:   ${CLI_COMMAND_NAME} eval --dir ${options.dir}\n`);
                 return 1;
             }
             process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n` +
-                `Evals run on the hosted deployment by default; pass --local to run on an ephemeral local server.\n`);
+                `Pass --prod for the hosted deployment, or omit it to run on an ephemeral local server.\n`);
             return 1;
         }
     });
@@ -543,81 +542,385 @@ export function cmdEval(options) {
     return __awaiter(this, void 0, void 0, function* () {
         var _a;
         const projectRoot = resolve((_a = options.cwd) !== null && _a !== void 0 ? _a : process.cwd(), options.dir);
+        const mode = resolveEvalTargetMode(options);
+        if (typeof mode === "object") {
+            process.stderr.write(`${mode.error}\n`);
+            return 2;
+        }
+        if (options.noWait === true) {
+            if (options.list === true) {
+                process.stderr.write("--no-wait and --list are mutually exclusive\n");
+                return 2;
+            }
+            if (mode === "local") {
+                process.stderr.write("--no-wait requires --prod or --url (not the default local target)\n");
+                return 2;
+            }
+        }
+        const filterIds = options.evalIds === undefined || options.evalIds.length === 0
+            ? undefined
+            : options.evalIds;
+        const tags = options.tags === undefined || options.tags.length === 0
+            ? undefined
+            : options.tags;
         if (options.list) {
-            const { evals } = yield discoverEvals(projectRoot);
-            const listed = filterDiscoveredEvals(evals, {
-                filterIds: options.evalIds,
-                tags: options.tags,
-            });
-            if (options.json) {
-                console.log(JSON.stringify(listed.map((e) => { var _a; return ({
+            // List the same surface a run will execute: local filesystem by default,
+            // GET /v1/dev/evals on --prod / --url.
+            if (mode === "local") {
+                return listLocalEvals(projectRoot, options, filterIds, tags);
+            }
+            const target = yield resolveEvalTarget(options);
+            if (typeof target === "number") {
+                return target;
+            }
+            try {
+                const listed = yield listRemoteEvals({
+                    baseUrl: target.agentUrl,
+                    auth: {
+                        bearerToken: options.bearerToken,
+                        headers: target.headers,
+                    },
+                    fetchImpl: options.fetchImpl,
+                    filterIds,
+                    tags,
+                });
+                return printListedEvals(listed.map((e) => { var _a; return ({
                     id: e.id,
                     fileId: e.fileId,
-                    path: e.path,
-                    description: e.definition.description,
-                    tags: (_a = e.definition.tags) !== null && _a !== void 0 ? _a : [],
-                }); }), null, 2));
+                    description: e.description,
+                    tags: (_a = e.tags) !== null && _a !== void 0 ? _a : [],
+                }); }), options, `No evals on ${target.agentUrl}`);
             }
-            else if (listed.length === 0) {
-                process.stdout.write(`No evals under ${projectRoot}/evals\n`);
+            catch (error) {
+                process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
+                return 1;
             }
-            else {
-                for (const e of listed) {
-                    const tags = e.definition.tags && e.definition.tags.length > 0
-                        ? `  [${e.definition.tags.join(", ")}]`
-                        : "";
-                    const desc = e.definition.description !== undefined
-                        ? ` — ${e.definition.description}`
-                        : "";
-                    process.stdout.write(`${e.id}${tags}${desc}\n`);
+            finally {
+                if (target.close !== undefined) {
+                    yield target.close();
                 }
             }
-            return 0;
         }
         const target = yield resolveEvalTarget(options);
         if (typeof target === "number") {
             return target;
         }
         const stream = shouldStreamProgress(options);
-        const progress = stream ? createStreamProgress() : undefined;
         try {
             if (options.verbose || stream) {
                 process.stderr.write(`eval target: ${target.agentUrl}\n`);
             }
-            const results = yield runDiscoveredEvals({
-                projectRoot,
+            // Default local: ephemeral in-process harness against local eval sources.
+            // --prod / --url: run on the serve process via /v1/dev/evals/runs so the
+            // batch lands in playground history and defineStorage evals (when set).
+            if (mode === "local") {
+                const progress = stream ? createStreamProgress() : undefined;
+                try {
+                    const results = yield runDiscoveredEvals({
+                        projectRoot,
+                        baseUrl: target.agentUrl,
+                        filterIds,
+                        tags,
+                        timeoutMs: options.timeoutMs,
+                        bearerToken: options.bearerToken,
+                        headers: target.headers,
+                        verbose: Boolean(options.verbose) || stream,
+                        onEvent: progress === undefined ? undefined : (e) => progress.onEvent(e),
+                    });
+                    if (results.length === 0) {
+                        process.stderr.write("No matching evals found.\n");
+                        return 2;
+                    }
+                    return printEvalResults(results, options);
+                }
+                finally {
+                    progress === null || progress === void 0 ? void 0 : progress.close();
+                }
+            }
+            try {
+                if (options.noWait === true) {
+                    const started = yield startRemoteEvalRun({
+                        baseUrl: target.agentUrl,
+                        filterIds,
+                        tags,
+                        timeoutMs: options.timeoutMs,
+                        verbose: Boolean(options.verbose),
+                        auth: {
+                            bearerToken: options.bearerToken,
+                            headers: target.headers,
+                        },
+                        fetchImpl: options.fetchImpl,
+                    });
+                    return printEvalAccepted(started, options, target.agentUrl);
+                }
+                const progress = createRemoteEvalProgress(options.verbose === true || stream);
+                const { snapshot, results } = yield runRemoteEvals({
+                    baseUrl: target.agentUrl,
+                    filterIds,
+                    tags,
+                    timeoutMs: options.timeoutMs,
+                    verbose: Boolean(options.verbose) || stream,
+                    bearerToken: options.bearerToken,
+                    headers: target.headers,
+                    fetchImpl: options.fetchImpl,
+                    onSnapshot: progress === undefined ? undefined : progress.onSnapshot,
+                });
+                if (results.length === 0 && snapshot.status !== "failed") {
+                    process.stderr.write("No matching evals found.\n");
+                    return 2;
+                }
+                return printEvalResults(results, options, {
+                    runId: snapshot.runId,
+                    status: snapshot.status,
+                    error: snapshot.error,
+                });
+            }
+            catch (error) {
+                if (error instanceof RemoteEvalError &&
+                    error.code === "no_matching_evals") {
+                    process.stderr.write(`${error.message}\n`);
+                    return 2;
+                }
+                process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
+                return 1;
+            }
+        }
+        finally {
+            if (target.close !== undefined) {
+                yield target.close();
+            }
+        }
+    });
+}
+/**
+ * Flags to re-target the same surface from an `eval --no-wait` hint.
+ * Exported for tests.
+ */
+export function formatEvalStatusFlags(options) {
+    const parts = [];
+    if (options.url !== undefined) {
+        parts.push(`--url ${options.url.replace(/\/$/, "")}`);
+    }
+    else {
+        parts.push("--prod");
+        if (options.slug !== undefined && options.slug !== "") {
+            parts.push(`--slug ${options.slug}`);
+        }
+    }
+    if (options.team !== undefined && options.team !== "") {
+        parts.push(`--team ${options.team}`);
+    }
+    return parts.length === 0 ? "" : ` ${parts.join(" ")}`;
+}
+/** `--no-wait`: print the accepted batch id (and how to poll) then exit. */
+function printEvalAccepted(started, options, agentUrl) {
+    const pollPath = `${agentUrl.replace(/\/$/, "")}/v1/dev/evals/runs/${started.runId}`;
+    const statusFlags = formatEvalStatusFlags(options);
+    if (options.json) {
+        console.log(JSON.stringify({
+            ok: true,
+            accepted: true,
+            runId: started.runId,
+            status: started.status,
+            summary: started.summary,
+            pollUrl: pollPath,
+            durableRuns: started.config.durableRuns === true,
+        }, null, 2));
+    }
+    else {
+        const durable = started.config.durableRuns === true ? " (persisted)" : "";
+        process.stdout.write(`Accepted eval run ${started.runId}${durable} — ${started.summary.total} case(s)\n`);
+        process.stdout.write(`Status: ${CLI_COMMAND_NAME} eval status ${started.runId}${statusFlags}\n`);
+        process.stdout.write(`Poll: ${pollPath}\n`);
+    }
+    return 0;
+}
+/**
+ * `eval status [runId]` — fetch one batch (or list recent) on the deployment /
+ * `--url` target. Exit `0` when the batch is completed with all cases passing,
+ * `1` when failed / any case failed / unknown, `2` for usage errors, `3` when
+ * the batch is still running (for poll loops after `--no-wait`).
+ */
+export function cmdEvalStatus(runId, options) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const mode = resolveEvalTargetMode(options);
+        if (typeof mode === "object") {
+            process.stderr.write(`${mode.error}\n`);
+            return 2;
+        }
+        if (mode === "local") {
+            process.stderr.write("eval status requires --prod or --url (not the default local target)\n");
+            return 2;
+        }
+        if (options.list === true || options.noWait === true) {
+            process.stderr.write("eval status does not accept --list or --no-wait\n");
+            return 2;
+        }
+        const target = yield resolveEvalTarget(options);
+        if (typeof target === "number") {
+            return target;
+        }
+        const auth = {
+            bearerToken: options.bearerToken,
+            headers: target.headers,
+        };
+        try {
+            if (runId === undefined || runId === "") {
+                const { runs, activeRunId } = yield listRemoteEvalRuns({
+                    baseUrl: target.agentUrl,
+                    auth,
+                    fetchImpl: options.fetchImpl,
+                });
+                if (options.json) {
+                    console.log(JSON.stringify({ runs, activeRunId }, null, 2));
+                }
+                else if (runs.length === 0) {
+                    process.stdout.write("No eval runs on this agent.\n");
+                }
+                else {
+                    if (activeRunId !== undefined) {
+                        process.stdout.write(`active: ${activeRunId}\n`);
+                    }
+                    for (const run of runs) {
+                        process.stdout.write(`${formatEvalRunSummaryLine(run)}\n`);
+                    }
+                }
+                return 0;
+            }
+            const snap = yield getRemoteEvalRun({
                 baseUrl: target.agentUrl,
-                filterIds: options.evalIds === undefined || options.evalIds.length === 0
-                    ? undefined
-                    : options.evalIds,
-                tags: options.tags === undefined || options.tags.length === 0
-                    ? undefined
-                    : options.tags,
-                timeoutMs: options.timeoutMs,
-                bearerToken: options.bearerToken,
-                headers: target.headers,
-                verbose: Boolean(options.verbose) || stream,
-                onEvent: progress === undefined ? undefined : (e) => progress.onEvent(e),
+                runId,
+                auth,
+                fetchImpl: options.fetchImpl,
             });
-            if (results.length === 0) {
-                process.stderr.write("No matching evals found.\n");
-                return 2;
+            if (options.json) {
+                console.log(JSON.stringify(snap, null, 2));
+            }
+            else {
+                process.stdout.write(`${formatEvalRunSummaryLine(snap)}\n`);
+                if (snap.error !== undefined && snap.error !== "") {
+                    process.stdout.write(`error: ${snap.error}\n`);
+                }
+                for (const c of snap.cases) {
+                    const mark = c.status !== "done" ? c.status : c.ok === true ? "PASS" : "FAIL";
+                    const dur = c.durationMs === undefined
+                        ? ""
+                        : ` (${(c.durationMs / 1000).toFixed(1)}s)`;
+                    process.stdout.write(`  ${mark}  ${c.id}${dur}\n`);
+                    if (c.error !== undefined && c.error !== "") {
+                        process.stdout.write(`    error: ${c.error}\n`);
+                    }
+                }
+            }
+            if (snap.status === "running") {
+                return 3;
             }
-            return printEvalResults(results, options);
+            const failed = snap.status === "failed" || snap.cases.some((c) => c.ok !== true);
+            return failed ? 1 : 0;
+        }
+        catch (error) {
+            process.stderr.write(`${sanitizeCustomerError(error instanceof Error ? error.message : String(error))}\n`);
+            return 1;
         }
         finally {
-            progress === null || progress === void 0 ? void 0 : progress.close();
             if (target.close !== undefined) {
                 yield target.close();
             }
         }
     });
 }
-function printEvalResults(results, options) {
+function formatEvalRunSummaryLine(run) {
+    const { passed, failed, total, done } = run.summary;
+    const durable = run.config.durableRuns === true ? " persisted" : "";
+    return `${run.runId}  ${run.status}  ${done}/${total} done (${passed} passed, ${failed} failed)${durable}`;
+}
+function listLocalEvals(projectRoot, options, filterIds, tags) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const { evals } = yield discoverEvals(projectRoot);
+        const listed = filterDiscoveredEvals(evals, { filterIds, tags });
+        return printListedEvals(listed.map((e) => { var _a; return ({
+            id: e.id,
+            fileId: e.fileId,
+            path: e.path,
+            description: e.definition.description,
+            tags: (_a = e.definition.tags) !== null && _a !== void 0 ? _a : [],
+        }); }), options, `No evals under ${projectRoot}/evals`);
+    });
+}
+function printListedEvals(listed, options, emptyMessage) {
+    if (options.json) {
+        console.log(JSON.stringify(listed, null, 2));
+    }
+    else if (listed.length === 0) {
+        process.stdout.write(`${emptyMessage}\n`);
+    }
+    else {
+        for (const e of listed) {
+            const tagSuffix = e.tags.length > 0 ? `  [${e.tags.join(", ")}]` : "";
+            const desc = e.description !== undefined ? ` — ${e.description}` : "";
+            process.stdout.write(`${e.id}${tagSuffix}${desc}\n`);
+        }
+    }
+    return 0;
+}
+/**
+ * Remote batches don't stream turn events to the CLI; print run/case progress
+ * (and case logs once they appear on the snapshot) instead.
+ */
+function createRemoteEvalProgress(enabled) {
+    if (!enabled) {
+        return undefined;
+    }
+    let announced = false;
+    let lastSummaryKey = "";
+    const seenCaseKeys = new Set();
+    return {
+        onSnapshot(snap) {
+            var _a, _b;
+            if (!announced) {
+                announced = true;
+                const durable = snap.config.durableRuns === true ? " (persisted)" : "";
+                process.stderr.write(`eval run: ${snap.runId}${durable}\n`);
+            }
+            const summaryKey = `${snap.status}:${snap.summary.done}:${snap.summary.passed}:${snap.summary.failed}`;
+            if (summaryKey !== lastSummaryKey) {
+                lastSummaryKey = summaryKey;
+                const { done, total, passed, failed } = snap.summary;
+                process.stderr.write(`eval progress: ${done}/${total} done (${passed} passed, ${failed} failed) [${snap.status}]\n`);
+            }
+            for (const c of snap.cases) {
+                if (c.status !== "done") {
+                    continue;
+                }
+                const caseKey = `${c.id}:${c.ok === true ? "pass" : "fail"}:${(_a = c.durationMs) !== null && _a !== void 0 ? _a : 0}`;
+                if (seenCaseKeys.has(caseKey)) {
+                    continue;
+                }
+                seenCaseKeys.add(caseKey);
+                process.stderr.write(`  eval ${c.id}: ${c.ok === true ? "PASS" : "FAIL"}${c.error !== undefined ? ` — ${c.error}` : ""}\n`);
+                for (const line of (_b = c.logs) !== null && _b !== void 0 ? _b : []) {
+                    process.stderr.write(`    ${line}\n`);
+                }
+            }
+        },
+    };
+}
+function printEvalResults(results, options, batch) {
     const passed = results.filter((r) => r.ok).length;
     const failed = results.length - passed;
+    const batchFailed = (batch === null || batch === void 0 ? void 0 : batch.status) === "failed";
+    const ok = failed === 0 && !batchFailed;
+    if ((batch === null || batch === void 0 ? void 0 : batch.error) !== undefined && batch.error !== "" && !options.json) {
+        // Always surface harness-level failures (not only --verbose).
+        process.stderr.write(`eval run error: ${batch.error}\n`);
+    }
     if (options.json) {
-        console.log(JSON.stringify({ ok: failed === 0, passed, failed, results }, null, 2));
+        console.log(JSON.stringify(Object.assign(Object.assign(Object.assign({ ok,
+            passed,
+            failed,
+            results }, ((batch === null || batch === void 0 ? void 0 : batch.runId) === undefined ? {} : { runId: batch.runId })), ((batch === null || batch === void 0 ? void 0 : batch.status) === undefined ? {} : { status: batch.status })), ((batch === null || batch === void 0 ? void 0 : batch.error) === undefined || batch.error === ""
+            ? {}
+            : { error: batch.error })), null, 2));
     }
     else {
         for (const r of results) {
@@ -638,7 +941,7 @@ function printEvalResults(results, options) {
         }
         process.stdout.write(`\n${passed} passed, ${failed} failed, ${results.length} total\n`);
     }
-    return failed === 0 ? 0 : 1;
+    return ok ? 0 : 1;
 }
 /**
  * `logs` — follow deploy progress (`--prod`) then runtime agent logs.

package/dist/internal/eval-run-store.d.ts

@@ -25,6 +25,8 @@ export interface StartEvalRunInput {
     tags?: string[];
     bearerToken?: string;
     verbose?: boolean;
+    /** Per-case timeout override (same semantics as CLI `--timeout-ms`). */
+    timeoutMs?: number;
 }
 export type EvalListConfig = {
     maxConcurrency?: number;

package/dist/internal/eval-run-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"eval-run-store.d.ts","sourceRoot":"","sources":["../../src/internal/eval-run-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAGL,KAAK,eAAe,EAErB,MAAM,aAAa,CAAC;AASrB,YAAY,EACV,eAAe,EACf,iBAAiB,IAAI,cAAc,EACnC,gBAAgB,EAChB,eAAe,GAChB,MAAM,aAAa,CAAC;AAErB;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,CAAC;IACjC,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAOF,qBAAa,YAAY;IASrB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;IAX3B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsC;IAC3D,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,cAAc,CAA4B;IAClD,OAAO,CAAC,QAAQ,CAAS;IACzB,wEAAwE;IACxE,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAEhC,YACmB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI;IAC/C,oEAAoE;IACnD,OAAO,CAAC,EAAE,cAAc,GAAG,SAAS,EACnD;IAEE,IAAI,IAAI,OAAO,CAAC;QACpB,KAAK,EAAE,KAAK,CAAC;YACX,EAAE,EAAE,MAAM,CAAC;YACX,MAAM,EAAE,MAAM,CAAC;YACf,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC,CAAC;QACH,MAAM,EAAE,cAAc,CAAC;KACxB,CAAC,CAcD;IAEK,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAG7D;IAED,+EAA+E;IACzE,QAAQ,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAG3C;IAED;;;;OAIG;IACG,kBAAkB,IAAI,OAAO,CAAC;QAClC,IAAI,EAAE,eAAe,EAAE,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAOD;IAED,cAAc,IAAI,MAAM,GAAG,SAAS,CAEnC;IAEK,KAAK,CAAC,KAAK,GAAE,iBAAsB,GAAG,OAAO,CAAC,eAAe,CAAC,CA4DnE;IAED,OAAO,CAAC,UAAU;IAMlB;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ7B;YAEa,kBAAkB;IAkDhC,OAAO,CAAC,YAAY;IAepB,6EAA6E;IAC7E,OAAO,CAAC,OAAO;IAIf;;;;;;OAMG;IACH,KAAK,IAAI,IAAI,CAYZ;YAEa,OAAO;IAoGrB,OAAO,CAAC,eAAe;CAqCxB;AAyED,qBAAa,oBAAqB,SAAQ,KAAK;IACjC,QAAQ,CAAC,WAAW,EAAE,MAAM;IAAxC,YAAqB,WAAW,EAAE,MAAM,EAGvC;CACF;AAED,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,cAGC;CACF"}
+{"version":3,"file":"eval-run-store.d.ts","sourceRoot":"","sources":["../../src/internal/eval-run-store.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAGL,KAAK,eAAe,EAErB,MAAM,aAAa,CAAC;AASrB,YAAY,EACV,eAAe,EACf,iBAAiB,IAAI,cAAc,EACnC,gBAAgB,EAChB,eAAe,GAChB,MAAM,aAAa,CAAC;AAErB;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,IAAI,CAAC,GAAG,EAAE,eAAe,GAAG,IAAI,CAAC;IACjC,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,IAAI,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,iBAAiB;IAChC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,wEAAwE;IACxE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,MAAM,cAAc,GAAG;IAC3B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,WAAW,EAAE,OAAO,CAAC;CACtB,CAAC;AAOF,qBAAa,YAAY;IASrB,OAAO,CAAC,QAAQ,CAAC,WAAW;IAC5B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,oEAAoE;IACpE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC;IAX3B,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAsC;IAC3D,OAAO,CAAC,WAAW,CAAqB;IACxC,OAAO,CAAC,cAAc,CAA4B;IAClD,OAAO,CAAC,QAAQ,CAAS;IACzB,wEAAwE;IACxE,WAAW,EAAE,MAAM,GAAG,SAAS,CAAC;IAEhC,YACmB,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI;IAC/C,oEAAoE;IACnD,OAAO,CAAC,EAAE,cAAc,GAAG,SAAS,EACnD;IAEE,IAAI,IAAI,OAAO,CAAC;QACpB,KAAK,EAAE,KAAK,CAAC;YACX,EAAE,EAAE,MAAM,CAAC;YACX,MAAM,EAAE,MAAM,CAAC;YACf,WAAW,CAAC,EAAE,MAAM,CAAC;YACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;SACjB,CAAC,CAAC;QACH,MAAM,EAAE,cAAc,CAAC;KACxB,CAAC,CAcD;IAEK,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC,CAG7D;IAED,+EAA+E;IACzE,QAAQ,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC,CAG3C;IAED;;;;OAIG;IACG,kBAAkB,IAAI,OAAO,CAAC;QAClC,IAAI,EAAE,eAAe,EAAE,CAAC;QACxB,WAAW,CAAC,EAAE,MAAM,CAAC;KACtB,CAAC,CAOD;IAED,cAAc,IAAI,MAAM,GAAG,SAAS,CAEnC;IAEK,KAAK,CAAC,KAAK,GAAE,iBAAsB,GAAG,OAAO,CAAC,eAAe,CAAC,CA6DnE;IAED,OAAO,CAAC,UAAU;IAMlB;;;;;OAKG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAQ7B;YAEa,kBAAkB;IAkDhC,OAAO,CAAC,YAAY;IAepB,6EAA6E;IAC7E,OAAO,CAAC,OAAO;IAIf;;;;;;OAMG;IACH,KAAK,IAAI,IAAI,CAYZ;YAEa,OAAO;IAsGrB,OAAO,CAAC,eAAe;CAqCxB;AAyED,qBAAa,oBAAqB,SAAQ,KAAK;IACjC,QAAQ,CAAC,WAAW,EAAE,MAAM;IAAxC,YAAqB,WAAW,EAAE,MAAM,EAGvC;CACF;AAED,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,cAGC;CACF"}

package/dist/internal/eval-run-store.js

@@ -112,9 +112,9 @@ export class EvalRunStore {
             this.pruneOldRuns(config);
             this.persist(snapshot);
             this.logger(`[agentkit] eval run ${runId}: starting ${cases.length} case(s)`);
-            void this.execute(runId, Object.assign(Object.assign({ baseUrl: loopbackUrl, evals: selected, config }, (input.bearerToken === undefined
+            void this.execute(runId, Object.assign(Object.assign(Object.assign({ baseUrl: loopbackUrl, evals: selected, config }, (input.bearerToken === undefined
                 ? {}
-                : { bearerToken: input.bearerToken })), (input.verbose === undefined ? {} : { verbose: input.verbose })));
+                : { bearerToken: input.bearerToken })), (input.verbose === undefined ? {} : { verbose: input.verbose })), (input.timeoutMs === undefined ? {} : { timeoutMs: input.timeoutMs })));
             return snapshot;
         });
     }
@@ -228,9 +228,9 @@ export class EvalRunStore {
                 return;
             }
             try {
-                yield runDiscoveredEvals(Object.assign(Object.assign(Object.assign({ projectRoot: this.projectRoot, baseUrl: opts.baseUrl, discovered: { evals: opts.evals, config: opts.config } }, (opts.bearerToken === undefined
+                yield runDiscoveredEvals(Object.assign(Object.assign(Object.assign(Object.assign({ projectRoot: this.projectRoot, baseUrl: opts.baseUrl, discovered: { evals: opts.evals, config: opts.config } }, (opts.bearerToken === undefined
                     ? {}
-                    : { bearerToken: opts.bearerToken })), (opts.verbose === undefined ? {} : { verbose: opts.verbose })), { onCaseStart: (id) => {
+                    : { bearerToken: opts.bearerToken })), (opts.verbose === undefined ? {} : { verbose: opts.verbose })), (opts.timeoutMs === undefined ? {} : { timeoutMs: opts.timeoutMs })), { onCaseStart: (id) => {
                         const run = this.runs.get(runId);
                         if (run === undefined) {
                             return;

package/dist/internal/evals-client.d.ts

@@ -0,0 +1,96 @@
+/**
+ * HTTP client for playground eval batches (`/v1/dev/evals/runs`).
+ *
+ * Used by `agentkit eval --prod` (or `--url`) so the batch runs on the serve
+ * process and is mirrored to `defineStorage` when an `evals` table is
+ * configured. Without those flags, eval keeps the in-process ephemeral path
+ * in `cli-ax.ts`.
+ */
+import type { EvalRunSnapshot } from "../evals.js";
+import type { EvalRunResult } from "./eval-runner.js";
+import { type RequestAuth } from "./request-headers.js";
+export type RemoteEvalErrorCode = "no_matching_evals" | "eval_run_in_progress" | "unknown_eval_run" | "start_failed" | "poll_failed" | "aborted";
+export declare class RemoteEvalError extends Error {
+    readonly status?: number | undefined;
+    readonly code?: RemoteEvalErrorCode | undefined;
+    readonly activeRunId?: string | undefined;
+    constructor(message: string, status?: number | undefined, code?: RemoteEvalErrorCode | undefined, activeRunId?: string | undefined);
+}
+export interface RemoteEvalListItem {
+    id: string;
+    fileId: string;
+    description?: string;
+    tags?: string[];
+}
+export interface RunRemoteEvalsOptions extends RequestAuth {
+    baseUrl: string;
+    filterIds?: string[];
+    tags?: string[];
+    /** Per-case timeout override forwarded to the serve process. */
+    timeoutMs?: number;
+    /** Ask the serve harness to emit verbose case logs (server-side). */
+    verbose?: boolean;
+    fetchImpl?: typeof fetch;
+    /** Poll interval while the batch is `running`. */
+    pollMs?: number;
+    /** Abort the wait (not the remote batch). */
+    signal?: AbortSignal;
+    /** Called each time the polled snapshot changes. */
+    onSnapshot?: (snap: EvalRunSnapshot) => void;
+}
+/**
+ * Start a batch on the target serve via `POST /v1/dev/evals/runs`, poll until
+ * terminal, and return CLI-shaped case results.
+ */
+export declare function runRemoteEvals(options: RunRemoteEvalsOptions): Promise<{
+    snapshot: EvalRunSnapshot;
+    results: EvalRunResult[];
+}>;
+/** List datapoints from `GET /v1/dev/evals` (same surface a remote run uses). */
+export declare function listRemoteEvals(args: {
+    baseUrl: string;
+    auth?: RequestAuth;
+    fetchImpl?: typeof fetch;
+    filterIds?: string[];
+    tags?: string[];
+    signal?: AbortSignal;
+}): Promise<RemoteEvalListItem[]>;
+export declare function startRemoteEvalRun(args: {
+    baseUrl: string;
+    auth?: RequestAuth;
+    fetchImpl?: typeof fetch;
+    filterIds?: string[];
+    tags?: string[];
+    timeoutMs?: number;
+    verbose?: boolean;
+    signal?: AbortSignal;
+}): Promise<EvalRunSnapshot>;
+/** One-shot `GET /v1/dev/evals/runs/:runId` (no polling). */
+export declare function getRemoteEvalRun(args: {
+    baseUrl: string;
+    runId: string;
+    auth?: RequestAuth;
+    fetchImpl?: typeof fetch;
+    signal?: AbortSignal;
+}): Promise<EvalRunSnapshot>;
+/** Recent batches from `GET /v1/dev/evals/runs`. */
+export declare function listRemoteEvalRuns(args: {
+    baseUrl: string;
+    auth?: RequestAuth;
+    fetchImpl?: typeof fetch;
+    signal?: AbortSignal;
+}): Promise<{
+    runs: EvalRunSnapshot[];
+    activeRunId?: string;
+}>;
+export declare function pollRemoteEvalRun(args: {
+    baseUrl: string;
+    runId: string;
+    auth?: RequestAuth;
+    fetchImpl?: typeof fetch;
+    pollMs?: number;
+    signal?: AbortSignal;
+    onSnapshot?: (snap: EvalRunSnapshot) => void;
+}): Promise<EvalRunSnapshot>;
+export declare function snapshotToEvalRunResults(snapshot: EvalRunSnapshot): EvalRunResult[];
+//# sourceMappingURL=evals-client.d.ts.map

package/dist/internal/evals-client.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"evals-client.d.ts","sourceRoot":"","sources":["../../src/internal/evals-client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAuB,KAAK,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAI7E,MAAM,MAAM,mBAAmB,GAC3B,mBAAmB,GACnB,sBAAsB,GACtB,kBAAkB,GAClB,cAAc,GACd,aAAa,GACb,SAAS,CAAC;AAEd,qBAAa,eAAgB,SAAQ,KAAK;IAGtC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM;IACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,mBAAmB;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM;IAJ/B,YACE,OAAO,EAAE,MAAM,EACN,MAAM,CAAC,EAAE,MAAM,YAAA,EACf,IAAI,CAAC,EAAE,mBAAmB,YAAA,EAC1B,WAAW,CAAC,EAAE,MAAM,YAAA,EAI9B;CACF;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,qBAAsB,SAAQ,WAAW;IACxD,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,kDAAkD;IAClD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;CAC9C;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC;IAAE,QAAQ,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE,CAAC,CA+BlE;AAED,iFAAiF;AACjF,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAgEhC;AAED,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,eAAe,CAAC,CAuF3B;AAED,6DAA6D;AAC7D,wBAAsB,gBAAgB,CAAC,IAAI,EAAE;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,eAAe,CAAC,CAmC3B;AAED,oDAAoD;AACpD,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,eAAe,EAAE,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAyC7D;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;CAC9C,GAAG,OAAO,CAAC,eAAe,CAAC,CAoB3B;AAED,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,eAAe,GACxB,aAAa,EAAE,CAgBjB"}