@cursor/july 0.1.15 → 0.1.17

package/dist/internal/eval-runner.js

@@ -19,8 +19,15 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 import { readdir } from "node:fs/promises";
 import { join, relative, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
-import { EVAL_MAX_CONCURRENCY_LIMIT, expandEvalDefinition, matchesEvalIdFilter, normalizeEvalMaxConcurrency, } from "../evals.js";
+import { computeVerdict, createAssertions, EvalAssertionRecorder, EvalRequireFailure, EvalSkipSignal, } from "../evals/assertions.js";
+import { evaluateExpectation } from "../evals/expect.js";
+import { EVAL_JUDGE_GRADERS, EvalJudgeUnavailableError, renderSubmission, } from "../evals/judge.js";
+import { EVAL_ROOT_ENV } from "../evals/loaders.js";
+import { combineReporters } from "../evals/reporters.js";
+import { summarizeEvalResults } from "../evals/results.js";
+import { EVAL_MAX_CONCURRENCY_LIMIT, expandEvalDefinition, expandSummarySuiteSelection, isEvalSummaryCase, matchesEvalIdFilter, normalizeEvalMaxConcurrency, } from "../evals.js";
 import { isDefinitionOfKind } from "./brand.js";
+import { callJudgeModel, EVAL_JUDGE_MODEL_ENV, resolveJudgeModel, } from "./eval-judge-model.js";
 import { mapPool } from "./map-pool.js";
 import { RunSessionTimeoutError, runSession } from "./run-client.js";
 import { buildTrajectory } from "./trajectory.js";
@@ -69,6 +76,10 @@ export function discoverEvals(projectRoot_1) {
         const files = yield listEvalFiles(evalsDir);
         const evals = [];
         let config = {};
+        // Fixture loaders resolve relative paths against the app root, so a dataset
+        // path reads the same however deep the eval file sits and whatever cwd the
+        // CLI ran from.
+        process.env[EVAL_ROOT_ENV] = root;
         for (const abs of files) {
             const rel = relative(evalsDir, abs).replace(/\\/g, "/");
             if (rel === "evals.config.ts" || rel === "evals.config.js") {
@@ -82,13 +93,9 @@ export function discoverEvals(projectRoot_1) {
                 continue;
             }
             const mod = yield importModule(abs);
-            const def = mod.default;
-            if (!isDefinitionOfKind(def, "eval")) {
-                throw new Error(`${rel}: default export must be defineEval({ ... })`);
-            }
             const fileId = rel.replace(/\.eval\.(ts|js)$/, "").replace(/\/index$/, "");
             try {
-                for (const expanded of expandEvalDefinition(fileId, def)) {
+                for (const expanded of expandEvalModule(fileId, mod.default)) {
                     evals.push({
                         id: expanded.id,
                         fileId,
@@ -105,6 +112,32 @@ export function discoverEvals(projectRoot_1) {
         return { evals, config };
     });
 }
+/**
+ * Expand one eval module's default export into runnable datapoints.
+ *
+ * A file normally exports a single `defineEval(...)`. It may instead export an
+ * **array** of them to fan out over a dataset, in which case ids come from the
+ * file name plus a zero-padded index in array order (`sql/0000`, `sql/0001`, …).
+ * Prefer `cases` when the datapoints are hand-written and deserve stable names;
+ * the array form is for generated datasets where the index *is* the name.
+ */
+export function expandEvalModule(fileId, defaultExport) {
+    if (Array.isArray(defaultExport)) {
+        if (defaultExport.length === 0) {
+            throw new Error("default export array is empty; export at least one defineEval({ ... })");
+        }
+        return defaultExport.flatMap((entry, index) => {
+            if (!isDefinitionOfKind(entry, "eval")) {
+                throw new Error(`default export array entry ${index} must be defineEval({ ... })`);
+            }
+            return expandEvalDefinition(`${fileId}/${String(index).padStart(4, "0")}`, entry);
+        });
+    }
+    if (!isDefinitionOfKind(defaultExport, "eval")) {
+        throw new Error("default export must be defineEval({ ... }) or an array of them");
+    }
+    return expandEvalDefinition(fileId, defaultExport);
+}
 /** Apply id/tag filters used by CLI, playground, and `runDiscoveredEvals`. */
 export function filterDiscoveredEvals(evals, options = {}) {
     let selected = evals;
@@ -114,11 +147,12 @@ export function filterDiscoveredEvals(evals, options = {}) {
     if (options.tags !== undefined && options.tags.length > 0) {
         selected = selected.filter((e) => { var _a; return ((_a = e.definition.tags) !== null && _a !== void 0 ? _a : []).some((t) => options.tags.includes(t)); });
     }
-    return selected;
+    // Suite aggregators (`summary` tag) need sibling cases from the same file.
+    return expandSummarySuiteSelection(evals, selected, (e) => isEvalSummaryCase(e.definition.tags));
 }
 export function runDiscoveredEvals(options) {
     return __awaiter(this, void 0, void 0, function* () {
-        var _a, _b, _c;
+        var _a, _b, _c, _d;
         let selected;
         let config;
         if (options.discovered !== undefined) {
@@ -137,14 +171,20 @@ export function runDiscoveredEvals(options) {
         if (selected.length === 0) {
             return [];
         }
-        const maxConcurrency = resolveEvalMaxConcurrency(config);
+        const maxConcurrency = options.maxConcurrency === undefined
+            ? resolveEvalMaxConcurrency(config)
+            : normalizeEvalMaxConcurrency(options.maxConcurrency);
         const defaultTimeoutMs = (_c = (_b = options.timeoutMs) !== null && _b !== void 0 ? _b : config.timeoutMs) !== null && _c !== void 0 ? _c : 180000;
+        const reporter = buildReporterHub(options, config, selected);
+        const startedAt = new Date().toISOString();
+        yield reporter === null || reporter === void 0 ? void 0 : reporter.runStart(selected, (_d = options.target) !== null && _d !== void 0 ? _d : { baseUrl: options.baseUrl, mode: "local" });
         // Iterations of the same authored datapoint share one concurrency slot and
         // run sequentially. maxConcurrency is therefore the max number of distinct
         // datapoints in flight, not datapoints × iterations.
         const groups = groupEvalsForIterationSchedule(selected);
         const nested = yield mapPool(groups, maxConcurrency, (group) => __awaiter(this, void 0, void 0, function* () {
-            var _a, _b, _c;
+            var _a, _b, _c, _d;
+            var _e;
             const results = [];
             for (const discovered of group) {
                 if ((_a = options.signal) === null || _a === void 0 ? void 0 : _a.aborted) {
@@ -161,20 +201,92 @@ export function runDiscoveredEvals(options) {
                     headers: options.headers,
                     verbose: Boolean(options.verbose),
                     signal: options.signal,
+                    judge: {
+                        configModel: (_c = config.judge) === null || _c === void 0 ? void 0 : _c.model,
+                        caller: (_e = options.judgeCaller) !== null && _e !== void 0 ? _e : callJudgeModel,
+                    },
                     onEvent: (event) => {
                         var _a, _b;
                         (_a = options.onCaseEvent) === null || _a === void 0 ? void 0 : _a.call(options, discovered.id, event);
                         (_b = options.onEvent) === null || _b === void 0 ? void 0 : _b.call(options, event);
                     },
                 });
-                (_c = options.onCaseDone) === null || _c === void 0 ? void 0 : _c.call(options, result);
+                (_d = options.onCaseDone) === null || _d === void 0 ? void 0 : _d.call(options, result);
+                yield reporter === null || reporter === void 0 ? void 0 : reporter.caseComplete(discovered, result);
                 results.push(result);
             }
             return results;
         }), { signal: options.signal });
-        return nested.flat();
+        const results = nested.flat();
+        yield reporter === null || reporter === void 0 ? void 0 : reporter.runComplete(summarizeEvalResults(results, {
+            strict: options.strict === true,
+            startedAt,
+            finishedAt: new Date().toISOString(),
+        }));
+        return results;
     });
 }
+/**
+ * Fan results out to the reporters that observe them.
+ *
+ * Run-wide reporters (CLI flags plus `evals.config.ts`) see every case; an
+ * eval's own `reporters` only see that eval. The run lifecycle hooks fire on
+ * every reporter in the run, so a per-eval destination still gets its
+ * `onRunStart` / `onRunComplete` pair.
+ */
+function buildReporterHub(options, config, selected) {
+    var _a, _b;
+    const skip = options.skipReport === true;
+    const runWide = [
+        ...((_a = options.reporters) !== null && _a !== void 0 ? _a : []),
+        ...(skip ? [] : ((_b = config.reporters) !== null && _b !== void 0 ? _b : [])),
+    ];
+    const perEval = skip
+        ? []
+        : selected.flatMap((e) => { var _a; return (_a = e.definition.reporters) !== null && _a !== void 0 ? _a : []; });
+    const everyReporter = [...new Set([...runWide, ...perEval])];
+    if (everyReporter.length === 0) {
+        return undefined;
+    }
+    const onError = (error) => {
+        // A broken destination must not fail the eval run itself.
+        const detail = error instanceof Error ? error.message : String(error);
+        process.stderr.write(`eval reporter error: ${detail}\n`);
+    };
+    const all = combineReporters(everyReporter, onError);
+    return {
+        runStart(evals, target) {
+            return __awaiter(this, void 0, void 0, function* () {
+                var _a;
+                yield (_a = all.onRunStart) === null || _a === void 0 ? void 0 : _a.call(all, evals.map((e) => ({
+                    id: e.id,
+                    fileId: e.fileId,
+                    description: e.definition.description,
+                    tags: e.definition.tags,
+                })), target);
+            });
+        },
+        caseComplete(discovered, result) {
+            return __awaiter(this, void 0, void 0, function* () {
+                var _a, _b;
+                var _c;
+                const observers = [
+                    ...new Set([
+                        ...runWide,
+                        ...(skip ? [] : ((_c = discovered.definition.reporters) !== null && _c !== void 0 ? _c : [])),
+                    ]),
+                ];
+                yield (_b = (_a = combineReporters(observers, onError)).onEvalComplete) === null || _b === void 0 ? void 0 : _b.call(_a, result);
+            });
+        },
+        runComplete(summary) {
+            return __awaiter(this, void 0, void 0, function* () {
+                var _a;
+                yield (_a = all.onRunComplete) === null || _a === void 0 ? void 0 : _a.call(all, summary);
+            });
+        },
+    };
+}
 /**
  * Schedule key for {@link groupEvalsForIterationSchedule}: iteration siblings
  * (`weather/nyc/1`…`/3`) share a key; single-iteration ids keep their full id.
@@ -224,34 +336,58 @@ function resolveEvalMaxConcurrency(config) {
 }
 function runOneEval(discovered, opts) {
     return __awaiter(this, void 0, void 0, function* () {
-        var _a;
-        var _b;
+        var _a, _b;
+        var _c;
         const logs = [];
         const metrics = {};
-        const assertions = [];
+        const recorder = new EvalAssertionRecorder();
         const def = discovered.definition;
-        const timeoutMs = (_b = def.timeoutMs) !== null && _b !== void 0 ? _b : opts.defaultTimeoutMs;
+        const timeoutMs = (_c = def.timeoutMs) !== null && _c !== void 0 ? _c : opts.defaultTimeoutMs;
         const startedAt = Date.now();
         let sessionId;
         let continuationToken;
+        // The session stream replays from an index and the client stops at the first
+        // terminal event, so each send must resume after the previous turn's last
+        // event or it would re-read that turn instead of the new one.
+        let nextEventIndex = 0;
         const allEvents = [];
         const inputs = [];
+        const turns = [];
         let reply = null;
-        if ((_a = opts.signal) === null || _a === void 0 ? void 0 : _a.aborted) {
-            return Object.assign(Object.assign({ id: discovered.id, path: discovered.path }, (def.description === undefined
-                ? {}
-                : { description: def.description })), { ok: false, assertions, error: "cancelled", logs,
-                metrics,
-                inputs, toolCalls: [], tools: [], durationMs: Date.now() - startedAt });
-        }
-        const record = (name, passed, detail) => {
-            assertions.push(Object.assign({ name,
-                passed }, (detail === undefined ? {} : { detail })));
-        };
         const toolsUsed = () => buildTrajectory(allEvents).turns.flatMap((t) => t.toolCalls.map((c) => c.toolName));
         const toolCallIo = () => buildTrajectory(allEvents).turns.flatMap((turn) => turn.toolCalls.map((c) => (Object.assign(Object.assign(Object.assign({ toolName: c.toolName }, (c.args === undefined ? {} : { args: c.args })), (c.output === undefined ? {} : { output: c.output })), { isError: c.isError }))));
-        const t = {
-            get reply() {
+        const finish = (extra) => (Object.assign(Object.assign(Object.assign(Object.assign({ id: discovered.id, path: discovered.path }, (def.description === undefined ? {} : { description: def.description })), { ok: extra.ok, verdict: extra.verdict, skipReason: extra.skipReason, assertions: recorder.results(), error: extra.error, logs,
+            metrics, metadata: def.metadata, tags: def.tags, inputs, toolCalls: toolCallIo() }), (sessionId === undefined ? {} : { sessionId })), { finalText: extra.finalText, tools: toolsUsed(), durationMs: Date.now() - startedAt }));
+        if ((_a = opts.signal) === null || _a === void 0 ? void 0 : _a.aborted) {
+            return finish({ verdict: "failed", ok: false, error: "cancelled" });
+        }
+        // A judge that cannot reach a model must not look like a quality regression:
+        // the case is skipped visibly instead.
+        let judgeUnavailable;
+        const timeout = new AbortController();
+        const timer = setTimeout(() => timeout.abort(), clampTimerDelay(timeoutMs));
+        // A pending case timer must not hold the process open on its own.
+        (_b = timer.unref) === null || _b === void 0 ? void 0 : _b.call(timer);
+        const runAssertions = createAssertions({
+            recorder,
+            events: () => allEvents,
+        });
+        const judge = createJudgeSurface({
+            recorder,
+            grader: (name) => EVAL_JUDGE_GRADERS[name],
+            resolveModel: (callModel) => { var _a; return resolveJudgeModel({
+                call: callModel,
+                evalLevel: (_a = def.judge) === null || _a === void 0 ? void 0 : _a.model,
+                config: opts.judge.configModel,
+                env: process.env[EVAL_JUDGE_MODEL_ENV],
+            }); },
+            caller: opts.judge.caller,
+            defaultValue: () => reply,
+            onUnavailable: (message) => {
+                judgeUnavailable = judgeUnavailable !== null && judgeUnavailable !== void 0 ? judgeUnavailable : message;
+            },
+        });
+        const t = Object.assign(Object.assign({}, runAssertions), { get reply() {
                 return reply;
             },
             get sessionId() {
@@ -260,6 +396,9 @@ function runOneEval(discovered, opts) {
             get events() {
                 return allEvents;
             },
+            get turns() {
+                return turns;
+            },
             get iteration() {
                 var _a;
                 return (_a = def.iteration) !== null && _a !== void 0 ? _a : 1;
@@ -268,14 +407,20 @@ function runOneEval(discovered, opts) {
                 var _a;
                 return (_a = def.iterations) !== null && _a !== void 0 ? _a : 1;
             },
+            get signal() {
+                return timeout.signal;
+            },
+            judge,
             send(message_1) {
                 return __awaiter(this, arguments, void 0, function* (message, options = {}) {
+                    var _a;
                     inputs.push(message);
                     const result = yield runSession({
                         baseUrl: opts.baseUrl,
                         messages: [message],
                         sessionId,
                         continuationToken,
+                        startIndex: nextEventIndex,
                         // Keep defineEval traffic out of live defineAB enrollment/metrics.
                         purpose: "eval",
                         bearerToken: opts.bearerToken,
@@ -289,39 +434,50 @@ function runOneEval(discovered, opts) {
                     });
                     sessionId = result.sessionId;
                     continuationToken = result.continuationToken;
+                    nextEventIndex = (_a = result.nextEventIndex) !== null && _a !== void 0 ? _a : nextEventIndex;
                     allEvents.push(...result.events);
                     if (result.trajectory.finalText !== "") {
                         reply = result.trajectory.finalText;
                     }
-                    return {
-                        message: result.trajectory.finalText,
-                        sessionId: result.sessionId,
-                        events: result.events,
-                        toolCalls: result.trajectory.turns.flatMap((turn) => turn.toolCalls.map((c) => c.toolName)),
-                        ok: result.ok,
-                    };
+                    const index = turns.length + 1;
+                    // The turn is immutable, so its assertions read a frozen event slice
+                    // even after later turns append to the run.
+                    const turnEvents = [...result.events];
+                    const turn = Object.assign(Object.assign({}, createAssertions({
+                        recorder,
+                        events: () => turnEvents,
+                        scope: `turn[${index}]`,
+                    })), { message: result.trajectory.finalText, sessionId: result.sessionId, events: turnEvents, toolCalls: result.trajectory.turns.flatMap((turn) => turn.toolCalls.map((c) => c.toolName)), ok: result.ok, index,
+                        expectOk() {
+                            var _a;
+                            var _b;
+                            if (!result.ok) {
+                                throw new Error(`turn[${index}] failed: ${(_b = (_a = result.trajectory.turns.find((x) => x.failed)) === null || _a === void 0 ? void 0 : _a.error) !== null && _b !== void 0 ? _b : "unknown error"}`);
+                            }
+                            return turn;
+                        } });
+                    turns.push(turn);
+                    return turn;
                 });
             },
-            succeeded() {
-                const traj = buildTrajectory(allEvents);
-                record("succeeded", traj.ok && traj.turns.length > 0, traj.ok ? undefined : "turn failed");
-            },
-            calledTool(name) {
-                const tools = toolsUsed();
-                record(`calledTool(${name})`, tools.includes(name), tools.includes(name) ? undefined : `got [${tools.join(", ")}]`);
-            },
-            notCalledTool(name) {
-                const tools = toolsUsed();
-                record(`notCalledTool(${name})`, !tools.includes(name), tools.includes(name) ? "tool was called" : undefined);
-            },
-            messageIncludes(token) {
-                const text = reply !== null && reply !== void 0 ? reply : "";
-                const passed = typeof token === "string" ? text.includes(token) : token.test(text);
-                record(`messageIncludes(${String(token)})`, passed, passed ? undefined : `reply=${JSON.stringify(text.slice(0, 120))}`);
+            require(value, assertion) {
+                return __awaiter(this, void 0, void 0, function* () {
+                    const outcome = evaluateExpectation(value, assertion);
+                    recorder.record({
+                        name: `require(${assertion.kind})`,
+                        passed: outcome.passed,
+                        detail: outcome.detail,
+                        score: outcome.score,
+                        severity: "gate",
+                    });
+                    if (!outcome.passed) {
+                        throw new EvalRequireFailure(`require(${assertion.kind})`);
+                    }
+                    return value;
+                });
             },
-            check(value, assertion) {
-                const { passed, detail } = matchExpectation(value, assertion);
-                record(`check(${assertion.kind})`, passed, detail);
+            skip(reason) {
+                throw new EvalSkipSignal(reason);
             },
             log(message) {
                 logs.push(message);
@@ -338,90 +494,138 @@ function runOneEval(discovered, opts) {
                 if (opts.verbose) {
                     process.stderr.write(`    metric ${key}=${String(value)}\n`);
                 }
-            },
-        };
+            } });
         try {
             yield def.test(t);
         }
         catch (error) {
-            const message = error instanceof Error ? error.message : String(error);
-            if (error instanceof RunSessionTimeoutError && sessionId === undefined) {
-                sessionId = error.sessionId;
+            if (error instanceof EvalSkipSignal) {
+                clearTimeout(timer);
+                return finish({
+                    verdict: "skipped",
+                    ok: true,
+                    skipReason: error.reason,
+                    finalText: reply !== null && reply !== void 0 ? reply : undefined,
+                });
             }
-            return Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({ id: discovered.id, path: discovered.path }, (def.description === undefined
-                ? {}
-                : { description: def.description })), { ok: false, assertions, error: message, logs,
-                metrics,
-                inputs, toolCalls: toolCallIo() }), (sessionId === undefined ? {} : { sessionId })), (reply === null ? {} : { finalText: reply })), { tools: toolsUsed(), durationMs: Date.now() - startedAt });
+            if (!(error instanceof EvalRequireFailure)) {
+                const message = error instanceof Error ? error.message : String(error);
+                if (error instanceof RunSessionTimeoutError && sessionId === undefined) {
+                    sessionId = error.sessionId;
+                }
+                yield recorder.waitForPending().catch(() => undefined);
+                clearTimeout(timer);
+                return finish({
+                    verdict: "failed",
+                    ok: false,
+                    error: message,
+                    finalText: reply !== null && reply !== void 0 ? reply : undefined,
+                });
+            }
+            // `require` already recorded the failing gate; fall through so the verdict
+            // comes from the assertions rather than a duplicate execution error.
         }
+        yield recorder.waitForPending();
+        clearTimeout(timer);
+        const assertions = recorder.results();
         const traj = buildTrajectory(allEvents);
-        const ok = assertions.length > 0
-            ? assertions.every((a) => a.passed)
-            : traj.ok && traj.turns.length > 0;
-        return Object.assign(Object.assign(Object.assign(Object.assign({ id: discovered.id, path: discovered.path }, (def.description === undefined ? {} : { description: def.description })), { ok,
-            assertions,
-            logs,
-            metrics,
-            inputs, toolCalls: toolCallIo() }), (sessionId === undefined ? {} : { sessionId })), { finalText: reply !== null && reply !== void 0 ? reply : traj.finalText, tools: toolsUsed(), durationMs: Date.now() - startedAt });
+        // With no assertions at all, keep the historical fallback: a case that just
+        // drives the agent passes when the run itself succeeded.
+        const verdict = assertions.length > 0
+            ? computeVerdict(assertions)
+            : traj.ok && traj.turns.length > 0
+                ? "passed"
+                : "failed";
+        const finalText = reply !== null && reply !== void 0 ? reply : traj.finalText;
+        // An unreachable judge means the *judged* signal is unknown, so the case is
+        // reported as skipped rather than as a quality regression. It must not also
+        // bury a deterministic gate that already failed: skipping a failed case would
+        // turn a red eval green on any machine that happens to lack credentials.
+        if (judgeUnavailable !== undefined && verdict !== "failed") {
+            logs.push(`judge skipped: ${judgeUnavailable}`);
+            return finish({
+                verdict: "skipped",
+                ok: true,
+                skipReason: judgeUnavailable,
+                finalText,
+            });
+        }
+        if (judgeUnavailable !== undefined) {
+            logs.push(`judge unavailable (${judgeUnavailable}), but a gate already failed — reporting the failure`);
+        }
+        return finish({
+            verdict,
+            ok: verdict !== "failed",
+            finalText,
+        });
     });
 }
-function matchExpectation(value, assertion) {
-    switch (assertion.kind) {
-        case "includes": {
-            const text = value === null || value === undefined ? "" : String(value);
-            const passed = typeof assertion.value === "string"
-                ? text.includes(assertion.value)
-                : assertion.value.test(text);
-            return Object.assign({ passed }, (passed
-                ? {}
-                : {
-                    detail: `value=${JSON.stringify(text.slice(0, 120))}`,
-                }));
+/** Clamp a case timeout to a delay `setTimeout` can represent. */
+function clampTimerDelay(timeoutMs) {
+    const MAX_TIMER = 2147483647;
+    return Math.min(Math.max(timeoutMs, 0), MAX_TIMER);
+}
+/**
+ * Build `t.judge`. Each grader records its slot immediately and resolves it
+ * during finalization, so several judges overlap instead of serializing.
+ */
+function createJudgeSurface(args) {
+    const grade = (graderName, reference, options) => {
+        const grader = args.grader(graderName);
+        const model = args.resolveModel(options === null || options === void 0 ? void 0 : options.model);
+        const name = `judge.${graderName}(${truncateForName(reference)})`;
+        if (model === undefined) {
+            // A missing judge model is an authoring error, not a flaky provider.
+            return args.recorder.record({
+                name,
+                passed: false,
+                severity: "gate",
+                detail: "no judge model configured; set judge.model in evals.config.ts, on the eval, or on the call",
+            });
         }
-        case "equals": {
-            const passed = deepEqual(value, assertion.value);
-            // Always attach a short detail so values like recall `40.0% (4/10)`
-            // show up on PASS lines / playground, not only on failure.
-            const rendered = typeof value === "string" ? value : JSON.stringify(value);
+        const submission = renderSubmission(options !== undefined && "on" in options
+            ? options.on
+            : args.defaultValue());
+        const settle = (() => __awaiter(this, void 0, void 0, function* () {
+            const reply = yield args.caller({
+                prompt: grader.buildPrompt({ submission, reference }),
+                model,
+            });
+            const result = grader.parse(reply);
             return {
-                passed,
-                detail: passed ? rendered : `got ${rendered}`,
+                passed: true,
+                score: result.score,
+                detail: result.rationale === undefined
+                    ? `${result.choice} (${result.score.toFixed(2)})`
+                    : `${result.choice} (${result.score.toFixed(2)}) — ${result.rationale}`,
             };
-        }
-        case "satisfies": {
-            const passed = assertion.predicate(value);
-            return Object.assign({ passed }, (passed ? {} : { detail: assertion.label }));
-        }
-    }
+        }))().catch((error) => {
+            if (error instanceof EvalJudgeUnavailableError) {
+                args.onUnavailable(error.message);
+                return { passed: true, detail: `skipped: ${error.message}` };
+            }
+            throw error;
+        });
+        // Soft by default: a judge is a tracked score until given a bar.
+        return args.recorder.recordAsync({ name, passed: true, severity: "soft", detail: "grading…" }, settle);
+    };
+    return {
+        model(prompt, options) {
+            return __awaiter(this, void 0, void 0, function* () {
+                const model = args.resolveModel(options === null || options === void 0 ? void 0 : options.model);
+                if (model === undefined) {
+                    throw new Error("t.judge.model: no judge model configured; set judge.model in evals.config.ts, on the eval, or on the call");
+                }
+                return args.caller({ prompt, model });
+            });
+        },
+        factuality: (expected, options) => grade("factuality", expected, options),
+        summarizes: (expected, options) => grade("summarizes", expected, options),
+        closedQA: (criteria, options) => grade("closedQA", criteria, options),
+        sql: (expected, options) => grade("sql", expected, options),
+    };
 }
-function deepEqual(a, b) {
-    if (Object.is(a, b)) {
-        return true;
-    }
-    if (typeof a !== "object" ||
-        typeof b !== "object" ||
-        a === null ||
-        b === null) {
-        return false;
-    }
-    if (Array.isArray(a) !== Array.isArray(b)) {
-        return false;
-    }
-    if (Array.isArray(a) && Array.isArray(b)) {
-        if (a.length !== b.length) {
-            return false;
-        }
-        return a.every((item, i) => deepEqual(item, b[i]));
-    }
-    const aKeys = Object.keys(a);
-    const bKeys = Object.keys(b);
-    if (aKeys.length !== bKeys.length) {
-        return false;
-    }
-    for (const key of aKeys) {
-        if (!deepEqual(a[key], b[key])) {
-            return false;
-        }
-    }
-    return true;
+function truncateForName(text) {
+    const oneLine = text.replace(/\s+/g, " ").trim();
+    return oneLine.length > 60 ? `${oneLine.slice(0, 59)}…` : oneLine;
 }

package/dist/internal/evals-client.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"evals-client.d.ts","sourceRoot":"","sources":["../../src/internal/evals-client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAEnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAuB,KAAK,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAI7E,MAAM,MAAM,mBAAmB,GAC3B,mBAAmB,GACnB,sBAAsB,GACtB,sBAAsB,GACtB,kBAAkB,GAClB,cAAc,GACd,aAAa,GACb,eAAe,GACf,SAAS,CAAC;AAEd,qBAAa,eAAgB,SAAQ,KAAK;IAGtC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM;IACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,mBAAmB;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM;IAJ/B,YACE,OAAO,EAAE,MAAM,EACN,MAAM,CAAC,EAAE,MAAM,YAAA,EACf,IAAI,CAAC,EAAE,mBAAmB,YAAA,EAC1B,WAAW,CAAC,EAAE,MAAM,YAAA,EAI9B;CACF;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,qBAAsB,SAAQ,WAAW;IACxD,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,kDAAkD;IAClD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;CAC9C;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC;IAAE,QAAQ,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE,CAAC,CA+BlE;AAED,iFAAiF;AACjF,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAgEhC;AAED,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,eAAe,CAAC,CAuF3B;AAED,0EAA0E;AAC1E,wBAAsB,mBAAmB,CAAC,IAAI,EAAE;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,eAAe,CAAC,CAsE3B;AAED,6DAA6D;AAC7D,wBAAsB,gBAAgB,CAAC,IAAI,EAAE;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,eAAe,CAAC,CAmC3B;AAED,oDAAoD;AACpD,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,eAAe,EAAE,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAyC7D;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;CAC9C,GAAG,OAAO,CAAC,eAAe,CAAC,CAoB3B;AAED,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,eAAe,GACxB,aAAa,EAAE,CAiBjB"}
+{"version":3,"file":"evals-client.d.ts","sourceRoot":"","sources":["../../src/internal/evals-client.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAMnD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAuB,KAAK,WAAW,EAAE,MAAM,sBAAsB,CAAC;AAI7E,MAAM,MAAM,mBAAmB,GAC3B,mBAAmB,GACnB,sBAAsB,GACtB,sBAAsB,GACtB,kBAAkB,GAClB,cAAc,GACd,aAAa,GACb,eAAe,GACf,SAAS,CAAC;AAEd,qBAAa,eAAgB,SAAQ,KAAK;IAGtC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM;IACxB,QAAQ,CAAC,IAAI,CAAC,EAAE,mBAAmB;IACnC,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM;IAJ/B,YACE,OAAO,EAAE,MAAM,EACN,MAAM,CAAC,EAAE,MAAM,YAAA,EACf,IAAI,CAAC,EAAE,mBAAmB,YAAA,EAC1B,WAAW,CAAC,EAAE,MAAM,YAAA,EAI9B;CACF;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,MAAM,WAAW,qBAAsB,SAAQ,WAAW;IACxD,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,gEAAgE;IAChE,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qEAAqE;IACrE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,kDAAkD;IAClD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,oDAAoD;IACpD,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;CAC9C;AAED;;;GAGG;AACH,wBAAsB,cAAc,CAClC,OAAO,EAAE,qBAAqB,GAC7B,OAAO,CAAC;IAAE,QAAQ,EAAE,eAAe,CAAC;IAAC,OAAO,EAAE,aAAa,EAAE,CAAA;CAAE,CAAC,CA+BlE;AAED,iFAAiF;AACjF,wBAAsB,eAAe,CAAC,IAAI,EAAE;IAC1C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAmEhC;AAED,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,eAAe,CAAC,CAuF3B;AAED,0EAA0E;AAC1E,wBAAsB,mBAAmB,CAAC,IAAI,EAAE;IAC9C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,eAAe,CAAC,CAsE3B;AAED,6DAA6D;AAC7D,wBAAsB,gBAAgB,CAAC,IAAI,EAAE;IAC3C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC,eAAe,CAAC,CAmC3B;AAED,oDAAoD;AACpD,wBAAsB,kBAAkB,CAAC,IAAI,EAAE;IAC7C,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,GAAG,OAAO,CAAC;IAAE,IAAI,EAAE,eAAe,EAAE,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC,CAyC7D;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE;IAC5C,OAAO,EAAE,MAAM,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,WAAW,CAAC;IACnB,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IACzB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,eAAe,KAAK,IAAI,CAAC;CAC9C,GAAG,OAAO,CAAC,eAAe,CAAC,CAoB3B;AAED,wBAAgB,wBAAwB,CACtC,QAAQ,EAAE,eAAe,GACxB,aAAa,EAAE,CAsBjB"}

package/dist/internal/evals-client.js

@@ -15,7 +15,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-import { matchesEvalIdFilter } from "../evals.js";
+import { expandSummarySuiteSelection, isEvalSummaryCase, matchesEvalIdFilter, } from "../evals.js";
 import { mergeRequestHeaders } from "./request-headers.js";
 const DEFAULT_POLL_MS = 1000;
 export class RemoteEvalError extends Error {
@@ -106,7 +106,8 @@ export function listRemoteEvals(args) {
         if (args.tags !== undefined && args.tags.length > 0) {
             selected = selected.filter((e) => { var _a; return ((_a = e.tags) !== null && _a !== void 0 ? _a : []).some((t) => args.tags.includes(t)); });
         }
-        return selected;
+        // Match server-side filterDiscoveredEvals: summary → whole file.
+        return expandSummarySuiteSelection(listed, selected, (e) => isEvalSummaryCase(e.tags));
     });
 }
 export function startRemoteEvalRun(args) {
@@ -284,7 +285,10 @@ export function pollRemoteEvalRun(args) {
     });
 }
 export function snapshotToEvalRunResults(snapshot) {
-    return snapshot.cases.map((c) => { var _a, _b, _c, _d, _e; return (Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({ id: c.id, path: c.fileId }, (c.description === undefined ? {} : { description: c.description })), { ok: c.ok === true, assertions: (_a = c.assertions) !== null && _a !== void 0 ? _a : [] }), (c.sessionId === undefined ? {} : { sessionId: c.sessionId })), { inputs: (_b = c.inputs) !== null && _b !== void 0 ? _b : [] }), (c.finalText === undefined ? {} : { finalText: c.finalText })), (c.tools === undefined ? {} : { tools: c.tools })), (c.toolCalls === undefined ? {} : { toolCalls: c.toolCalls })), (c.error === undefined ? {} : { error: c.error })), { logs: (_c = c.logs) !== null && _c !== void 0 ? _c : [], metrics: (_d = c.metrics) !== null && _d !== void 0 ? _d : {}, durationMs: (_e = c.durationMs) !== null && _e !== void 0 ? _e : 0 })); });
+    return snapshot.cases.map((c) => { var _a, _b, _c, _d, _e, _f; return (Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({ id: c.id, path: c.fileId }, (c.description === undefined ? {} : { description: c.description })), { ok: c.ok === true, 
+        // Older serve processes send no verdict; infer it from `ok` so a remote
+        // batch still prints and reports consistently.
+        verdict: (_a = c.verdict) !== null && _a !== void 0 ? _a : (c.ok === true ? "passed" : "failed") }), (c.skipReason === undefined ? {} : { skipReason: c.skipReason })), (c.metadata === undefined ? {} : { metadata: c.metadata })), { assertions: (_b = c.assertions) !== null && _b !== void 0 ? _b : [] }), (c.sessionId === undefined ? {} : { sessionId: c.sessionId })), { inputs: (_c = c.inputs) !== null && _c !== void 0 ? _c : [] }), (c.finalText === undefined ? {} : { finalText: c.finalText })), (c.tools === undefined ? {} : { tools: c.tools })), (c.toolCalls === undefined ? {} : { toolCalls: c.toolCalls })), (c.error === undefined ? {} : { error: c.error })), { logs: (_d = c.logs) !== null && _d !== void 0 ? _d : [], metrics: (_e = c.metrics) !== null && _e !== void 0 ? _e : {}, durationMs: (_f = c.durationMs) !== null && _f !== void 0 ? _f : 0 })); });
 }
 function sleep(ms, signal) {
     return new Promise((resolve, reject) => {