@cursor/bdk 0.2.8 → 0.2.10

package/dist/internal/eval-runner.js

@@ -7,15 +7,6 @@
  * Drive with `t.send`, assert with `t.succeeded` / `t.calledTool` /
  * `t.messageIncludes` / `t.check`.
  */
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 import { readdir } from "node:fs/promises";
 import { join, relative, resolve } from "node:path";
 import { computeVerdict, createAssertions, EvalAssertionRecorder, EvalRequireFailure, EvalSkipSignal, } from "../evals/assertions.js";
@@ -28,84 +19,79 @@ import { EVAL_MAX_CONCURRENCY_LIMIT, expandEvalDefinition, expandSummarySuiteSel
 import { importAuthoredModule } from "./authored-loaders.js";
 import { isDefinitionOfKind } from "./brand.js";
 import { callJudgeModel, EVAL_JUDGE_MODEL_ENV, resolveJudgeModel } from "./eval-judge-model.js";
+import { localHttpEvalTransport } from "./eval-transport.js";
 import { mapPool } from "./map-pool.js";
-import { RunSessionTimeoutError, runSession } from "./run-client.js";
+import { RunSessionTimeoutError } from "./run-client.js";
 import { buildTrajectory } from "./trajectory.js";
-function listEvalFiles(dir) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const out = [];
-        function walk(current) {
-            return __awaiter(this, void 0, void 0, function* () {
-                let entries;
-                try {
-                    entries = yield readdir(current, { withFileTypes: true });
-                }
-                catch (_a) {
-                    return;
-                }
-                for (const entry of entries) {
-                    if (entry.name.startsWith(".")) {
-                        continue;
-                    }
-                    const abs = join(current, entry.name);
-                    if (entry.isDirectory()) {
-                        if (entry.name === "node_modules") {
-                            continue;
-                        }
-                        yield walk(abs);
-                    }
-                    else if (entry.isFile()) {
-                        out.push(abs);
-                    }
-                }
-            });
+async function listEvalFiles(dir) {
+    const out = [];
+    async function walk(current) {
+        let entries;
+        try {
+            entries = await readdir(current, { withFileTypes: true });
         }
-        yield walk(dir);
-        return out.sort();
-    });
-}
-export function discoverEvals(projectRoot_1) {
-    return __awaiter(this, arguments, void 0, function* (projectRoot, importModule = importAuthoredModule) {
-        const root = resolve(projectRoot);
-        const evalsDir = join(root, "evals");
-        const files = yield listEvalFiles(evalsDir);
-        const evals = [];
-        let config = {};
-        // Fixture loaders resolve relative paths against the app root, so a dataset
-        // path reads the same however deep the eval file sits and whatever cwd the
-        // CLI ran from.
-        process.env[EVAL_ROOT_ENV] = root;
-        for (const abs of files) {
-            const rel = relative(evalsDir, abs).replace(/\\/g, "/");
-            if (rel === "evals.config.ts" || rel === "evals.config.js") {
-                const mod = yield importModule(abs);
-                if (mod.default !== null && typeof mod.default === "object") {
-                    config = mod.default;
-                }
-                continue;
-            }
-            if (!rel.endsWith(".eval.ts") && !rel.endsWith(".eval.js")) {
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            if (entry.name.startsWith(".")) {
                 continue;
             }
-            const mod = yield importModule(abs);
-            const fileId = rel.replace(/\.eval\.(ts|js)$/, "").replace(/\/index$/, "");
-            try {
-                for (const expanded of expandEvalModule(fileId, mod.default)) {
-                    evals.push({
-                        id: expanded.id,
-                        fileId,
-                        path: abs,
-                        definition: expanded.definition,
-                    });
+            const abs = join(current, entry.name);
+            if (entry.isDirectory()) {
+                if (entry.name === "node_modules") {
+                    continue;
                 }
+                await walk(abs);
             }
-            catch (error) {
-                const message = error instanceof Error ? error.message : String(error);
-                throw new Error(`${rel}: ${message}`);
+            else if (entry.isFile()) {
+                out.push(abs);
             }
         }
-        return { evals, config };
-    });
+    }
+    await walk(dir);
+    return out.sort();
+}
+export async function discoverEvals(projectRoot, importModule = importAuthoredModule) {
+    const root = resolve(projectRoot);
+    const evalsDir = join(root, "evals");
+    const files = await listEvalFiles(evalsDir);
+    const evals = [];
+    let config = {};
+    // Fixture loaders resolve relative paths against the app root, so a dataset
+    // path reads the same however deep the eval file sits and whatever cwd the
+    // CLI ran from.
+    process.env[EVAL_ROOT_ENV] = root;
+    for (const abs of files) {
+        const rel = relative(evalsDir, abs).replace(/\\/g, "/");
+        if (rel === "evals.config.ts" || rel === "evals.config.js") {
+            const mod = await importModule(abs);
+            if (mod.default !== null && typeof mod.default === "object") {
+                config = mod.default;
+            }
+            continue;
+        }
+        if (!rel.endsWith(".eval.ts") && !rel.endsWith(".eval.js")) {
+            continue;
+        }
+        const mod = await importModule(abs);
+        const fileId = rel.replace(/\.eval\.(ts|js)$/, "").replace(/\/index$/, "");
+        try {
+            for (const expanded of expandEvalModule(fileId, mod.default)) {
+                evals.push({
+                    id: expanded.id,
+                    fileId,
+                    path: abs,
+                    definition: expanded.definition,
+                });
+            }
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            throw new Error(`${rel}: ${message}`);
+        }
+    }
+    return { evals, config };
 }
 /**
  * Expand one eval module's default export into runnable datapoints.
@@ -140,86 +126,83 @@ export function filterDiscoveredEvals(evals, options = {}) {
         selected = selected.filter(e => options.filterIds.some(f => matchesEvalIdFilter(e.id, f)));
     }
     if (options.tags !== undefined && options.tags.length > 0) {
-        selected = selected.filter(e => { var _a; return ((_a = e.definition.tags) !== null && _a !== void 0 ? _a : []).some(t => options.tags.includes(t)); });
+        selected = selected.filter(e => (e.definition.tags ?? []).some(t => options.tags.includes(t)));
     }
     // Suite aggregators (`summary` tag) need sibling cases from the same file.
     return expandSummarySuiteSelection(evals, selected, e => isEvalSummaryCase(e.definition.tags));
 }
-export function runDiscoveredEvals(options) {
-    return __awaiter(this, void 0, void 0, function* () {
-        var _a, _b, _c, _d;
-        let selected;
-        let config;
-        if (options.discovered !== undefined) {
-            selected = options.discovered.evals;
-            config = options.discovered.config;
-        }
-        else {
-            const importModule = (_a = options.importModule) !== null && _a !== void 0 ? _a : importAuthoredModule;
-            const discovered = yield discoverEvals(options.projectRoot, importModule);
-            config = discovered.config;
-            selected = filterDiscoveredEvals(discovered.evals, {
-                filterIds: options.filterIds,
-                tags: options.tags,
+export async function runDiscoveredEvals(options) {
+    let selected;
+    let config;
+    if (options.discovered !== undefined) {
+        selected = options.discovered.evals;
+        config = options.discovered.config;
+    }
+    else {
+        const importModule = options.importModule ?? importAuthoredModule;
+        const discovered = await discoverEvals(options.projectRoot, importModule);
+        config = discovered.config;
+        selected = filterDiscoveredEvals(discovered.evals, {
+            filterIds: options.filterIds,
+            tags: options.tags,
+        });
+    }
+    if (selected.length === 0) {
+        return [];
+    }
+    const maxConcurrency = options.maxConcurrency === undefined
+        ? resolveEvalMaxConcurrency(config)
+        : normalizeEvalMaxConcurrency(options.maxConcurrency);
+    const defaultTimeoutMs = options.timeoutMs ?? config.timeoutMs ?? DEFAULT_EVAL_TIMEOUT_MS;
+    const transport = options.transport ??
+        localHttpEvalTransport({
+            baseUrl: options.baseUrl,
+            auth: { bearerToken: options.bearerToken, headers: options.headers },
+        });
+    const reporter = buildReporterHub(options, config, selected);
+    const startedAt = new Date().toISOString();
+    await reporter?.runStart(selected, options.target ?? { baseUrl: options.baseUrl, mode: "local" });
+    // Iterations of the same authored datapoint share one concurrency slot and
+    // run sequentially. maxConcurrency is therefore the max number of distinct
+    // datapoints in flight, not datapoints × iterations.
+    const groups = groupEvalsForIterationSchedule(selected);
+    const nested = await mapPool(groups, maxConcurrency, async (group) => {
+        const results = [];
+        for (const discovered of group) {
+            if (options.signal?.aborted) {
+                break;
+            }
+            if (options.verbose) {
+                process.stderr.write(`  eval ${discovered.id}…\n`);
+            }
+            options.onCaseStart?.(discovered.id);
+            const result = await runOneEval(discovered, {
+                transport,
+                defaultTimeoutMs,
+                verbose: Boolean(options.verbose),
+                signal: options.signal,
+                judge: {
+                    configModel: config.judge?.model,
+                    caller: options.judgeCaller ?? callJudgeModel,
+                },
+                onEvent: event => {
+                    options.onCaseEvent?.(discovered.id, event);
+                    options.onEvent?.(event);
+                },
             });
+            options.onCaseDone?.(result);
+            await reporter?.caseComplete(discovered, result);
+            results.push(result);
         }
-        if (selected.length === 0) {
-            return [];
-        }
-        const maxConcurrency = options.maxConcurrency === undefined
-            ? resolveEvalMaxConcurrency(config)
-            : normalizeEvalMaxConcurrency(options.maxConcurrency);
-        const defaultTimeoutMs = (_c = (_b = options.timeoutMs) !== null && _b !== void 0 ? _b : config.timeoutMs) !== null && _c !== void 0 ? _c : 180000;
-        const reporter = buildReporterHub(options, config, selected);
-        const startedAt = new Date().toISOString();
-        yield reporter === null || reporter === void 0 ? void 0 : reporter.runStart(selected, (_d = options.target) !== null && _d !== void 0 ? _d : { baseUrl: options.baseUrl, mode: "local" });
-        // Iterations of the same authored datapoint share one concurrency slot and
-        // run sequentially. maxConcurrency is therefore the max number of distinct
-        // datapoints in flight, not datapoints × iterations.
-        const groups = groupEvalsForIterationSchedule(selected);
-        const nested = yield mapPool(groups, maxConcurrency, (group) => __awaiter(this, void 0, void 0, function* () {
-            var _a, _b, _c, _d;
-            var _e;
-            const results = [];
-            for (const discovered of group) {
-                if ((_a = options.signal) === null || _a === void 0 ? void 0 : _a.aborted) {
-                    break;
-                }
-                if (options.verbose) {
-                    process.stderr.write(`  eval ${discovered.id}…\n`);
-                }
-                (_b = options.onCaseStart) === null || _b === void 0 ? void 0 : _b.call(options, discovered.id);
-                const result = yield runOneEval(discovered, {
-                    baseUrl: options.baseUrl,
-                    defaultTimeoutMs,
-                    bearerToken: options.bearerToken,
-                    headers: options.headers,
-                    verbose: Boolean(options.verbose),
-                    signal: options.signal,
-                    judge: {
-                        configModel: (_c = config.judge) === null || _c === void 0 ? void 0 : _c.model,
-                        caller: (_e = options.judgeCaller) !== null && _e !== void 0 ? _e : callJudgeModel,
-                    },
-                    onEvent: event => {
-                        var _a, _b;
-                        (_a = options.onCaseEvent) === null || _a === void 0 ? void 0 : _a.call(options, discovered.id, event);
-                        (_b = options.onEvent) === null || _b === void 0 ? void 0 : _b.call(options, event);
-                    },
-                });
-                (_d = options.onCaseDone) === null || _d === void 0 ? void 0 : _d.call(options, result);
-                yield reporter === null || reporter === void 0 ? void 0 : reporter.caseComplete(discovered, result);
-                results.push(result);
-            }
-            return results;
-        }), { signal: options.signal });
-        const results = nested.flat();
-        yield reporter === null || reporter === void 0 ? void 0 : reporter.runComplete(summarizeEvalResults(results, {
-            strict: options.strict === true,
-            startedAt,
-            finishedAt: new Date().toISOString(),
-        }));
         return results;
-    });
+    }, { signal: options.signal });
+    const results = nested.flat();
+    await reporter?.runComplete(summarizeEvalResults(results, {
+        strict: options.strict === true,
+        startedAt,
+        finishedAt: new Date().toISOString(),
+    }));
+    return results;
 }
 /**
  * Fan results out to the reporters that observe them.
@@ -230,10 +213,9 @@ export function runDiscoveredEvals(options) {
  * `onRunStart` / `onRunComplete` pair.
  */
 function buildReporterHub(options, config, selected) {
-    var _a, _b;
     const skip = options.skipReport === true;
-    const runWide = [...((_a = options.reporters) !== null && _a !== void 0 ? _a : []), ...(skip ? [] : ((_b = config.reporters) !== null && _b !== void 0 ? _b : []))];
-    const perEval = skip ? [] : selected.flatMap(e => { var _a; return (_a = e.definition.reporters) !== null && _a !== void 0 ? _a : []; });
+    const runWide = [...(options.reporters ?? []), ...(skip ? [] : (config.reporters ?? []))];
+    const perEval = skip ? [] : selected.flatMap(e => e.definition.reporters ?? []);
     const everyReporter = [...new Set([...runWide, ...perEval])];
     if (everyReporter.length === 0) {
         return undefined;
@@ -245,32 +227,22 @@ function buildReporterHub(options, config, selected) {
     };
     const all = combineReporters(everyReporter, onError);
     return {
-        runStart(evals, target) {
-            return __awaiter(this, void 0, void 0, function* () {
-                var _a;
-                yield (_a = all.onRunStart) === null || _a === void 0 ? void 0 : _a.call(all, evals.map(e => ({
-                    id: e.id,
-                    fileId: e.fileId,
-                    description: e.definition.description,
-                    tags: e.definition.tags,
-                })), target);
-            });
+        async runStart(evals, target) {
+            await all.onRunStart?.(evals.map(e => ({
+                id: e.id,
+                fileId: e.fileId,
+                description: e.definition.description,
+                tags: e.definition.tags,
+            })), target);
         },
-        caseComplete(discovered, result) {
-            return __awaiter(this, void 0, void 0, function* () {
-                var _a, _b;
-                var _c;
-                const observers = [
-                    ...new Set([...runWide, ...(skip ? [] : ((_c = discovered.definition.reporters) !== null && _c !== void 0 ? _c : []))]),
-                ];
-                yield (_b = (_a = combineReporters(observers, onError)).onEvalComplete) === null || _b === void 0 ? void 0 : _b.call(_a, result);
-            });
+        async caseComplete(discovered, result) {
+            const observers = [
+                ...new Set([...runWide, ...(skip ? [] : (discovered.definition.reporters ?? []))]),
+            ];
+            await combineReporters(observers, onError).onEvalComplete?.(result);
         },
-        runComplete(summary) {
-            return __awaiter(this, void 0, void 0, function* () {
-                var _a;
-                yield (_a = all.onRunComplete) === null || _a === void 0 ? void 0 : _a.call(all, summary);
-            });
+        async runComplete(summary) {
+            await all.onRunComplete?.(summary);
         },
     };
 }
@@ -279,9 +251,8 @@ function buildReporterHub(options, config, selected) {
  * (`weather/nyc/1`…`/3`) share a key; single-iteration ids keep their full id.
  */
 export function iterationScheduleKey(discovered) {
-    var _a, _b;
-    const iterations = (_a = discovered.definition.iterations) !== null && _a !== void 0 ? _a : 1;
-    const iteration = (_b = discovered.definition.iteration) !== null && _b !== void 0 ? _b : 1;
+    const iterations = discovered.definition.iterations ?? 1;
+    const iteration = discovered.definition.iteration ?? 1;
     if (iterations <= 1) {
         return discovered.id;
     }
@@ -310,7 +281,7 @@ export function groupEvalsForIterationSchedule(evals) {
     }
     return order.map(key => {
         const group = groups.get(key);
-        return group.sort((a, b) => { var _a, _b; return ((_a = a.definition.iteration) !== null && _a !== void 0 ? _a : 1) - ((_b = b.definition.iteration) !== null && _b !== void 0 ? _b : 1); });
+        return group.sort((a, b) => (a.definition.iteration ?? 1) - (b.definition.iteration ?? 1));
     });
 }
 function resolveEvalMaxConcurrency(config) {
@@ -321,237 +292,266 @@ function resolveEvalMaxConcurrency(config) {
     }
     return normalizeEvalMaxConcurrency(config.maxConcurrency);
 }
-function runOneEval(discovered, opts) {
-    return __awaiter(this, void 0, void 0, function* () {
-        var _a, _b;
-        var _c;
-        const logs = [];
-        const metrics = {};
-        const recorder = new EvalAssertionRecorder();
-        const def = discovered.definition;
-        const timeoutMs = (_c = def.timeoutMs) !== null && _c !== void 0 ? _c : opts.defaultTimeoutMs;
-        const startedAt = Date.now();
-        let sessionId;
-        let continuationToken;
-        // The session stream replays from an index and the client stops at the first
-        // terminal event, so each send must resume after the previous turn's last
-        // event or it would re-read that turn instead of the new one.
-        let nextEventIndex = 0;
-        const allEvents = [];
-        const inputs = [];
-        const turns = [];
-        let reply = null;
-        const toolsUsed = () => buildTrajectory(allEvents).turns.flatMap(t => t.toolCalls.map(c => c.toolName));
-        const toolCallIo = () => buildTrajectory(allEvents).turns.flatMap(turn => turn.toolCalls.map(c => (Object.assign(Object.assign(Object.assign({ toolName: c.toolName }, (c.args === undefined ? {} : { args: c.args })), (c.output === undefined ? {} : { output: c.output })), { isError: c.isError }))));
-        const finish = (extra) => (Object.assign(Object.assign(Object.assign(Object.assign({ id: discovered.id, path: discovered.path }, (def.description === undefined ? {} : { description: def.description })), { ok: extra.ok, verdict: extra.verdict, skipReason: extra.skipReason, assertions: recorder.results(), error: extra.error, logs,
-            metrics, metadata: def.metadata, tags: def.tags, inputs, toolCalls: toolCallIo() }), (sessionId === undefined ? {} : { sessionId })), { finalText: extra.finalText, tools: toolsUsed(), durationMs: Date.now() - startedAt }));
-        if ((_a = opts.signal) === null || _a === void 0 ? void 0 : _a.aborted) {
-            return finish({ verdict: "failed", ok: false, error: "cancelled" });
-        }
-        // A judge that cannot reach a model must not look like a quality regression:
-        // the case is skipped visibly instead.
-        let judgeUnavailable;
-        const timeout = new AbortController();
-        const timer = setTimeout(() => timeout.abort(), clampTimerDelay(timeoutMs));
-        // A pending case timer must not hold the process open on its own.
-        (_b = timer.unref) === null || _b === void 0 ? void 0 : _b.call(timer);
-        const runAssertions = createAssertions({
-            recorder,
-            events: () => allEvents,
-        });
-        const judge = createJudgeSurface({
-            recorder,
-            grader: name => EVAL_JUDGE_GRADERS[name],
-            resolveModel: callModel => { var _a; return resolveJudgeModel({
-                call: callModel,
-                evalLevel: (_a = def.judge) === null || _a === void 0 ? void 0 : _a.model,
-                config: opts.judge.configModel,
-                env: process.env[EVAL_JUDGE_MODEL_ENV],
-            }); },
-            caller: opts.judge.caller,
-            defaultValue: () => reply,
-            onUnavailable: message => {
-                judgeUnavailable = judgeUnavailable !== null && judgeUnavailable !== void 0 ? judgeUnavailable : message;
-            },
-        });
-        // Accessors lead the literal: the es2015 build lowers an object spread to
-        // `Object.assign`, which reads any accessor after it once and freezes it.
-        const t = Object.assign({ get reply() {
-                return reply;
-            },
-            get sessionId() {
-                return sessionId;
-            },
-            get events() {
-                return allEvents;
-            },
-            get turns() {
-                return turns;
-            },
-            get iteration() {
-                var _a;
-                return (_a = def.iteration) !== null && _a !== void 0 ? _a : 1;
-            },
-            get iterations() {
-                var _a;
-                return (_a = def.iterations) !== null && _a !== void 0 ? _a : 1;
-            },
-            get signal() {
-                return timeout.signal;
-            },
-            judge,
-            send(message_1) {
-                return __awaiter(this, arguments, void 0, function* (message, options = {}) {
-                    var _a;
-                    inputs.push(message);
-                    const result = yield runSession({
-                        baseUrl: opts.baseUrl,
-                        messages: [message],
-                        sessionId,
-                        continuationToken,
-                        startIndex: nextEventIndex,
-                        purpose: "eval",
-                        bearerToken: opts.bearerToken,
-                        headers: opts.headers,
-                        workspaceDir: options.workspaceDir,
-                        workspaceFiles: options.workspaceFiles,
-                        cloud: options.cloud,
-                        timeoutMs,
-                        signal: opts.signal,
-                        onEvent: opts.onEvent,
-                    });
-                    sessionId = result.sessionId;
-                    continuationToken = result.continuationToken;
-                    nextEventIndex = (_a = result.nextEventIndex) !== null && _a !== void 0 ? _a : nextEventIndex;
-                    allEvents.push(...result.events);
-                    if (result.trajectory.finalText !== "") {
-                        reply = result.trajectory.finalText;
+const DEFAULT_EVAL_TIMEOUT_MS = 180_000;
+/**
+ * One discovered case on a caller-supplied transport, with the defaults
+ * {@link runDiscoveredEvals} applies and no reporters.
+ */
+export function runEvalCase(discovered, config, transport, signal) {
+    return runOneEval(discovered, {
+        transport,
+        defaultTimeoutMs: config.timeoutMs ?? DEFAULT_EVAL_TIMEOUT_MS,
+        verbose: false,
+        judge: { configModel: config.judge?.model, caller: callJudgeModel },
+        signal,
+    });
+}
+async function runOneEval(discovered, opts) {
+    const logs = [];
+    const metrics = {};
+    const recorder = new EvalAssertionRecorder();
+    const def = discovered.definition;
+    const timeoutMs = def.timeoutMs ?? opts.defaultTimeoutMs;
+    const startedAt = Date.now();
+    let sessionId;
+    let continuationToken;
+    // The session stream replays from an index and the client stops at the first
+    // terminal event, so each send must resume after the previous turn's last
+    // event or it would re-read that turn instead of the new one.
+    let nextEventIndex = 0;
+    const allEvents = [];
+    const inputs = [];
+    const turns = [];
+    let reply = null;
+    const toolsUsed = () => buildTrajectory(allEvents).turns.flatMap(t => t.toolCalls.map(c => c.toolName));
+    const toolCallIo = () => buildTrajectory(allEvents).turns.flatMap(turn => turn.toolCalls.map(c => ({
+        toolName: c.toolName,
+        ...(c.args === undefined ? {} : { args: c.args }),
+        ...(c.output === undefined ? {} : { output: c.output }),
+        isError: c.isError,
+    })));
+    const finish = (extra) => ({
+        id: discovered.id,
+        path: discovered.path,
+        ...(def.description === undefined ? {} : { description: def.description }),
+        ok: extra.ok,
+        verdict: extra.verdict,
+        skipReason: extra.skipReason,
+        assertions: recorder.results(),
+        error: extra.error,
+        logs,
+        metrics,
+        metadata: def.metadata,
+        tags: def.tags,
+        inputs,
+        toolCalls: toolCallIo(),
+        ...(sessionId === undefined ? {} : { sessionId }),
+        finalText: extra.finalText,
+        tools: toolsUsed(),
+        durationMs: Date.now() - startedAt,
+    });
+    if (opts.signal?.aborted) {
+        return finish({ verdict: "failed", ok: false, error: "cancelled" });
+    }
+    // A judge that cannot reach a model must not look like a quality regression:
+    // the case is skipped visibly instead.
+    let judgeUnavailable;
+    const timeout = new AbortController();
+    const timer = setTimeout(() => timeout.abort(), clampTimerDelay(timeoutMs));
+    // A pending case timer must not hold the process open on its own.
+    timer.unref?.();
+    const runAssertions = createAssertions({
+        recorder,
+        events: () => allEvents,
+    });
+    const judge = createJudgeSurface({
+        recorder,
+        grader: name => EVAL_JUDGE_GRADERS[name],
+        resolveModel: callModel => resolveJudgeModel({
+            call: callModel,
+            evalLevel: def.judge?.model,
+            config: opts.judge.configModel,
+            env: process.env[EVAL_JUDGE_MODEL_ENV],
+        }),
+        caller: opts.judge.caller,
+        defaultValue: () => reply,
+        onUnavailable: message => {
+            judgeUnavailable = judgeUnavailable ?? message;
+        },
+    });
+    // Accessors lead the literal: the es2015 build lowers an object spread to
+    // `Object.assign`, which reads any accessor after it once and freezes it.
+    const t = {
+        get reply() {
+            return reply;
+        },
+        get sessionId() {
+            return sessionId;
+        },
+        get events() {
+            return allEvents;
+        },
+        get turns() {
+            return turns;
+        },
+        get iteration() {
+            return def.iteration ?? 1;
+        },
+        get iterations() {
+            return def.iterations ?? 1;
+        },
+        get signal() {
+            return timeout.signal;
+        },
+        judge,
+        async send(message, options = {}) {
+            inputs.push(message);
+            const result = await opts.transport.send({
+                message,
+                sessionId,
+                continuationToken,
+                startIndex: nextEventIndex,
+                options,
+                timeoutMs,
+                signal: opts.signal,
+                onEvent: opts.onEvent,
+            });
+            sessionId = result.sessionId;
+            continuationToken = result.continuationToken;
+            nextEventIndex = result.nextEventIndex ?? nextEventIndex;
+            allEvents.push(...result.events);
+            if (result.trajectory.finalText !== "") {
+                reply = result.trajectory.finalText;
+            }
+            const index = turns.length + 1;
+            // The turn is immutable, so its assertions read a frozen event slice
+            // even after later turns append to the run.
+            const turnEvents = [...result.events];
+            const turn = {
+                ...createAssertions({
+                    recorder,
+                    events: () => turnEvents,
+                    scope: `turn[${index}]`,
+                }),
+                message: result.trajectory.finalText,
+                sessionId: result.sessionId,
+                events: turnEvents,
+                toolCalls: result.trajectory.turns.flatMap(turn => turn.toolCalls.map(c => c.toolName)),
+                ok: result.ok,
+                index,
+                expectOk() {
+                    if (!result.ok) {
+                        throw new Error(`turn[${index}] failed: ${result.trajectory.turns.find(x => x.failed)?.error ?? "unknown error"}`);
                     }
-                    const index = turns.length + 1;
-                    // The turn is immutable, so its assertions read a frozen event slice
-                    // even after later turns append to the run.
-                    const turnEvents = [...result.events];
-                    const turn = Object.assign(Object.assign({}, createAssertions({
-                        recorder,
-                        events: () => turnEvents,
-                        scope: `turn[${index}]`,
-                    })), { message: result.trajectory.finalText, sessionId: result.sessionId, events: turnEvents, toolCalls: result.trajectory.turns.flatMap(turn => turn.toolCalls.map(c => c.toolName)), ok: result.ok, index,
-                        expectOk() {
-                            var _a;
-                            var _b;
-                            if (!result.ok) {
-                                throw new Error(`turn[${index}] failed: ${(_b = (_a = result.trajectory.turns.find(x => x.failed)) === null || _a === void 0 ? void 0 : _a.error) !== null && _b !== void 0 ? _b : "unknown error"}`);
-                            }
-                            return turn;
-                        } });
-                    turns.push(turn);
                     return turn;
-                });
-            },
-            require(value, assertion) {
-                return __awaiter(this, void 0, void 0, function* () {
-                    const outcome = evaluateExpectation(value, assertion);
-                    recorder.record({
-                        name: `require(${assertion.kind})`,
-                        passed: outcome.passed,
-                        detail: outcome.detail,
-                        score: outcome.score,
-                        severity: "gate",
-                    });
-                    if (!outcome.passed) {
-                        throw new EvalRequireFailure(`require(${assertion.kind})`);
-                    }
-                    return value;
-                });
-            },
-            skip(reason) {
-                throw new EvalSkipSignal(reason);
-            },
-            log(message) {
-                logs.push(message);
-                if (opts.verbose) {
-                    process.stderr.write(`    ${message}\n`);
-                }
-            },
-            metric(name, value) {
-                const key = name.trim();
-                if (key === "") {
-                    return;
-                }
-                metrics[key] = value;
-                if (opts.verbose) {
-                    process.stderr.write(`    metric ${key}=${String(value)}\n`);
-                }
-            } }, runAssertions);
-        try {
-            yield def.test(t);
-        }
-        catch (error) {
-            if (error instanceof EvalSkipSignal) {
-                clearTimeout(timer);
-                return finish({
-                    verdict: "skipped",
-                    ok: true,
-                    skipReason: error.reason,
-                    finalText: reply !== null && reply !== void 0 ? reply : undefined,
-                });
+                },
+            };
+            turns.push(turn);
+            return turn;
+        },
+        async require(value, assertion) {
+            const outcome = evaluateExpectation(value, assertion);
+            recorder.record({
+                name: `require(${assertion.kind})`,
+                passed: outcome.passed,
+                detail: outcome.detail,
+                score: outcome.score,
+                severity: "gate",
+            });
+            if (!outcome.passed) {
+                throw new EvalRequireFailure(`require(${assertion.kind})`);
             }
-            if (!(error instanceof EvalRequireFailure)) {
-                const message = error instanceof Error ? error.message : String(error);
-                if (error instanceof RunSessionTimeoutError) {
-                    sessionId !== null && sessionId !== void 0 ? sessionId : (sessionId = error.sessionId);
-                    allEvents.push(...error.events);
-                }
-                yield recorder.waitForPending().catch(() => undefined);
-                clearTimeout(timer);
-                return finish({
-                    verdict: "failed",
-                    ok: false,
-                    error: message,
-                    finalText: reply !== null && reply !== void 0 ? reply : undefined,
-                });
+            return value;
+        },
+        skip(reason) {
+            throw new EvalSkipSignal(reason);
+        },
+        log(message) {
+            logs.push(message);
+            if (opts.verbose) {
+                process.stderr.write(`    ${message}\n`);
             }
-            // `require` already recorded the failing gate; fall through so the verdict
-            // comes from the assertions rather than a duplicate execution error.
-        }
-        yield recorder.waitForPending();
-        clearTimeout(timer);
-        const assertions = recorder.results();
-        const traj = buildTrajectory(allEvents);
-        // With no assertions at all, keep the historical fallback: a case that just
-        // drives the agent passes when the run itself succeeded.
-        const verdict = assertions.length > 0
-            ? computeVerdict(assertions)
-            : traj.ok && traj.turns.length > 0
-                ? "passed"
-                : "failed";
-        const finalText = reply !== null && reply !== void 0 ? reply : traj.finalText;
-        // An unreachable judge means the *judged* signal is unknown, so the case is
-        // reported as skipped rather than as a quality regression. It must not also
-        // bury a deterministic gate that already failed: skipping a failed case would
-        // turn a red eval green on any machine that happens to lack credentials.
-        if (judgeUnavailable !== undefined && verdict !== "failed") {
-            logs.push(`judge skipped: ${judgeUnavailable}`);
+        },
+        metric(name, value) {
+            const key = name.trim();
+            if (key === "") {
+                return;
+            }
+            metrics[key] = value;
+            if (opts.verbose) {
+                process.stderr.write(`    metric ${key}=${String(value)}\n`);
+            }
+        },
+        ...runAssertions,
+    };
+    try {
+        await def.test(t);
+    }
+    catch (error) {
+        if (error instanceof EvalSkipSignal) {
+            clearTimeout(timer);
             return finish({
                 verdict: "skipped",
                 ok: true,
-                skipReason: judgeUnavailable,
-                finalText,
+                skipReason: error.reason,
+                finalText: reply ?? undefined,
             });
         }
-        if (judgeUnavailable !== undefined) {
-            logs.push(`judge unavailable (${judgeUnavailable}), but a gate already failed — reporting the failure`);
+        if (!(error instanceof EvalRequireFailure)) {
+            const message = error instanceof Error ? error.message : String(error);
+            if (error instanceof RunSessionTimeoutError) {
+                sessionId ??= error.sessionId;
+                allEvents.push(...error.events);
+            }
+            await recorder.waitForPending().catch(() => undefined);
+            clearTimeout(timer);
+            return finish({
+                verdict: "failed",
+                ok: false,
+                error: message,
+                finalText: reply ?? undefined,
+            });
         }
+        // `require` already recorded the failing gate; fall through so the verdict
+        // comes from the assertions rather than a duplicate execution error.
+    }
+    await recorder.waitForPending();
+    clearTimeout(timer);
+    const assertions = recorder.results();
+    const traj = buildTrajectory(allEvents);
+    // With no assertions at all, keep the historical fallback: a case that just
+    // drives the agent passes when the run itself succeeded.
+    const verdict = assertions.length > 0
+        ? computeVerdict(assertions)
+        : traj.ok && traj.turns.length > 0
+            ? "passed"
+            : "failed";
+    const finalText = reply ?? traj.finalText;
+    // An unreachable judge means the *judged* signal is unknown, so the case is
+    // reported as skipped rather than as a quality regression. It must not also
+    // bury a deterministic gate that already failed: skipping a failed case would
+    // turn a red eval green on any machine that happens to lack credentials.
+    if (judgeUnavailable !== undefined && verdict !== "failed") {
+        logs.push(`judge skipped: ${judgeUnavailable}`);
         return finish({
-            verdict,
-            ok: verdict !== "failed",
+            verdict: "skipped",
+            ok: true,
+            skipReason: judgeUnavailable,
             finalText,
         });
+    }
+    if (judgeUnavailable !== undefined) {
+        logs.push(`judge unavailable (${judgeUnavailable}), but a gate already failed — reporting the failure`);
+    }
+    return finish({
+        verdict,
+        ok: verdict !== "failed",
+        finalText,
     });
 }
 /** Clamp a case timeout to a delay `setTimeout` can represent. */
 function clampTimerDelay(timeoutMs) {
-    const MAX_TIMER = 2147483647;
+    const MAX_TIMER = 2_147_483_647;
     return Math.min(Math.max(timeoutMs, 0), MAX_TIMER);
 }
 /**
@@ -561,7 +561,7 @@ function clampTimerDelay(timeoutMs) {
 function createJudgeSurface(args) {
     const grade = (graderName, reference, options) => {
         const grader = args.grader(graderName);
-        const model = args.resolveModel(options === null || options === void 0 ? void 0 : options.model);
+        const model = args.resolveModel(options?.model);
         const name = `judge.${graderName}(${truncateForName(reference)})`;
         if (model === undefined) {
             // A missing judge model is an authoring error, not a flaky provider.
@@ -573,8 +573,8 @@ function createJudgeSurface(args) {
             });
         }
         const submission = renderSubmission(options !== undefined && "on" in options ? options.on : args.defaultValue());
-        const settle = (() => __awaiter(this, void 0, void 0, function* () {
-            const reply = yield args.caller({
+        const settle = (async () => {
+            const reply = await args.caller({
                 prompt: grader.buildPrompt({ submission, reference }),
                 model,
             });
@@ -586,7 +586,7 @@ function createJudgeSurface(args) {
                     ? `${result.choice} (${result.score.toFixed(2)})`
                     : `${result.choice} (${result.score.toFixed(2)}) — ${result.rationale}`,
             };
-        }))().catch((error) => {
+        })().catch((error) => {
             if (error instanceof EvalJudgeUnavailableError) {
                 args.onUnavailable(error.message);
                 return { passed: true, detail: `skipped: ${error.message}` };
@@ -597,14 +597,12 @@ function createJudgeSurface(args) {
         return args.recorder.recordAsync({ name, passed: true, severity: "soft", detail: "grading…" }, settle);
     };
     return {
-        model(prompt, options) {
-            return __awaiter(this, void 0, void 0, function* () {
-                const model = args.resolveModel(options === null || options === void 0 ? void 0 : options.model);
-                if (model === undefined) {
-                    throw new Error("t.judge.model: no judge model configured; set judge.model in evals.config.ts, on the eval, or on the call");
-                }
-                return args.caller({ prompt, model });
-            });
+        async model(prompt, options) {
+            const model = args.resolveModel(options?.model);
+            if (model === undefined) {
+                throw new Error("t.judge.model: no judge model configured; set judge.model in evals.config.ts, on the eval, or on the call");
+            }
+            return args.caller({ prompt, model });
         },
         factuality: (expected, options) => grade("factuality", expected, options),
         summarizes: (expected, options) => grade("summarizes", expected, options),