chivgent 0.19.0 → 1.0.1

package/dist/evals/baseline.js

@@ -0,0 +1,121 @@
+import { execFile } from "node:child_process";
+import { createHash } from "node:crypto";
+import { readdir, readFile, stat } from "node:fs/promises";
+import path from "node:path";
+import { promisify } from "node:util";
+import { createGrader } from "./graders.js";
+import { TaskError } from "./task.js";
+const exec = promisify(execFile);
+function sha256(value) {
+    return createHash("sha256").update(value).digest("hex");
+}
+function inside(root, relative) {
+    const resolved = path.resolve(root, relative);
+    const remainder = path.relative(root, resolved);
+    return remainder !== ".." && !remainder.startsWith(`..${path.sep}`) && !path.isAbsolute(remainder);
+}
+/** Fail before any billed model calls when a grader is malformed. */
+async function preflightTask(task) {
+    for (const grader of task.graders) {
+        try {
+            createGrader(grader);
+        }
+        catch (error) {
+            throw new TaskError(`${task.name}: invalid ${grader.type} grader: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        if (typeof grader.path === "string" && !inside(task.fixtureDirectory, grader.path)) {
+            throw new TaskError(`${task.name}: grader path leaves the fixture: ${grader.path}`);
+        }
+        if (grader.type === "file-unchanged") {
+            const file = path.join(task.fixtureDirectory, grader.path);
+            try {
+                if (!(await stat(file)).isFile())
+                    throw new Error("not a file");
+            }
+            catch {
+                throw new TaskError(`${task.name}: file-unchanged requires an existing fixture file: ${grader.path}`);
+            }
+        }
+    }
+}
+/** Hash both the task definition and fixture bytes, independent of directory location. */
+async function hashTask(task) {
+    const hash = createHash("sha256");
+    async function addFile(file, relative) {
+        const bytes = await readFile(file);
+        hash.update("file\0").update(relative).update("\0").update(String(bytes.length)).update("\0").update(bytes);
+    }
+    await addFile(path.join(task.directory, "task.json"), "task.json");
+    async function walk(directory, prefix) {
+        let entries;
+        try {
+            entries = await readdir(directory, { withFileTypes: true });
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return;
+            throw error;
+        }
+        for (const entry of entries.sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) {
+            const relative = `${prefix}${entry.name}`;
+            if (entry.isSymbolicLink()) {
+                throw new TaskError(`${task.name}: fixture contains a symlink: ${relative}`);
+            }
+            if (entry.isDirectory()) {
+                hash.update("dir\0").update(relative).update("\0");
+                await walk(path.join(directory, entry.name), `${relative}/`);
+            }
+            else if (entry.isFile()) {
+                await addFile(path.join(directory, entry.name), relative);
+            }
+            else {
+                throw new TaskError(`${task.name}: unsupported fixture entry: ${relative}`);
+            }
+        }
+    }
+    await walk(task.fixtureDirectory, "fixture/");
+    return hash.digest("hex");
+}
+export async function describeSuite(tasks) {
+    const names = new Set();
+    const entries = [];
+    for (const task of [...tasks].sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) {
+        if (names.has(task.name))
+            throw new TaskError(`Duplicate task name: ${task.name}`);
+        names.add(task.name);
+        await preflightTask(task);
+        entries.push({
+            name: task.name,
+            sha256: await hashTask(task),
+            attempts: task.attempts,
+            maxTurns: task.maxTurns,
+            capabilities: task.capabilities,
+        });
+    }
+    return { suiteSha256: sha256(entries.map((entry) => `${entry.name}\0${entry.sha256}\n`).join("")), tasks: entries };
+}
+/** Unknown Git state stays unknown; do not pretend it is a clean checkout. */
+export async function gitRevision(cwd = process.cwd()) {
+    try {
+        const revision = (await exec("git", ["rev-parse", "HEAD"], { cwd, timeout: 3000 })).stdout.trim();
+        const status = (await exec("git", ["status", "--porcelain"], { cwd, timeout: 3000 })).stdout;
+        return { revision, dirty: status.trim().length > 0 };
+    }
+    catch {
+        return { revision: null, dirty: null };
+    }
+}
+export function buildBaselineManifest(suite, git, options) {
+    return {
+        suite,
+        ...git,
+        provider: options.provider,
+        model: options.model,
+        grantedCapabilities: options.capabilities,
+        attemptedTasks: suite.tasks.filter(t => t.capabilities.every(cap => options.capabilities.includes(cap))).map(t => t.name),
+        skippedTasks: suite.tasks.filter(t => t.capabilities.some(cap => !options.capabilities.includes(cap))).map(t => t.name),
+        attemptsOverride: options.attemptsOverride ?? null,
+        promptSha256: sha256(options.systemPrompts.join("\0")),
+    };
+}
+//# sourceMappingURL=baseline.js.map

package/dist/evals/baseline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"baseline.js","sourceRoot":"","sources":["../../src/evals/baseline.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,SAAS,EAA8B,MAAM,WAAW,CAAC;AAElE,MAAM,IAAI,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AA4BjC,SAAS,MAAM,CAAC,KAAa;IAC3B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,SAAS,MAAM,CAAC,IAAY,EAAE,QAAgB;IAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAChD,OAAO,SAAS,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;AACrG,CAAC;AAED,qEAAqE;AACrE,KAAK,UAAU,aAAa,CAAC,IAAU;IACrC,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QAClC,IAAI,CAAC;YACH,YAAY,CAAC,MAAM,CAAC,CAAC;QACvB,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,IAAI,aAAa,MAAM,CAAC,IAAI,YAAY,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChI,CAAC;QACD,IAAI,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACnF,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,IAAI,qCAAqC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;QACtF,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,KAAK,gBAAgB,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,MAAM,CAAC,IAAc,CAAC,CAAC;YACrE,IAAI,CAAC;gBACH,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE;oBAAE,MAAM,IAAI,KAAK,CAAC,YAAY,CAAC,CAAC;YAClE,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,IAAI,uDAAuD,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;YACxG,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,0FAA0F;AAC1F,KAAK,UAAU,QAAQ,CAAC,IAAU;IAChC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,UAAU,OAAO,CAAC,IAAY,EAAE,QAAgB;QACnD,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC9G,CAAC;IACD,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;IAEnE,KAAK,UAAU,IAAI,CAAC,SAAiB,EAAE,MAAc;QACnD,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,OAAO,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9D,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,OAAO;YAC/D,MAAM,KAAK,CAAC;QACd,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3F,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;YAC1C,IAAI,KAAK,CAAC,cAAc,EAAE,EAAE,CAAC;gBAC3B,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,IAAI,iCAAiC,QAAQ,EAAE,CAAC,CAAC;YAC/E,CAAC;YACD,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACnD,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,QAAQ,GAAG,CAAC,CAAC;YAC/D,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;gBAC1B,MAAM,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,CAAC;YAC5D,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,CAAC,IAAI,gCAAgC,QAAQ,EAAE,CAAC,CAAC;YAC9E,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,UAAU,CAAC,CAAC;IAC9C,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAAsB;IACxD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7F,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,SAAS,CAAC,wBAAwB,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACnF,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrB,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAC1B,OAAO,CAAC,IAAI,CAAC;YACX,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,MAAM,QAAQ,CAAC,IAAI,CAAC;YAC5B,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC;AACtH,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,GAAG,GAAW,OAAO,CAAC,GAAG,EAAE;IAC3D,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,WAAW,EAAE,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QAClG,MAAM,MAAM,GAAG,CAAC,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,QAAQ,EAAE,aAAa,CAAC,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC7F,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACzC,CAAC;AACH,CAAC;AAED,MAAM,UAAU,qBAAqB,CACnC,KAAoB,EACpB,GAAuD,EACvD,OAMC;IAED,OAAO;QACL,KAAK;QACL,GAAG,GAAG;QACN,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,mBAAmB,EAAE,OAAO,CAAC,YAAY;QACzC,cAAc,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACzH,YAAY,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACvH,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,IAAI;QAClD,YAAY,EAAE,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KACvD,CAAC;AACJ,CAAC"}

package/dist/evals/cli.js

@@ -9,12 +9,14 @@ import { formatFailures, formatTable, toJsonReport } from "./report.js";
 import { MissingCapabilityError, runTask } from "./runner.js";
 import { loadTasks, TaskError } from "./task.js";
 import { parseEvalArgs } from "./parse-args.js";
+import { buildBaselineManifest, describeSuite, gitRevision } from "./baseline.js";
 const HELP = `chivgent evals ${VERSION}
 
 Usage:
   npm run eval -- [options]
 
 Options:
+  --dry-run          Validate and fingerprint the suite; no API calls
   --task NAME        Run only this task (repeatable)
   --attempts N       Override the attempts each task declares
   --dir PATH         Task directory (default: evals)
@@ -79,6 +81,29 @@ async function main(argv) {
         return 1;
     }
     const cliOptions = parseCliArgs([...options.providerArgs], process.env);
+    // Capture the exact task definitions and fixture bytes before spending tokens.
+    const suite = await describeSuite(tasks);
+    const baseline = buildBaselineManifest(suite, await gitRevision(), {
+        provider: cliOptions.provider,
+        model: cliOptions.model ?? "unknown",
+        capabilities: options.capabilities,
+        ...(options.attempts === undefined ? {} : { attemptsOverride: options.attempts }),
+        systemPrompts: tasks.map((task) => `${task.name}\n${buildSystemPrompt(task.capabilities)}`),
+    });
+    if (options.dryRun) {
+        const preflight = { schemaVersion: 1, kind: "eval-preflight", baseline };
+        const json = `${JSON.stringify(preflight, null, 2)}\n`;
+        process.stdout.write(json);
+        if (options.jsonPath !== undefined) {
+            await writeFile(options.jsonPath, json);
+            process.stderr.write(`Wrote ${options.jsonPath} (no model calls)\n`);
+        }
+        return 0;
+    }
+    if (baseline.attemptedTasks.length === 0) {
+        process.stderr.write("No runnable tasks: grant required capabilities with --allow-writes and/or --allow-shell.\n");
+        return 1;
+    }
     const llm = await createConfiguredClient(cliOptions);
     if (typeof llm === "string") {
         // The same configuration message the main CLI gives, rather than a run of
@@ -87,7 +112,7 @@ async function main(argv) {
         return 1;
     }
     const startedAt = new Date().toISOString();
-    process.stderr.write(`Evaluating ${cliOptions.provider}/${cliOptions.model ?? "?"} on ${tasks.length} task(s).\n\n`);
+    process.stderr.write(`Evaluating ${cliOptions.provider}/${cliOptions.model ?? "?"} on ${baseline.attemptedTasks.length}/${tasks.length} task(s), suite ${suite.suiteSha256.slice(0, 12)}.\n\n`);
     const results = [];
     for (const task of tasks) {
         try {
@@ -120,6 +145,7 @@ async function main(argv) {
             model: cliOptions.model ?? "unknown",
             version: VERSION,
             startedAt,
+            baseline,
         });
         await writeFile(options.jsonPath, `${JSON.stringify(report, null, 2)}\n`);
         process.stderr.write(`\nWrote ${options.jsonPath}\n`);

package/dist/evals/cli.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/evals/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,OAAO,MAAM,cAAc,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,mBAAmB,GACpB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,EAAE,sBAAsB,EAAE,OAAO,EAAmB,MAAM,aAAa,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,SAAS,EAA8B,MAAM,WAAW,CAAC;AAC7E,OAAO,EAAE,aAAa,EAAoB,MAAM,iBAAiB,CAAC;AAElE,MAAM,IAAI,GAAG,kBAAkB,OAAO;;;;;;;;;;;;;;;;;;;;;CAqBrC,CAAC;AAGF,SAAS,iBAAiB,CAAC,YAAmC;IAC5D,MAAM,QAAQ,GAAG,CAAC,aAAa,CAAC,CAAC;IACjC,IAAI,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACnC,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,WAAW,CAClB,GAAoB,EACpB,MAAyB;IAEzB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACpD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,SAAS,CACjB,iBAAiB,IAAI,iBAAiB,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC/D,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,IAAuB;IACzC,IAAI,OAAoB,CAAC;IACzB,IAAI,CAAC;QACH,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAC9D,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,KAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,KAAK,GAAG,WAAW,CACjB,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAChD,OAAO,CAAC,KAAK,CACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAC9D,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,OAAO,CAAC,SAAS,KAAK,CAAC,CAAC;QAClE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,UAAU,GAAG,YAAY,CAAC,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IACxE,MAAM,GAAG,GAAG,MAAM,sBAAsB,CAAC,UAAU,CAAC,CAAC;IACrD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,0EAA0E;QAC1E,iDAAiD;QACjD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;QACjC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,cAAc,UAAU,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,IAAI,GAAG,OAAO,KAAK,CAAC,MAAM,eAAe,CAC/F,CAAC;IAEF,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,OAAO,CAAC,IAAI,CACV,MAAM,OAAO,CAAC,IAAI,EAAE;gBAClB,YAAY,EAAE,GAAG,EAAE,CAAC,GAAG;gBACvB,YAAY,EAAE,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC;gBAClD,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACzE,SAAS,EAAE,CAAC,MAAM,EAAE,EAAE;oBACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,KAAK,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CACzE,CAAC;gBACJ,CAAC;aACF,CAAC,CACH,CAAC;QACJ,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAI,KAAK,YAAY,sBAAsB,EAAE,CAAC;gBAC5C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;gBACpD,SAAS;YACX,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,EAAE;YACnC,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,KAAK,EAAE,UAAU,CAAC,KAAK,IAAI,SAAS;YACpC,OAAO,EAAE,OAAO;YAChB,SAAS;SACV,CAAC,CAAC;QACH,MAAM,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,yEAAyE;IACzE,OAAO,CAAC,CAAC;AACX,CAAC;AAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KACxB,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;IACjB,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;AAC9B,CAAC,CAAC;KACD,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;IACzE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,OAAO,IAAI,CAAC,CAAC;IACzD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}
+{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/evals/cli.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC7C,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,OAAO,MAAM,cAAc,CAAC;AACnC,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC1D,OAAO,EAAE,sBAAsB,EAAE,MAAM,wBAAwB,CAAC;AAChE,OAAO,EACL,mBAAmB,EACnB,aAAa,EACb,mBAAmB,GACpB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,EAAE,sBAAsB,EAAE,OAAO,EAAmB,MAAM,aAAa,CAAC;AAC/E,OAAO,EAAE,SAAS,EAAE,SAAS,EAA8B,MAAM,WAAW,CAAC;AAC7E,OAAO,EAAE,aAAa,EAAoB,MAAM,iBAAiB,CAAC;AAClE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAElF,MAAM,IAAI,GAAG,kBAAkB,OAAO;;;;;;;;;;;;;;;;;;;;;;CAsBrC,CAAC;AAGF,SAAS,iBAAiB,CAAC,YAAmC;IAC5D,MAAM,QAAQ,GAAG,CAAC,aAAa,CAAC,CAAC;IACjC,IAAI,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACnC,QAAQ,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,WAAW,CAClB,GAAoB,EACpB,MAAyB;IAEzB,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,GAAG,CAAC;IACb,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACpD,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YACrB,MAAM,IAAI,SAAS,CACjB,iBAAiB,IAAI,iBAAiB,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC/D,CAAC;QACJ,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED,KAAK,UAAU,IAAI,CAAC,IAAuB;IACzC,IAAI,OAAoB,CAAC;IACzB,IAAI,CAAC;QACH,OAAO,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAC9D,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,KAAsB,CAAC;IAC3B,IAAI,CAAC;QACH,KAAK,GAAG,WAAW,CACjB,MAAM,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAChD,OAAO,CAAC,KAAK,CACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAC9D,CAAC;QACF,OAAO,CAAC,CAAC;IACX,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,OAAO,CAAC,SAAS,KAAK,CAAC,CAAC;QAClE,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,UAAU,GAAG,YAAY,CAAC,CAAC,GAAG,OAAO,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;IACxE,+EAA+E;IAC/E,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,qBAAqB,CAAC,KAAK,EAAE,MAAM,WAAW,EAAE,EAAE;QACjE,QAAQ,EAAE,UAAU,CAAC,QAAQ;QAC7B,KAAK,EAAE,UAAU,CAAC,KAAK,IAAI,SAAS;QACpC,YAAY,EAAE,OAAO,CAAC,YAAY;QAClC,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;QACjF,aAAa,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,KAAK,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;KAC5F,CAAC,CAAC;IAEH,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,SAAS,GAAG,EAAE,aAAa,EAAE,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,QAAQ,EAAE,CAAC;QACzE,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;QACvD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC3B,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;YACnC,MAAM,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YACxC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,OAAO,CAAC,QAAQ,qBAAqB,CAAC,CAAC;QACvE,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,QAAQ,CAAC,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,4FAA4F,CAAC,CAAC;QACnH,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,GAAG,GAAG,MAAM,sBAAsB,CAAC,UAAU,CAAC,CAAC;IACrD,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,0EAA0E;QAC1E,iDAAiD;QACjD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;QACjC,OAAO,CAAC,CAAC;IACX,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,cAAc,UAAU,CAAC,QAAQ,IAAI,UAAU,CAAC,KAAK,IAAI,GAAG,OAAO,QAAQ,CAAC,cAAc,CAAC,MAAM,IAAI,KAAK,CAAC,MAAM,mBAAmB,KAAK,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAC1K,CAAC;IAEF,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,OAAO,CAAC,IAAI,CACV,MAAM,OAAO,CAAC,IAAI,EAAE;gBAClB,YAAY,EAAE,GAAG,EAAE,CAAC,GAAG;gBACvB,YAAY,EAAE,iBAAiB,CAAC,IAAI,CAAC,YAAY,CAAC;gBAClD,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACzE,SAAS,EAAE,CAAC,MAAM,EAAE,EAAE;oBACpB,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,KAAK,IAAI,CAAC,IAAI,IAAI,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,IAAI,CACzE,CAAC;gBACJ,CAAC;aACF,CAAC,CACH,CAAC;QACJ,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,IAAI,KAAK,YAAY,sBAAsB,EAAE,CAAC;gBAC5C,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;gBACpD,SAAS;YACX,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAClD,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAC;IACxC,CAAC;IAED,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,EAAE;YACnC,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,KAAK,EAAE,UAAU,CAAC,KAAK,IAAI,SAAS;YACpC,OAAO,EAAE,OAAO;YAChB,SAAS;YACT,QAAQ;SACT,CAAC,CAAC;QACH,MAAM,SAAS,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAC1E,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAC;IACxD,CAAC;IAED,yEAAyE;IACzE,OAAO,CAAC,CAAC;AACX,CAAC;AAED,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;KACxB,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;IACjB,OAAO,CAAC,QAAQ,GAAG,QAAQ,CAAC;AAC9B,CAAC,CAAC;KACD,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC;IACzE,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,OAAO,IAAI,CAAC,CAAC;IACzD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAC;AACvB,CAAC,CAAC,CAAC"}

package/dist/evals/parse-args.d.ts

@@ -13,6 +13,8 @@ export interface EvalOptions {
     readonly capabilities: readonly Capability[];
     readonly providerArgs: readonly string[];
     readonly help: boolean;
+    /** Validate and fingerprint the suite without credentials or model calls. */
+    readonly dryRun: boolean;
 }
 export declare function parseEvalArgs(argv: readonly string[]): EvalOptions;
 //# sourceMappingURL=parse-args.d.ts.map

package/dist/evals/parse-args.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-args.d.ts","sourceRoot":"","sources":["../../src/evals/parse-args.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAE5C;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;CACxB;AAUD,wBAAgB,aAAa,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,WAAW,CA4DlE"}
+{"version":3,"file":"parse-args.d.ts","sourceRoot":"","sources":["../../src/evals/parse-args.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAE5C;;;;;GAKG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IAClC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,YAAY,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C,QAAQ,CAAC,YAAY,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,6EAA6E;IAC7E,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;CAC1B;AAUD,wBAAgB,aAAa,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,GAAG,WAAW,CAgElE"}

package/dist/evals/parse-args.js

@@ -13,6 +13,7 @@ export function parseEvalArgs(argv) {
     let directory = "evals";
     let jsonPath;
     let help = false;
+    let dryRun = false;
     const value = (index, option) => {
         const next = argv[index + 1];
         if (next === undefined || next.startsWith("-")) {
@@ -25,6 +26,9 @@ export function parseEvalArgs(argv) {
         if (argument === "--help" || argument === "-h") {
             help = true;
         }
+        else if (argument === "--dry-run") {
+            dryRun = true;
+        }
         else if (argument === "--task") {
             tasks.push(value(index, "--task"));
             index += 1;
@@ -69,6 +73,7 @@ export function parseEvalArgs(argv) {
         capabilities,
         providerArgs,
         help,
+        dryRun,
     };
 }
 //# sourceMappingURL=parse-args.js.map

package/dist/evals/parse-args.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-args.js","sourceRoot":"","sources":["../../src/evals/parse-args.ts"],"names":[],"mappings":"AAkBA;;;;;GAKG;AACH,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;AAEzE,MAAM,UAAU,aAAa,CAAC,IAAuB;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,YAAY,GAAiB,EAAE,CAAC;IACtC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,QAA4B,CAAC;IACjC,IAAI,SAAS,GAAG,OAAO,CAAC;IACxB,IAAI,QAA4B,CAAC;IACjC,IAAI,IAAI,GAAG,KAAK,CAAC;IAEjB,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,MAAc,EAAU,EAAE;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC7B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,SAAS,CAAC,GAAG,MAAM,oBAAoB,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7B,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC/C,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;aAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;YACnC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;YAClD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,EAAE,EAAE,CAAC;gBAC/D,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;YACrE,CAAC;YACD,QAAQ,GAAG,MAAM,CAAC;YAClB,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YAChC,SAAS,GAAG,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAClC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,QAAQ,GAAG,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,gBAAgB,EAAE,CAAC;YACzC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;aAAM,IAAI,QAAQ,KAAK,eAAe,EAAE,CAAC;YACxC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;aAAM,IAAI,QAAQ,KAAK,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpE,uEAAuE;YACvE,kDAAkD;YAClD,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;YACpD,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAClC,MAAM,IAAI,SAAS,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK;QACL,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;QAC/C,SAAS;QACT,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;QAC/C,YAAY;QACZ,YAAY;QACZ,IAAI;KACL,CAAC;AACJ,CAAC"}
+{"version":3,"file":"parse-args.js","sourceRoot":"","sources":["../../src/evals/parse-args.ts"],"names":[],"mappings":"AAoBA;;;;;GAKG;AACH,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;AAEzE,MAAM,UAAU,aAAa,CAAC,IAAuB;IACnD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,YAAY,GAAiB,EAAE,CAAC;IACtC,MAAM,YAAY,GAAa,EAAE,CAAC;IAClC,IAAI,QAA4B,CAAC;IACjC,IAAI,SAAS,GAAG,OAAO,CAAC;IACxB,IAAI,QAA4B,CAAC;IACjC,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,IAAI,MAAM,GAAG,KAAK,CAAC;IAEnB,MAAM,KAAK,GAAG,CAAC,KAAa,EAAE,MAAc,EAAU,EAAE;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC7B,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,SAAS,CAAC,GAAG,MAAM,oBAAoB,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC;IAEF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAC7B,IAAI,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC/C,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;aAAM,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;YACpC,MAAM,GAAG,IAAI,CAAC;QAChB,CAAC;aAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;YACnC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;YACrC,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC,CAAC;YAClD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,EAAE,EAAE,CAAC;gBAC/D,MAAM,IAAI,SAAS,CAAC,6CAA6C,CAAC,CAAC;YACrE,CAAC;YACD,QAAQ,GAAG,MAAM,CAAC;YAClB,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,OAAO,EAAE,CAAC;YAChC,SAAS,GAAG,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAClC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,QAAQ,GAAG,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,gBAAgB,EAAE,CAAC;YACzC,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;aAAM,IAAI,QAAQ,KAAK,eAAe,EAAE,CAAC;YACxC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC7B,CAAC;aAAM,IAAI,QAAQ,KAAK,SAAS,IAAI,gBAAgB,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACpE,uEAAuE;YACvE,kDAAkD;YAClD,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;YACpD,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;aAAM,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAClC,MAAM,IAAI,SAAS,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;QACrD,CAAC;IACH,CAAC;IAED,OAAO;QACL,KAAK;QACL,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;QAC/C,SAAS;QACT,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC;QAC/C,YAAY;QACZ,YAAY;QACZ,IAAI;QACJ,MAAM;KACP,CAAC;AACJ,CAAC"}

package/dist/evals/report.d.ts

@@ -1,9 +1,11 @@
 import type { TaskResult } from "./runner.js";
+import type { BaselineManifest } from "./baseline.js";
 export interface ReportMeta {
     readonly provider: string;
     readonly model: string;
     readonly version: string;
     readonly startedAt: string;
+    readonly baseline?: BaselineManifest;
 }
 /**
  * The table always shows the sample size next to the score.

package/dist/evals/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/evals/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAG9C,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AA4CD;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,MAAM,CA0ClE;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,MAAM,CAgBrE;AAED,wBAAgB,YAAY,CAC1B,OAAO,EAAE,SAAS,UAAU,EAAE,EAC9B,IAAI,EAAE,UAAU,GACf,MAAM,CAmCR"}
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/evals/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAEtD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;CACtC;AAuCD;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,MAAM,CA0ClE;AAED,wEAAwE;AACxE,wBAAgB,cAAc,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,GAAG,MAAM,CAgBrE;AAED,wBAAgB,YAAY,CAC1B,OAAO,EAAE,SAAS,UAAU,EAAE,EAC9B,IAAI,EAAE,UAAU,GACf,MAAM,CAkCR"}

package/dist/evals/report.js

@@ -22,9 +22,11 @@ function medianTokens(result) {
         .filter((value) => value !== undefined);
     return totals.length === 0 ? "-" : formatTokens(median(totals));
 }
-function totalTokens(results) {
-    return results.reduce((sum, result) => sum +
-        result.attempts.reduce((inner, attempt) => inner + (attempt.usage?.usage.totalTokens ?? 0), 0), 0);
+function measuredTokens(results) {
+    const attempts = results.flatMap((result) => result.attempts);
+    const missing = attempts.filter((attempt) => attempt.usage === undefined || !attempt.usage.complete).length;
+    const sum = attempts.reduce((total, attempt) => total + (attempt.usage?.usage.totalTokens ?? 0), 0);
+    return { total: missing > 0 ? null : sum, missing };
 }
 function pad(value, width) {
     return value.length >= width ? value : value + " ".repeat(width - value.length);
@@ -65,9 +67,9 @@ export function formatTable(results) {
     const passed = results.reduce((total, result) => total + result.passed, 0);
     const total = results.reduce((sum, result) => sum + result.total, 0);
     const percent = total === 0 ? 0 : Math.round((passed / total) * 100);
-    const spent = totalTokens(results);
+    const tokens = measuredTokens(results);
     lines.push("");
-    lines.push(`overall  ${passed}/${total} (${percent}%)${spent === 0 ? "" : `  ${formatTokens(spent)} tokens`}`);
+    lines.push(`overall  ${passed}/${total} (${percent}%)${tokens.total === null ? `  token usage unavailable/incomplete (${tokens.missing} attempt(s))` : `  ${formatTokens(tokens.total)} tokens`}`);
     return `${lines.join("\n")}\n`;
 }
 /** Failures, grouped so the same reason across attempts is one line. */
@@ -90,7 +92,7 @@ export function formatFailures(results) {
 }
 export function toJsonReport(results, meta) {
     return {
-        schemaVersion: 1,
+        schemaVersion: 2,
         ...meta,
         tasks: results.map((result) => ({
             name: result.task,
@@ -98,7 +100,8 @@ export function toJsonReport(results, meta) {
             total: result.total,
             passRate: result.total === 0 ? 0 : result.passed / result.total,
             meanTurns: mean(result.attempts.map((attempt) => attempt.turnCount)),
-            totalTokens: result.attempts.reduce((sum, attempt) => sum + (attempt.usage?.usage.totalTokens ?? 0), 0),
+            totalTokens: measuredTokens([result]).total,
+            missingUsageAttempts: measuredTokens([result]).missing,
             medianDurationMs: median(result.attempts.map((attempt) => attempt.durationMs)),
             attempts: result.attempts.map((attempt) => ({
                 attempt: attempt.attempt,
@@ -115,7 +118,8 @@ export function toJsonReport(results, meta) {
         overall: {
             passed: results.reduce((total, result) => total + result.passed, 0),
             total: results.reduce((sum, result) => sum + result.total, 0),
-            totalTokens: totalTokens(results),
+            totalTokens: measuredTokens(results).total,
+            missingUsageAttempts: measuredTokens(results).missing,
         },
     };
 }

package/dist/evals/report.js.map

@@ -1 +1 @@
-{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/evals/report.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AASrD,SAAS,MAAM,CAAC,MAAyB;IACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC;QAC5B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACrE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,IAAI,CAAC,MAAyB;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAC3E,CAAC;AAED,iEAAiE;AACjE,SAAS,YAAY,CAAC,MAAkB;IACtC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ;SAC3B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC;SAClD,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAC3D,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,WAAW,CAAC,OAA8B;IACjD,OAAO,OAAO,CAAC,MAAM,CACnB,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CACd,GAAG;QACH,MAAM,CAAC,QAAQ,CAAC,MAAM,CACpB,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC,EACnE,CAAC,CACF,EACH,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa;IACvC,OAAO,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;AAClF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,OAA8B;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACpC,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,IAAI,EAAE,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,EAAE;QACxC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3E,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC;QAC5B,KAAK,EAAE;YACL,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;SACpE,CAAC,IAAI,CAAC,GAAG,CAAC;QACX,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;KAC7F,CAAC,CAAC,CAAC;IAEJ,MAAM,MAAM,GAAG;QACb,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1D,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5D,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;KAC3D,CAAC;IAEF,MAAM,KAAK,GAAG;QACZ,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO;KAChK,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CACjL,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACrE,MAAM,OAAO,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;IACrE,MAAM,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IACnC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CACR,YAAY,MAAM,IAAI,KAAK,KAAK,OAAO,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,KAAK,CAAC,SAAS,EAAE,CACnG,CAAC;IACF,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACjC,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,OAA8B;IAC3D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC7C,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC7C,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC/B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QACD,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,QAAQ,EAAE,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,YAAY,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,OAA8B,EAC9B,IAAgB;IAEhB,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,GAAG,IAAI;QACP,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAC9B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK;YAC/D,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACpE,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,MAAM,CACjC,CAAC,GAAG,EAAE,OAAO,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC,EAC/D,CAAC,CACF;YACD,gBAAgB,EAAE,MAAM,CACtB,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CACrD;YACD,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC1C,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;gBAChE,QAAQ,EAAE,OAAO,CAAC,QAAQ;aAC3B,CAAC,CAAC;SACJ,CAAC,CAAC;QACH,OAAO,EAAE;YACP,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YACnE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YAC7D,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC;SAClC;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/evals/report.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAWrD,SAAS,MAAM,CAAC,MAAyB;IACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC;QAC5B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACrE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;AAC5B,CAAC;AAED,SAAS,IAAI,CAAC,MAAyB;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,CAAC;IACX,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;AAC3E,CAAC;AAED,iEAAiE;AACjE,SAAS,YAAY,CAAC,MAAkB;IACtC,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ;SAC3B,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC;SAClD,MAAM,CAAC,CAAC,KAAK,EAAmB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAC3D,OAAO,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,cAAc,CAAC,OAA8B;IACpD,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC9D,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;IAC5G,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACpG,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,OAAO,EAAE,CAAC;AACtD,CAAC;AAED,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa;IACvC,OAAO,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;AAClF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,OAA8B;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,iBAAiB,CAAC;IAC3B,CAAC;IAED,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACpC,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,IAAI,EAAE,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,EAAE;QACxC,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;QAC3E,MAAM,EAAE,YAAY,CAAC,MAAM,CAAC;QAC5B,KAAK,EAAE;YACL,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;SACpE,CAAC,IAAI,CAAC,GAAG,CAAC;QACX,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG;KAC7F,CAAC,CAAC,CAAC;IAEJ,MAAM,MAAM,GAAG;QACb,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACxD,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC1D,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC5D,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;KAC3D,CAAC;IAEF,MAAM,KAAK,GAAG;QACZ,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,SAAS,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,OAAO,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO;KAChK,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,KAAK,CAAC,IAAI,CACR,GAAG,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CACjL,CAAC;IACJ,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3E,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACrE,MAAM,OAAO,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;IACrE,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACvC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CACR,YAAY,MAAM,IAAI,KAAK,KAAK,OAAO,KAAK,MAAM,CAAC,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,yCAAyC,MAAM,CAAC,OAAO,cAAc,CAAC,CAAC,CAAC,KAAK,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CACvL,CAAC;IACF,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AACjC,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,cAAc,CAAC,OAA8B;IAC3D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC7C,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;gBAC7C,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;gBAC/B,QAAQ,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YAClC,CAAC;QACH,CAAC;QACD,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,QAAQ,EAAE,CAAC;YAC1C,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,YAAY,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,MAAM,EAAE,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;AAC3D,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,OAA8B,EAC9B,IAAgB;IAEhB,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,GAAG,IAAI;QACP,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;YAC9B,IAAI,EAAE,MAAM,CAAC,IAAI;YACjB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,QAAQ,EAAE,MAAM,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK;YAC/D,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;YACpE,WAAW,EAAE,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK;YAC3C,oBAAoB,EAAE,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO;YACtD,gBAAgB,EAAE,MAAM,CACtB,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC,CACrD;YACD,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC1C,OAAO,EAAE,OAAO,CAAC,OAAO;gBACxB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,UAAU,EAAE,OAAO,CAAC,UAAU;gBAC9B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,SAAS,EAAE,OAAO,CAAC,SAAS;gBAC5B,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;gBAChE,QAAQ,EAAE,OAAO,CAAC,QAAQ;aAC3B,CAAC,CAAC;SACJ,CAAC,CAAC;QACH,OAAO,EAAE;YACP,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YACnE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YAC7D,WAAW,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC,KAAK;YAC1C,oBAAoB,EAAE,cAAc,CAAC,OAAO,CAAC,CAAC,OAAO;SACtD;KACF,CAAC;AACJ,CAAC"}

package/dist/evals/runner.d.ts

@@ -1,5 +1,6 @@
 import type { LLMClient } from "../llm.js";
 import type { UsageTotal } from "../providers/usage.js";
+import type { ShellOperations } from "../shell/types.js";
 import { type AttemptFacts } from "./graders.js";
 import type { Capability, Task } from "./task.js";
 /** One tool result, in call order. */
@@ -52,6 +53,8 @@ export interface RunnerOptions {
      * leak into one another through it.
      */
     readonly createClient: () => LLMClient;
+    /** Test-only execution override; the default is the Docker sandbox. */
+    readonly shellOperations?: ShellOperations;
     readonly systemPrompt: string;
     readonly capabilities: readonly Capability[];
     readonly attempts?: number;

package/dist/evals/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAKxD,OAAO,EAAgC,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAC/E,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAElD,sCAAsC;AACtC,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB;;;;;;;;;OASG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IACxC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,EAAE,SAAS,cAAc,EAAE,CAAC;IAC9C,4DAA4D;IAC5D,QAAQ,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC;IAC5B,kDAAkD;IAClD,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,CAAC;IAC5C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B;;;;;OAKG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,SAAS,CAAC;IACvC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,YAAY,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC;IACjE,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;CAC/B;AAED,qBAAa,sBAAuB,SAAQ,KAAK;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM;IAAE,QAAQ,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE;IAA1E,YAAqB,IAAI,EAAE,MAAM,EAAW,OAAO,EAAE,SAAS,UAAU,EAAE,EAOzE;CACF;AAwHD;;;;;;GAMG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,UAAU,CAAC,CAoBrB;AAED,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,SAAS,UAAU,EAAE,CAAC,CAMhC"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AAIxD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAGzD,OAAO,EAAgC,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAC/E,OAAO,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAElD,sCAAsC;AACtC,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB;;;;;;;;;OASG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;CAC1B;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC,QAAQ,CAAC,CAAC;IACxC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC;;;;;;OAMG;IACH,QAAQ,CAAC,SAAS,EAAE,SAAS,cAAc,EAAE,CAAC;IAC9C,4DAA4D;IAC5D,QAAQ,CAAC,KAAK,CAAC,EAAE,UAAU,CAAC;IAC5B,kDAAkD;IAClD,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,CAAC;IAC5C,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,aAAa;IAC5B;;;;;OAKG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,SAAS,CAAC;IACvC,uEAAuE;IACvE,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAC;IAC3C,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,YAAY,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC;IACjE,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;CAC/B;AAED,qBAAa,sBAAuB,SAAQ,KAAK;IACnC,QAAQ,CAAC,IAAI,EAAE,MAAM;IAAE,QAAQ,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE;IAA1E,YAAqB,IAAI,EAAE,MAAM,EAAW,OAAO,EAAE,SAAS,UAAU,EAAE,EAOzE;CACF;AA2HD;;;;;;GAMG;AACH,wBAAsB,OAAO,CAC3B,IAAI,EAAE,IAAI,EACV,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,UAAU,CAAC,CAoBrB;AAED,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,SAAS,IAAI,EAAE,EACtB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,SAAS,UAAU,EAAE,CAAC,CAMhC"}

package/dist/evals/runner.js

@@ -3,6 +3,7 @@ import { LocalWorkspace } from "../workspace.js";
 import { callTarget } from "../tools/target.js";
 import { toolsFor } from "./tools.js";
 import { createAttemptWorkspace } from "./fixture.js";
+import { ScopedProjectInstructions } from "../context/project-instructions.js";
 import { createGrader, describeGrader } from "./graders.js";
 export class MissingCapabilityError extends Error {
     task;
@@ -28,11 +29,14 @@ async function runAttempt(task, attempt, options) {
             systemPrompt: options.systemPrompt,
             maxTurns: task.maxTurns,
             llm: options.createClient(),
-            tools: toolsFor(task.capabilities, workspace.path),
+            tools: toolsFor(task.capabilities, workspace.path, {
+                ...(options.shellOperations === undefined ? {} : { shellOperations: options.shellOperations }),
+            }),
             workspace: new LocalWorkspace(workspace.path, {
                 allowWrites: task.capabilities.includes("writes"),
             }),
             streaming: false,
+            projectInstructions: new ScopedProjectInstructions(workspace.path),
             onEvent: (event) => events.push(event),
         });
         let status = "error";

package/dist/evals/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,IAAI,UAAU,EAAE,MAAM,aAAa,CAAC;AAIlD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AACtC,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,cAAc,EAAqB,MAAM,cAAc,CAAC;AA+D/E,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAC1B,IAAI;IAAmB,OAAO;IAAnD,YAAqB,IAAY,EAAW,OAA8B;QACxE,KAAK,CACH,SAAS,IAAI,WAAW,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,OAAO;aACjE,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;aACnF,IAAI,CAAC,GAAG,CAAC,GAAG,CAChB,CAAC;oBALiB,IAAI;uBAAmB,OAAO;QAMjD,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED,SAAS,mBAAmB,CAC1B,IAAU,EACV,OAA8B;IAE9B,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAC7B,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAC9C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,IAAU,EACV,OAAe,EACf,OAAsB;IAEtB,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACtE,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,CAAC;QACH,MAAM,KAAK,GAAU,IAAI,UAAU,CAAC;YAClC,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,GAAG,EAAE,OAAO,CAAC,YAAY,EAAE;YAC3B,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,IAAI,CAAC;YAClD,SAAS,EAAE,IAAI,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE;gBAC5C,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC;aAClD,CAAC;YACF,SAAS,EAAE,KAAK;YAChB,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;SACvC,CAAC,CAAC;QAEH,IAAI,MAAM,GAA2B,OAAO,CAAC;QAC7C,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,KAA6B,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE;gBAC1C,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;aACpE,CAAC,CAAC;YACH,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YACvB,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YAC7B,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YACrB,WAAW;gBACT,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,yEAAyE;YACzE,mEAAmE;YACnE,MAAM,GAAG,OAAO,CAAC;YACjB,WAAW,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,CAAC;QAED,MAAM,KAAK,GAAiB;YAC1B,SAAS,EAAE,SAAS,CAAC,IAAI;YACzB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,WAAW;YACX,SAAS;YACT,MAAM;YACN,MAAM;SACP,CAAC;QAEF,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;YAC3B,QAAQ,CAAC,IAAI,CACX,MAAM,KAAK,WAAW;gBACpB,CAAC,CAAC,WAAW,IAAI,CAAC,QAAQ,oCAAoC;gBAC9D,CAAC,CAAC,gBAAgB,MAAM,KAAK,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CACrE,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBACnB,QAAQ,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,uEAAuE;QACvE,kCAAkC;QAClC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;gBAC1C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBAC3C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBACzB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBACxC,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAA8B,MAAM;aAC5C,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,oBAAoB,CAAC;aACtD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACb,MAAM,GAAG,GAAG,KAIX,CAAC;YACF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC3C,OAAO;gBACL,IAAI,EAAE,GAAG,CAAC,QAAQ;gBAClB,EAAE,EAAE,GAAG,CAAC,OAAO,KAAK,KAAK;gBACzB,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;aAC5C,CAAC;QACJ,CAAC,CAAC,CAAC;QAEL,OAAO;YACL,OAAO;YACP,MAAM,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC7B,MAAM;YACN,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;YACzC,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACvD,SAAS,EAAE,KAAK;YAChB,QAAQ;SACT,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;IAC5B,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAAU,EACV,OAAsB;IAEtB,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IAChE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,sBAAsB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC;IAChD,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACtD,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,OAAO,CAAC,SAAS,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,QAAQ;QACR,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM;QAC3D,KAAK;KACN,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAsB,EACtB,OAAsB;IAEtB,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,IAAI,UAAU,EAAE,MAAM,aAAa,CAAC;AAIlD,OAAO,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEtC,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,yBAAyB,EAAE,MAAM,oCAAoC,CAAC;AAC/E,OAAO,EAAE,YAAY,EAAE,cAAc,EAAqB,MAAM,cAAc,CAAC;AAiE/E,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAC1B,IAAI;IAAmB,OAAO;IAAnD,YAAqB,IAAY,EAAW,OAA8B;QACxE,KAAK,CACH,SAAS,IAAI,WAAW,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,gBAAgB,OAAO;aACjE,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;aACnF,IAAI,CAAC,GAAG,CAAC,GAAG,CAChB,CAAC;oBALiB,IAAI;uBAAmB,OAAO;QAMjD,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAED,SAAS,mBAAmB,CAC1B,IAAU,EACV,OAA8B;IAE9B,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAC7B,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC,CAC9C,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,IAAU,EACV,OAAe,EACf,OAAsB;IAEtB,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IACtE,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,CAAC;QACH,MAAM,KAAK,GAAU,IAAI,UAAU,CAAC;YAClC,YAAY,EAAE,OAAO,CAAC,YAAY;YAClC,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,GAAG,EAAE,OAAO,CAAC,YAAY,EAAE;YAC3B,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,SAAS,CAAC,IAAI,EAAE;gBACjD,GAAG,CAAC,OAAO,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,OAAO,CAAC,eAAe,EAAE,CAAC;aAC/F,CAAC;YACF,SAAS,EAAE,IAAI,cAAc,CAAC,SAAS,CAAC,IAAI,EAAE;gBAC5C,WAAW,EAAE,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC;aAClD,CAAC;YACF,SAAS,EAAE,KAAK;YAChB,mBAAmB,EAAE,IAAI,yBAAyB,CAAC,SAAS,CAAC,IAAI,CAAC;YAClE,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;SACvC,CAAC,CAAC;QAEH,IAAI,MAAM,GAA2B,OAAO,CAAC;QAC7C,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,KAA6B,CAAC;QAClC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,EAAE;gBAC1C,GAAG,CAAC,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;aACpE,CAAC,CAAC;YACH,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;YACvB,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;YAC7B,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;YACrB,WAAW;gBACT,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACrE,CAAC;QAAC,OAAO,KAAc,EAAE,CAAC;YACxB,yEAAyE;YACzE,mEAAmE;YACnE,MAAM,GAAG,OAAO,CAAC;YACjB,WAAW,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACvE,CAAC;QAED,MAAM,KAAK,GAAiB;YAC1B,SAAS,EAAE,SAAS,CAAC,IAAI;YACzB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,WAAW;YACX,SAAS;YACT,MAAM;YACN,MAAM;SACP,CAAC;QAEF,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,IAAI,MAAM,KAAK,WAAW,EAAE,CAAC;YAC3B,QAAQ,CAAC,IAAI,CACX,MAAM,KAAK,WAAW;gBACpB,CAAC,CAAC,WAAW,IAAI,CAAC,QAAQ,oCAAoC;gBAC9D,CAAC,CAAC,gBAAgB,MAAM,KAAK,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CACrE,CAAC;QACJ,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBACnB,QAAQ,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC;QAED,uEAAuE;QACvE,kCAAkC;QAClC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,IAAI,KAAK,sBAAsB,EAAE,CAAC;gBAC1C,MAAM,MAAM,GAAG,UAAU,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;gBAC3C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;oBACzB,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;gBACxC,CAAC;YACH,CAAC;QACH,CAAC;QACD,MAAM,KAAK,GAA8B,MAAM;aAC5C,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,oBAAoB,CAAC;aACtD,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACb,MAAM,GAAG,GAAG,KAIX,CAAC;YACF,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC3C,OAAO;gBACL,IAAI,EAAE,GAAG,CAAC,QAAQ;gBAClB,EAAE,EAAE,GAAG,CAAC,OAAO,KAAK,KAAK;gBACzB,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;aAC5C,CAAC;QACJ,CAAC,CAAC,CAAC;QAEL,OAAO;YACL,OAAO;YACP,MAAM,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC7B,MAAM;YACN,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;YACzC,SAAS,EAAE,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;YACvD,SAAS,EAAE,KAAK;YAChB,QAAQ;SACT,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;IAC5B,CAAC;AACH,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAC3B,IAAU,EACV,OAAsB;IAEtB,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,EAAE,OAAO,CAAC,YAAY,CAAC,CAAC;IAChE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,sBAAsB,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvD,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC;IAChD,MAAM,QAAQ,GAAoB,EAAE,CAAC;IACrC,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QACtD,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtB,OAAO,CAAC,SAAS,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACpC,CAAC;IAED,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,QAAQ;QACR,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM;QAC3D,KAAK;KACN,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAsB,EACtB,OAAsB;IAEtB,MAAM,OAAO,GAAiB,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/evals/task.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"task.d.ts","sourceRoot":"","sources":["../../src/evals/task.ts"],"names":[],"mappings":"AAIA,qBAAa,SAAU,SAAQ,KAAK;IAClC,YAAY,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,EAMzD;CACF;AAED,yEAAyE;AACzE,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE5C,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACjC;AAED,MAAM,WAAW,IAAI;IACnB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,YAAY,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IACxC,iEAAiE;IACjE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAoFD,wBAAgB,SAAS,CACvB,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,MAAM,EACjB,gBAAgB,EAAE,MAAM,GACvB,IAAI,CA8CN;AAED,wBAAsB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB/D;AAED,6DAA6D;AAC7D,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAwBtE"}
+{"version":3,"file":"task.d.ts","sourceRoot":"","sources":["../../src/evals/task.ts"],"names":[],"mappings":"AAIA,qBAAa,SAAU,SAAQ,KAAK;IAClC,YAAY,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,EAMzD;CACF;AAED,yEAAyE;AACzE,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,OAAO,CAAC;AAE5C,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACjC;AAED,MAAM,WAAW,IAAI;IACnB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,YAAY,EAAE,SAAS,UAAU,EAAE,CAAC;IAC7C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,OAAO,EAAE,SAAS,UAAU,EAAE,CAAC;IACxC,iEAAiE;IACjE,QAAQ,CAAC,gBAAgB,EAAE,MAAM,CAAC;IAClC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAqFD,wBAAgB,SAAS,CACvB,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,MAAM,EACjB,gBAAgB,EAAE,MAAM,GACvB,IAAI,CA8CN;AAED,wBAAsB,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB/D;AAED,6DAA6D;AAC7D,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC,CAwBtE"}

package/dist/evals/task.js

@@ -45,6 +45,7 @@ const TOOL_NAME_GRADERS = new Set([
     "not-used-tool",
     "tool-succeeded",
     "tool-never-failed",
+    "max-tool-calls",
 ]);
 /**
  * Refuses a grader that names a tool the task never hands to the model.

package/dist/evals/task.js.map

@@ -1 +1 @@
-{"version":3,"file":"task.js","sourceRoot":"","sources":["../../src/evals/task.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,OAAO,SAAU,SAAQ,KAAK;IAClC,YAAY,OAAe,EAAE,OAA6B;QACxD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC;QACxB,IAAI,OAAO,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC7B,CAAC;IACH,CAAC;CACF;AAsBD,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,YAAY,GAAG,EAAE,CAAC;AAExB,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAE,IAAY;IACpD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,kCAAkC,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACzB,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YAC5C,MAAM,IAAI,SAAS,CACjB,GAAG,IAAI,wBAAwB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,iCAAiC,CACtF,CAAC;QACJ,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAc,EACd,QAAgB,EAChB,OAAe,EACf,KAAa,EACb,IAAY;IAEZ,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAK,KAAgB,GAAG,CAAC,IAAK,KAAgB,GAAG,OAAO,EAAE,CAAC;QACzF,MAAM,IAAI,SAAS,CACjB,GAAG,IAAI,KAAK,KAAK,iCAAiC,OAAO,GAAG,CAC7D,CAAC;IACJ,CAAC;IACD,OAAO,KAAe,CAAC;AACzB,CAAC;AAED,yEAAyE;AACzE,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,WAAW;IACX,eAAe;IACf,gBAAgB;IAChB,mBAAmB;CACpB,CAAC,CAAC;AAEH;;;;;;;;GAQG;AACH,SAAS,oBAAoB,CAC3B,IAAY,EACZ,YAAmC,EACnC,OAA8B;IAE9B,MAAM,SAAS,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;IAC7C,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QAChD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzD,SAAS;QACX,CAAC;QACD,MAAM,IAAI,SAAS,CACjB,GAAG,IAAI,aAAa,KAAK,MAAM,MAAM,CAAC,IAAI,YAAY,IAAI,mCAAmC;YAC3F,gCAAgC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YACrF,0CAA0C,CAC7C,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAc,EACd,SAAiB,EACjB,gBAAwB;IAExB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,SAAS,CAAC,GAAG,SAAS,yCAAyC,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,IAAI,GACR,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;QACrD,CAAC,CAAC,KAAK,CAAC,IAAI;QACZ,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC/B,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,sCAAsC,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,oCAAoC,CAAC,CAAC;IACnE,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QACjD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,aAAa,KAAK,iBAAiB,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,KAAmB,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAChE,oBAAoB,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IAElD,OAAO;QACL,IAAI;QACJ,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,YAAY;QACZ,QAAQ,EAAE,mBAAmB,CAC3B,KAAK,CAAC,QAAQ,EACd,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,IAAI,CACL;QACD,QAAQ,EAAE,mBAAmB,CAC3B,KAAK,CAAC,QAAQ,EACd,iBAAiB,EACjB,GAAG,EACH,UAAU,EACV,IAAI,CACL;QACD,OAAO;QACP,gBAAgB;QAChB,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,SAAiB;IAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAC/C,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,IAAI,SAAS,CAAC,kBAAkB,IAAI,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,qBAAqB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,6DAA6D;AAC7D,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,IAAI,SAAS,CAAC,qCAAqC,IAAI,GAAG,EAAE;YAChE,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACzE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACzB,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
+{"version":3,"file":"task.js","sourceRoot":"","sources":["../../src/evals/task.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE1C,MAAM,OAAO,SAAU,SAAQ,KAAK;IAClC,YAAY,OAAe,EAAE,OAA6B;QACxD,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,WAAW,CAAC;QACxB,IAAI,OAAO,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC7B,CAAC;IACH,CAAC;CACF;AAsBD,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAC3B,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAC7B,MAAM,YAAY,GAAG,EAAE,CAAC;AAExB,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc,EAAE,IAAY;IACpD,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,kCAAkC,CAAC,CAAC;IACjE,CAAC;IACD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACzB,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,OAAO,EAAE,CAAC;YAC5C,MAAM,IAAI,SAAS,CACjB,GAAG,IAAI,wBAAwB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,iCAAiC,CACtF,CAAC;QACJ,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,mBAAmB,CAC1B,KAAc,EACd,QAAgB,EAChB,OAAe,EACf,KAAa,EACb,IAAY;IAEZ,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAK,KAAgB,GAAG,CAAC,IAAK,KAAgB,GAAG,OAAO,EAAE,CAAC;QACzF,MAAM,IAAI,SAAS,CACjB,GAAG,IAAI,KAAK,KAAK,iCAAiC,OAAO,GAAG,CAC7D,CAAC;IACJ,CAAC;IACD,OAAO,KAAe,CAAC;AACzB,CAAC;AAED,yEAAyE;AACzE,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,WAAW;IACX,eAAe;IACf,gBAAgB;IAChB,mBAAmB;IACnB,gBAAgB;CACjB,CAAC,CAAC;AAEH;;;;;;;;GAQG;AACH,SAAS,oBAAoB,CAC3B,IAAY,EACZ,YAAmC,EACnC,OAA8B;IAE9B,MAAM,SAAS,GAAG,YAAY,CAAC,YAAY,CAAC,CAAC;IAC7C,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;QAChD,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;QACzB,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACzD,SAAS;QACX,CAAC;QACD,MAAM,IAAI,SAAS,CACjB,GAAG,IAAI,aAAa,KAAK,MAAM,MAAM,CAAC,IAAI,YAAY,IAAI,mCAAmC;YAC3F,gCAAgC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI;YACrF,0CAA0C,CAC7C,CAAC;IACJ,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,KAAc,EACd,SAAiB,EACjB,gBAAwB;IAExB,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,SAAS,CAAC,GAAG,SAAS,yCAAyC,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,IAAI,GACR,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC;QACrD,CAAC,CAAC,KAAK,CAAC,IAAI;QACZ,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IAC/B,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,sCAAsC,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChE,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,oCAAoC,CAAC,CAAC;IACnE,CAAC;IACD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QACjD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACvD,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,aAAa,KAAK,iBAAiB,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,KAAmB,CAAC;IAC7B,CAAC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,gBAAgB,CAAC,KAAK,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC;IAChE,oBAAoB,CAAC,IAAI,EAAE,YAAY,EAAE,OAAO,CAAC,CAAC;IAElD,OAAO;QACL,IAAI;QACJ,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,YAAY;QACZ,QAAQ,EAAE,mBAAmB,CAC3B,KAAK,CAAC,QAAQ,EACd,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,IAAI,CACL;QACD,QAAQ,EAAE,mBAAmB,CAC3B,KAAK,CAAC,QAAQ,EACd,iBAAiB,EACjB,GAAG,EACH,UAAU,EACV,IAAI,CACL;QACD,OAAO;QACP,gBAAgB;QAChB,SAAS;KACV,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,SAAiB;IAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAC/C,IAAI,QAAgB,CAAC;IACrB,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC1C,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,IAAI,SAAS,CAAC,kBAAkB,IAAI,GAAG,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,qBAAqB,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,OAAO,SAAS,CAAC,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC,CAAC;AACvE,CAAC;AAED,6DAA6D;AAC7D,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAY;IAC1C,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACzD,CAAC;IAAC,OAAO,KAAc,EAAE,CAAC;QACxB,MAAM,IAAI,SAAS,CAAC,qCAAqC,IAAI,GAAG,EAAE;YAChE,KAAK,EAAE,KAAK;SACb,CAAC,CAAC;IACL,CAAC;IAED,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACzE,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACzB,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC9C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/evals/tools.d.ts

@@ -1,5 +1,8 @@
 import type { Tool } from "../tools/tool.js";
 import type { Capability } from "./task.js";
+import type { ShellOperations } from "../shell/types.js";
 export declare function toolNamesFor(capabilities: readonly Capability[]): readonly string[];
-export declare function toolsFor(capabilities: readonly Capability[], cwd: string): readonly Tool[];
+export declare function toolsFor(capabilities: readonly Capability[], cwd: string, options?: {
+    readonly shellOperations?: ShellOperations;
+}): readonly Tool[];
 //# sourceMappingURL=tools.d.ts.map

package/dist/evals/tools.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../src/evals/tools.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAO7C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAgB5C,wBAAgB,YAAY,CAC1B,YAAY,EAAE,SAAS,UAAU,EAAE,GAClC,SAAS,MAAM,EAAE,CAKnB;AAED,wBAAgB,QAAQ,CACtB,YAAY,EAAE,SAAS,UAAU,EAAE,EACnC,GAAG,EAAE,MAAM,GACV,SAAS,IAAI,EAAE,CAajB"}
+{"version":3,"file":"tools.d.ts","sourceRoot":"","sources":["../../src/evals/tools.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAQ7C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAC5C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAgBzD,wBAAgB,YAAY,CAC1B,YAAY,EAAE,SAAS,UAAU,EAAE,GAClC,SAAS,MAAM,EAAE,CAKnB;AAED,wBAAgB,QAAQ,CACtB,YAAY,EAAE,SAAS,UAAU,EAAE,EACnC,GAAG,EAAE,MAAM,EACX,OAAO,GAAE;IAAE,QAAQ,CAAC,eAAe,CAAC,EAAE,eAAe,CAAA;CAAO,GAC3D,SAAS,IAAI,EAAE,CAmBjB"}

package/dist/evals/tools.js

@@ -3,6 +3,7 @@ import { ReadFileTool } from "../tools/read-file.js";
 import { SearchTextTool } from "../tools/search-text.js";
 import { WriteFileTool } from "../tools/write-file.js";
 import { EditFileTool } from "../tools/edit-file.js";
+import { ApplyPatchTool } from "../tools/apply-patch.js";
 import { BashTool } from "../tools/bash.js";
 /**
  * What each capability puts in the model's hands.
@@ -14,7 +15,7 @@ import { BashTool } from "../tools/bash.js";
  */
 const ALWAYS = ["list_files", "search_text", "read_file"];
 const BY_CAPABILITY = {
-    writes: ["write_file", "edit_file"],
+    writes: ["write_file", "edit_file", "apply_patch"],
     shell: ["bash"],
 };
 export function toolNamesFor(capabilities) {
@@ -23,17 +24,23 @@ export function toolNamesFor(capabilities) {
         ...capabilities.flatMap((capability) => BY_CAPABILITY[capability]),
     ];
 }
-export function toolsFor(capabilities, cwd) {
+export function toolsFor(capabilities, cwd, options = {}) {
     const tools = [
         new ListFilesTool(),
         new SearchTextTool(),
         new ReadFileTool(),
     ];
     if (capabilities.includes("writes")) {
-        tools.push(new WriteFileTool(), new EditFileTool());
+        tools.push(new WriteFileTool(), new EditFileTool(), new ApplyPatchTool());
     }
     if (capabilities.includes("shell")) {
-        tools.push(new BashTool({ cwd }));
+        // The CLI has already explicitly granted the shell capability to this
+        // task; Docker still enforces the host/container boundary.
+        tools.push(new BashTool({
+            cwd,
+            approve: async () => true,
+            ...(options.shellOperations === undefined ? {} : { operations: options.shellOperations }),
+        }));
     }
     return tools;
 }

package/dist/evals/tools.js.map

@@ -1 +1 @@
-{"version":3,"file":"tools.js","sourceRoot":"","sources":["../../src/evals/tools.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAG5C;;;;;;;GAOG;AACH,MAAM,MAAM,GAAG,CAAC,YAAY,EAAE,aAAa,EAAE,WAAW,CAAU,CAAC;AACnE,MAAM,aAAa,GAA0C;IAC3D,MAAM,EAAE,CAAC,YAAY,EAAE,WAAW,CAAC;IACnC,KAAK,EAAE,CAAC,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,UAAU,YAAY,CAC1B,YAAmC;IAEnC,OAAO;QACL,GAAG,MAAM;QACT,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;KACnE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,YAAmC,EACnC,GAAW;IAEX,MAAM,KAAK,GAAW;QACpB,IAAI,aAAa,EAAE;QACnB,IAAI,cAAc,EAAE;QACpB,IAAI,YAAY,EAAE;KACnB,CAAC;IACF,IAAI,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,IAAI,aAAa,EAAE,EAAE,IAAI,YAAY,EAAE,CAAC,CAAC;IACtD,CAAC;IACD,IAAI,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACnC,KAAK,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;IACpC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
+{"version":3,"file":"tools.js","sourceRoot":"","sources":["../../src/evals/tools.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAI5C;;;;;;;GAOG;AACH,MAAM,MAAM,GAAG,CAAC,YAAY,EAAE,aAAa,EAAE,WAAW,CAAU,CAAC;AACnE,MAAM,aAAa,GAA0C;IAC3D,MAAM,EAAE,CAAC,YAAY,EAAE,WAAW,EAAE,aAAa,CAAC;IAClD,KAAK,EAAE,CAAC,MAAM,CAAC;CAChB,CAAC;AAEF,MAAM,UAAU,YAAY,CAC1B,YAAmC;IAEnC,OAAO;QACL,GAAG,MAAM;QACT,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC;KACnE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,YAAmC,EACnC,GAAW,EACX,OAAO,GAAmD,EAAE;IAE5D,MAAM,KAAK,GAAW;QACpB,IAAI,aAAa,EAAE;QACnB,IAAI,cAAc,EAAE;QACpB,IAAI,YAAY,EAAE;KACnB,CAAC;IACF,IAAI,YAAY,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,IAAI,aAAa,EAAE,EAAE,IAAI,YAAY,EAAE,EAAE,IAAI,cAAc,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,IAAI,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;QACnC,sEAAsE;QACtE,2DAA2D;QAC3D,KAAK,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC;YACtB,GAAG;YACH,OAAO,EAAE,KAAK,IAAI,EAAE,CAAC,IAAI;YACzB,GAAG,CAAC,OAAO,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,OAAO,CAAC,eAAe,EAAE,CAAC;SAC1F,CAAC,CAAC,CAAC;IACN,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/events.d.ts

@@ -1,6 +1,7 @@
 import type { AssistantMessage, Message, ToolResultMessage } from "./messages.js";
 import type { Usage } from "./llm.js";
 import type { UsageTotal } from "./providers/usage.js";
+import type { PlanUpdate } from "./tools/update-plan.js";
 /**
  * Terminal state of one Agent run. `aborted` is reported, never thrown, so a
  * cancelled run stays as observable as a completed one.
@@ -14,7 +15,7 @@ export type AgentRunStatus = "completed" | "max_turns" | "aborted" | "error";
  * Every event is a plain, structured-cloneable object: renderers, loggers, and
  * a future RPC mode can serialise it directly.
  */
-export type AgentEvent = AgentStartEvent | TurnStartEvent | MessageStartEvent | MessageUpdateEvent | MessageEndEvent | ToolExecutionStartEvent | ToolExecutionUpdateEvent | ToolExecutionEndEvent | TurnEndEvent | AgentEndEvent;
+export type AgentEvent = AgentStartEvent | TurnStartEvent | MessageStartEvent | MessageUpdateEvent | MessageEndEvent | ToolExecutionStartEvent | ToolExecutionUpdateEvent | ToolExecutionEndEvent | PlanUpdateEvent | TurnEndEvent | AgentEndEvent;
 export interface AgentStartEvent {
     readonly type: "agent_start";
     readonly prompt: string;
@@ -77,6 +78,11 @@ export interface ToolExecutionEndEvent {
     readonly content: string;
     readonly isError: boolean;
 }
+/** An authoritative checklist snapshot from a successful update_plan tool. */
+export interface PlanUpdateEvent extends PlanUpdate {
+    readonly type: "plan_update";
+    readonly turn: number;
+}
 export interface TurnEndEvent {
     readonly type: "turn_end";
     readonly turn: number;