@jean.gnc/harness-kit 0.12.7 → 0.13.0

package/dist/eval/index.js

@@ -1,9 +1,11 @@
 import { defaultSources, discoverInstalled, indexInstalled } from "../installed.js";
 import { err, ok } from "../result.js";
 import { loadCases, unresolvedSkills } from "./cases.js";
+import { gradeResults } from "./grade.js";
+import { createAnthropicJudge } from "./judge.js";
 import { buildReport } from "./report.js";
 import { runCases } from "./runner.js";
-import { scoreCase } from "./score.js";
+const DEFAULT_JUDGE_MODEL = "claude-sonnet-4-5";
 export async function runEval(options) {
     const loaded = await loadCases(options.casesDir);
     if (!loaded.ok)
@@ -17,20 +19,40 @@ export async function runEval(options) {
     const unresolved = unresolvedSkills(selected, installedIds);
     if (unresolved.length > 0)
         return err(unresolved);
+    const judge = resolveJudge(selected, options);
+    if (!judge.ok)
+        return err(judge.error);
     const runnerOptions = {
         cwd: options.cwd,
         ...(options.runs !== undefined && { runs: options.runs }),
         ...(options.concurrency !== undefined && { concurrency: options.concurrency }),
         ...(options.model !== undefined && { model: options.model }),
+        ...(options.solvingTimeoutMs !== undefined && { solvingTimeoutMs: options.solvingTimeoutMs }),
         ...(options.onRun !== undefined && { onRun: options.onRun }),
     };
     const results = await runCases(selected, runnerOptions);
-    const reports = results.map(({ evalCase, runs }) => ({
-        evalCase,
-        score: scoreCase(evalCase.expect, runs, evalCase.threshold),
-    }));
+    const reports = await gradeResults(results, judge.value);
     return ok(buildReport(reports));
 }
+function needsJudge(cases) {
+    return cases.some((c) => c.tier === "solving" && c.rubric !== undefined);
+}
+function resolveJudge(cases, options) {
+    if (options.judge)
+        return ok(options.judge);
+    if (!needsJudge(cases))
+        return ok(undefined);
+    const apiKey = process.env["ANTHROPIC_API_KEY"];
+    if (!apiKey) {
+        return err([
+            {
+                file: options.casesDir,
+                message: "solving rubric grading requires ANTHROPIC_API_KEY for the judge",
+            },
+        ]);
+    }
+    return ok(createAnthropicJudge({ model: options.judgeModel ?? DEFAULT_JUDGE_MODEL, apiKey }));
+}
 function select(cases, options) {
     return cases.filter((evalCase) => {
         if (options.caseId && evalCase.id !== options.caseId)

package/dist/eval/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAuC,MAAM,YAAY,CAAC;AAC9F,OAAO,EAAE,WAAW,EAAoC,MAAM,aAAa,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAsB,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAcvC,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,OAAoB;IAChD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjD,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEzC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,oCAAoC,EAAE,CAAC,CAAC,CAAC;IAC1F,CAAC;IAED,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,iBAAiB,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC5D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC,UAAU,CAAC,CAAC;IAElD,MAAM,aAAa,GAAkB;QACnC,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;QACzD,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;QAC9E,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;QAC5D,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;KAC7D,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAExD,MAAM,OAAO,GAAiB,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACjE,QAAQ;QACR,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC;KAC5D,CAAC,CAAC,CAAC;IAEJ,OAAO,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,MAAM,CAAC,KAA4B,EAAE,OAAoB;IAChE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC/B,IAAI,OAAO,CAAC,MAAM,IAAI,QAAQ,CAAC,EAAE,KAAK,OAAO,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACnE,IAAI,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QACpE,IAAI,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACjE,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAGD,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAuC,MAAM,YAAY,CAAC;AAC9F,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,oBAAoB,EAAc,MAAM,YAAY,CAAC;AAC9D,OAAO,EAAE,WAAW,EAAmB,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,QAAQ,EAAsB,MAAM,aAAa,CAAC;AAE3D,MAAM,mBAAmB,GAAG,mBAAmB,CAAC;AAiBhD,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,OAAoB;IAChD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjD,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEzC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,oCAAoC,EAAE,CAAC,CAAC,CAAC;IAC1F,CAAC;IAED,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,iBAAiB,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC5D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC,UAAU,CAAC,CAAC;IAElD,MAAM,KAAK,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9C,IAAI,CAAC,KAAK,CAAC,EAAE;QAAE,OAAO,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEvC,MAAM,aAAa,GAAkB;QACnC,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;QACzD,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;QAC9E,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;QAC5D,GAAG,CAAC,OAAO,CAAC,gBAAgB,KAAK,SAAS,IAAI,EAAE,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAC7F,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;KAC7D,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAEzD,OAAO,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,UAAU,CAAC,KAA4B;IAC9C,OAAO,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;AAC3E,CAAC;AAED,SAAS,YAAY,CACnB,KAA4B,EAC5B,OAAoB;IAEpB,IAAI,OAAO,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5C,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC,SAAS,CAAC,CAAC;IAE7C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;IAChD,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,GAAG,CAAC;YACT;gBACE,IAAI,EAAE,OAAO,CAAC,QAAQ;gBACtB,OAAO,EAAE,iEAAiE;aAC3E;SACF,CAAC,CAAC;IACL,CAAC;IACD,OAAO,EAAE,CAAC,oBAAoB,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,UAAU,IAAI,mBAAmB,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;AAChG,CAAC;AAED,SAAS,MAAM,CAAC,KAA4B,EAAE,OAAoB;IAChE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC/B,IAAI,OAAO,CAAC,MAAM,IAAI,QAAQ,CAAC,EAAE,KAAK,OAAO,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACnE,IAAI,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QACpE,IAAI,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACjE,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAGD,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC"}

package/dist/eval/judge.d.ts

@@ -0,0 +1,26 @@
+import { z } from "zod";
+export declare const Verdict: z.ZodObject<{
+    pass: z.ZodUnion<[z.ZodBoolean, z.ZodLiteral<"unknown">]>;
+    evidence: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    pass: boolean | "unknown";
+    evidence: string;
+}, {
+    pass: boolean | "unknown";
+    evidence: string;
+}>;
+export type Verdict = z.infer<typeof Verdict>;
+export interface JudgeRequest {
+    readonly dimension: string;
+    readonly criterion: string;
+    readonly output: string;
+}
+export type Judge = (request: JudgeRequest) => Promise<Verdict>;
+export declare function parseVerdict(raw: unknown): Verdict;
+export interface AnthropicJudgeOptions {
+    readonly model: string;
+    readonly apiKey: string;
+    readonly maxTokens?: number;
+}
+export declare function createAnthropicJudge(options: AnthropicJudgeOptions): Judge;
+//# sourceMappingURL=judge.d.ts.map

package/dist/eval/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,OAAO;;;;;;;;;EAGlB,CAAC;AAEH,MAAM,MAAM,OAAO,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,OAAO,CAAC,CAAC;AAE9C,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,MAAM,KAAK,GAAG,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;AAMhE,wBAAgB,YAAY,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAIlD;AAwBD,MAAM,WAAW,qBAAqB;IACpC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,qBAAqB,GAAG,KAAK,CAmB1E"}

package/dist/eval/judge.js

@@ -0,0 +1,55 @@
+import Anthropic from "@anthropic-ai/sdk";
+import { z } from "zod";
+export const Verdict = z.object({
+    pass: z.union([z.boolean(), z.literal("unknown")]),
+    evidence: z.string(),
+});
+function abstain(evidence) {
+    return { pass: "unknown", evidence };
+}
+export function parseVerdict(raw) {
+    if (raw === undefined || raw === null)
+        return abstain("judge returned no verdict");
+    const result = Verdict.safeParse(raw);
+    return result.success ? result.data : abstain("judge returned an unparseable verdict");
+}
+const SYSTEM_PREAMBLE = "You are a strict grader for AI agent outputs. Judge the output against the single criterion " +
+    "you are given, in isolation. Record a verdict via the tool: pass=true only if the criterion is " +
+    'clearly met, pass=false if clearly unmet, and pass="unknown" if you genuinely cannot tell. ' +
+    "Always cite concrete evidence from the output.";
+const VERDICT_TOOL = {
+    name: "record_verdict",
+    description: "Record the grading verdict for one rubric dimension.",
+    input_schema: {
+        type: "object",
+        properties: {
+            pass: {
+                description: 'true if met, false if unmet, "unknown" if undeterminable',
+                anyOf: [{ type: "boolean" }, { type: "string", enum: ["unknown"] }],
+            },
+            evidence: { type: "string", description: "concrete evidence from the output" },
+        },
+        required: ["pass", "evidence"],
+    },
+};
+export function createAnthropicJudge(options) {
+    const client = new Anthropic({ apiKey: options.apiKey });
+    return async ({ dimension, criterion, output }) => {
+        const response = await client.messages.create({
+            model: options.model,
+            max_tokens: options.maxTokens ?? 1024,
+            tools: [VERDICT_TOOL],
+            tool_choice: { type: "tool", name: VERDICT_TOOL.name },
+            system: [{ type: "text", text: SYSTEM_PREAMBLE, cache_control: { type: "ephemeral" } }],
+            messages: [
+                {
+                    role: "user",
+                    content: `Dimension: ${dimension}\nCriterion: ${criterion}\n\nOutput to grade:\n${output}`,
+                },
+            ],
+        });
+        const block = response.content.find((b) => b.type === "tool_use");
+        return parseVerdict(block?.input);
+    };
+}
+//# sourceMappingURL=judge.js.map

package/dist/eval/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/eval/judge.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,mBAAmB,CAAC;AAC1C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,MAAM,CAAC,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC;IAClD,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE;CACrB,CAAC,CAAC;AAYH,SAAS,OAAO,CAAC,QAAgB;IAC/B,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AACvC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAY;IACvC,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,KAAK,IAAI;QAAE,OAAO,OAAO,CAAC,2BAA2B,CAAC,CAAC;IACnF,MAAM,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACtC,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;AACzF,CAAC;AAED,MAAM,eAAe,GACnB,8FAA8F;IAC9F,iGAAiG;IACjG,6FAA6F;IAC7F,gDAAgD,CAAC;AAEnD,MAAM,YAAY,GAAmB;IACnC,IAAI,EAAE,gBAAgB;IACtB,WAAW,EAAE,sDAAsD;IACnE,YAAY,EAAE;QACZ,IAAI,EAAE,QAAQ;QACd,UAAU,EAAE;YACV,IAAI,EAAE;gBACJ,WAAW,EAAE,0DAA0D;gBACvE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;aACpE;YACD,QAAQ,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,mCAAmC,EAAE;SAC/E;QACD,QAAQ,EAAE,CAAC,MAAM,EAAE,UAAU,CAAC;KAC/B;CACF,CAAC;AAQF,MAAM,UAAU,oBAAoB,CAAC,OAA8B;IACjE,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IACzD,OAAO,KAAK,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE;QAChD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;YAC5C,KAAK,EAAE,OAAO,CAAC,KAAK;YACpB,UAAU,EAAE,OAAO,CAAC,SAAS,IAAI,IAAI;YACrC,KAAK,EAAE,CAAC,YAAY,CAAC;YACrB,WAAW,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,YAAY,CAAC,IAAI,EAAE;YACtD,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,EAAE,aAAa,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,EAAE,CAAC;YACvF,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE,cAAc,SAAS,gBAAgB,SAAS,yBAAyB,MAAM,EAAE;iBAC3F;aACF;SACF,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC;QAClE,OAAO,YAAY,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC"}

package/dist/eval/report.d.ts

@@ -1,8 +1,12 @@
 import type { LoadedCase } from "./cases.js";
-import type { CaseScore } from "./score.js";
+import type { CaseScore, SolvingRunResult } from "./score.js";
+export interface SolvingBreakdown {
+    readonly perRun: readonly SolvingRunResult[];
+}
 export interface CaseReport {
     readonly evalCase: LoadedCase;
     readonly score: CaseScore;
+    readonly solving?: SolvingBreakdown;
 }
 export interface EvalReport {
     readonly cases: readonly CaseReport[];

package/dist/eval/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAG5C,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;CAC3B;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,GAAG,UAAU,CAGpE;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAuBxD;AAED,wBAAgB,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAyBjD"}
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AAG9D,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,MAAM,EAAE,SAAS,gBAAgB,EAAE,CAAC;CAC9C;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,gBAAgB,CAAC;CACrC;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,GAAG,UAAU,CAGpE;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAgBxD;AAED,wBAAgB,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CA2BjD"}

package/dist/eval/report.js

@@ -7,16 +7,8 @@ export function formatConsole(report) {
     const groups = groupBy(report.cases, (c) => `${c.evalCase.tier}/${c.evalCase.suite}`);
     for (const [group, entries] of groups) {
         lines.push(group);
-        for (const { evalCase, score } of entries) {
-            const tag = score.pass ? "PASS" : "FAIL";
-            const tally = `${score.matched}/${score.runs}`;
-            lines.push(`  ${tag}  ${evalCase.id}  ${tally}  → ${describeExpectation(evalCase.expect)}`);
-            if (!score.pass) {
-                lines.push(`          got: ${formatHistogram(score.histogram)}`);
-                lines.push(`          prompt: ${truncate(evalCase.prompt)}`);
-                if (evalCase.note)
-                    lines.push(`          note: ${evalCase.note}`);
-            }
+        for (const entry of entries) {
+            lines.push(...formatCase(entry));
         }
         lines.push("");
     }
@@ -32,21 +24,82 @@ export function toJson(report) {
             passed: report.passed,
             failed: report.failed,
         },
-        cases: report.cases.map(({ evalCase, score }) => ({
+        cases: report.cases.map(({ evalCase, score, solving }) => ({
             id: evalCase.id,
             suite: evalCase.suite,
             tier: evalCase.tier,
             prompt: evalCase.prompt,
-            expect: evalCase.expect,
+            ...(evalCase.tier === "routing" && { expect: evalCase.expect }),
             pass: score.pass,
             matched: score.matched,
             runs: score.runs,
             triggerRate: score.triggerRate,
             threshold: score.threshold,
-            histogram: Object.fromEntries(score.histogram),
+            ...(solving
+                ? { solving: jsonSolving(solving) }
+                : { histogram: Object.fromEntries(score.histogram) }),
         })),
     }, null, 2);
 }
+const INDENT = "          ";
+function formatCase({ evalCase, score, solving }) {
+    const tag = score.pass ? "PASS" : "FAIL";
+    const tally = `${score.matched}/${score.runs}`;
+    const lines = [`  ${tag}  ${evalCase.id}  ${tally}  → ${describeCase(evalCase)}`];
+    if (score.pass)
+        return lines;
+    if (solving)
+        lines.push(...solvingFailures(solving));
+    else
+        lines.push(`${INDENT}got: ${formatHistogram(score.histogram)}`);
+    lines.push(`${INDENT}prompt: ${truncate(evalCase.prompt)}`);
+    if (evalCase.note)
+        lines.push(`${INDENT}note: ${evalCase.note}`);
+    return lines;
+}
+function describeCase(evalCase) {
+    if (evalCase.tier === "routing")
+        return describeExpectation(evalCase.expect);
+    const parts = [`${evalCase.assert.length} assertions`];
+    if (evalCase.rubric)
+        parts.push(`${evalCase.rubric.dimensions.length} rubric dims`);
+    return parts.join(", ");
+}
+function solvingFailures(solving) {
+    const lines = [];
+    solving.perRun.forEach((run, index) => {
+        const prefix = solving.perRun.length > 1 ? `run ${index + 1} ` : "";
+        for (const a of run.assertions) {
+            if (!a.pass)
+                lines.push(`${INDENT}${prefix}assert ${a.assertion.kind}: ${a.evidence}`);
+        }
+        for (const d of run.rubric?.dimensions ?? []) {
+            if (d.verdict.pass !== true) {
+                lines.push(`${INDENT}${prefix}rubric ${d.dimension} (${d.verdict.pass}): ${d.verdict.evidence}`);
+            }
+        }
+    });
+    return lines;
+}
+function jsonSolving(solving) {
+    return solving.perRun.map((run) => ({
+        assertions: run.assertions.map((a) => ({
+            kind: a.assertion.kind,
+            pass: a.pass,
+            evidence: a.evidence,
+        })),
+        rubric: run.rubric
+            ? {
+                pass: run.rubric.pass,
+                dimensions: run.rubric.dimensions.map((d) => ({
+                    dimension: d.dimension,
+                    pass: d.verdict.pass,
+                    evidence: d.verdict.evidence,
+                })),
+            }
+            : null,
+    }));
+}
 function describeExpectation(expectation) {
     if ("noSkill" in expectation)
         return "(no skill)";

package/dist/eval/report.js.map

@@ -1 +1 @@
-{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAeA,MAAM,UAAU,WAAW,CAAC,KAA4B;IACtD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACxD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAkB;IAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAEtF,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,KAAK,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,OAAO,EAAE,CAAC;YAC1C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YACzC,MAAM,KAAK,GAAG,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,QAAQ,CAAC,EAAE,KAAK,KAAK,OAAO,mBAAmB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC5F,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAChB,KAAK,CAAC,IAAI,CAAC,kBAAkB,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBACjE,KAAK,CAAC,IAAI,CAAC,qBAAqB,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC7D,IAAI,QAAQ,CAAC,IAAI;oBAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5C,MAAM,GAAG,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;IACxE,KAAK,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,MAAM,IAAI,KAAK,kBAAkB,GAAG,KAAK,CAAC,CAAC;IACzE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,MAAkB;IACvC,OAAO,IAAI,CAAC,SAAS,CACnB;QACE,OAAO,EAAE;YACP,KAAK,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;YACpC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB;QACD,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;YAChD,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC;SAC/C,CAAC,CAAC;KACJ,EACD,IAAI,EACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,WAAwB;IACnD,IAAI,SAAS,IAAI,WAAW;QAAE,OAAO,YAAY,CAAC;IAClD,IAAI,OAAO,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC,KAAK,CAAC;IACrD,IAAI,OAAO,IAAI,WAAW;QAAE,OAAO,WAAW,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IAC9E,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,eAAe,CAAC,SAAsC;IAC7D,OAAO,CAAC,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;SAC5B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,EAAE,CAAC;SACvC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,GAAG,GAAG,EAAE;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAChD,OAAO,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;AACvE,CAAC;AAED,SAAS,OAAO,CAAI,KAAmB,EAAE,GAAwB;IAC/D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAe,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;YAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAoBA,MAAM,UAAU,WAAW,CAAC,KAA4B;IACtD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACxD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAkB;IAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAEtF,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;QACnC,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5C,MAAM,GAAG,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;IACxE,KAAK,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,MAAM,IAAI,KAAK,kBAAkB,GAAG,KAAK,CAAC,CAAC;IACzE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,MAAkB;IACvC,OAAO,IAAI,CAAC,SAAS,CACnB;QACE,OAAO,EAAE;YACP,KAAK,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;YACpC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB;QACD,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC,CAAC;YACzD,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC/D,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,GAAG,CAAC,OAAO;gBACT,CAAC,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE;gBACnC,CAAC,CAAC,EAAE,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;SACxD,CAAC,CAAC;KACJ,EACD,IAAI,EACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED,MAAM,MAAM,GAAG,YAAY,CAAC;AAE5B,SAAS,UAAU,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,OAAO,EAAc;IAC1D,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IACzC,MAAM,KAAK,GAAG,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;IAC/C,MAAM,KAAK,GAAG,CAAC,KAAK,GAAG,KAAK,QAAQ,CAAC,EAAE,KAAK,KAAK,OAAO,YAAY,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IAClF,IAAI,KAAK,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IAE7B,IAAI,OAAO;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;;QAChD,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,QAAQ,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;IACrE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,WAAW,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAC5D,IAAI,QAAQ,CAAC,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IACjE,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,YAAY,CAAC,QAAoB;IACxC,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,mBAAmB,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC7E,MAAM,KAAK,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,MAAM,aAAa,CAAC,CAAC;IACvD,IAAI,QAAQ,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,cAAc,CAAC,CAAC;IACpF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,eAAe,CAAC,OAAyB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;QACpC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QACpE,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;YAC/B,IAAI,CAAC,CAAC,CAAC,IAAI;gBAAE,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,MAAM,UAAU,CAAC,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QACzF,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,UAAU,IAAI,EAAE,EAAE,CAAC;YAC7C,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;gBAC5B,KAAK,CAAC,IAAI,CACR,GAAG,MAAM,GAAG,MAAM,UAAU,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,CACrF,CAAC;YACJ,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IACH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,OAAyB;IAC5C,OAAO,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAClC,UAAU,EAAE,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACrC,IAAI,EAAE,CAAC,CAAC,SAAS,CAAC,IAAI;YACtB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;SACrB,CAAC,CAAC;QACH,MAAM,EAAE,GAAG,CAAC,MAAM;YAChB,CAAC,CAAC;gBACE,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI;gBACrB,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC5C,SAAS,EAAE,CAAC,CAAC,SAAS;oBACtB,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI;oBACpB,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ;iBAC7B,CAAC,CAAC;aACJ;YACH,CAAC,CAAC,IAAI;KACT,CAAC,CAAC,CAAC;AACN,CAAC;AAED,SAAS,mBAAmB,CAAC,WAAwB;IACnD,IAAI,SAAS,IAAI,WAAW;QAAE,OAAO,YAAY,CAAC;IAClD,IAAI,OAAO,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC,KAAK,CAAC;IACrD,IAAI,OAAO,IAAI,WAAW;QAAE,OAAO,WAAW,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IAC9E,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,eAAe,CAAC,SAAsC;IAC7D,OAAO,CAAC,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;SAC5B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,EAAE,CAAC;SACvC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,GAAG,GAAG,EAAE;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAChD,OAAO,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;AACvE,CAAC;AAED,SAAS,OAAO,CAAI,KAAmB,EAAE,GAAwB;IAC/D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAe,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;YAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/eval/runner.d.ts

@@ -1,17 +1,25 @@
+import { type SolvingCapture } from "./capture.js";
 import { type DetectionResult } from "./detect.js";
-import type { LoadedCase } from "./cases.js";
+import type { LoadedCase, LoadedRoutingCase, LoadedSolvingCase } from "./cases.js";
 export interface RunnerOptions {
     readonly cwd: string;
     readonly runs?: number;
     readonly concurrency?: number;
     readonly timeoutMs?: number;
+    readonly solvingTimeoutMs?: number;
     readonly model?: string;
     readonly claudeBin?: string;
     readonly onRun?: (caseId: string, result: DetectionResult) => void;
+    readonly onCapture?: (caseId: string, capture: SolvingCapture) => void;
 }
-export interface CaseRuns {
-    readonly evalCase: LoadedCase;
+export type CaseResult = {
+    readonly tier: "routing";
+    readonly evalCase: LoadedRoutingCase;
     readonly runs: readonly DetectionResult[];
-}
-export declare function runCases(cases: readonly LoadedCase[], options: RunnerOptions): Promise<CaseRuns[]>;
+} | {
+    readonly tier: "solving";
+    readonly evalCase: LoadedSolvingCase;
+    readonly captures: readonly SolvingCapture[];
+};
+export declare function runCases(cases: readonly LoadedCase[], options: RunnerOptions): Promise<CaseResult[]>;
 //# sourceMappingURL=runner.d.ts.map

package/dist/eval/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAIA,OAAO,EAAkB,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AACnE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAM7C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;CACpE;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,SAAS,eAAe,EAAE,CAAC;CAC3C;AAED,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,SAAS,UAAU,EAAE,EAC5B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAgBrB"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAKA,OAAO,EAAgB,KAAK,cAAc,EAAoB,MAAM,cAAc,CAAC;AACnF,OAAO,EAAkB,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AACnE,OAAO,KAAK,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAQnF,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;IACnE,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,cAAc,KAAK,IAAI,CAAC;CACxE;AAED,MAAM,MAAM,UAAU,GAClB;IACE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,CAAC;IACrC,QAAQ,CAAC,IAAI,EAAE,SAAS,eAAe,EAAE,CAAC;CAC3C,GACD;IACE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,QAAQ,EAAE,iBAAiB,CAAC;IACrC,QAAQ,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC;CAC9C,CAAC;AAEN,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,SAAS,UAAU,EAAE,EAC5B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,UAAU,EAAE,CAAC,CA2BvB"}

package/dist/eval/runner.js

@@ -1,46 +1,74 @@
 import { spawn } from "node:child_process";
+import { readFile } from "node:fs/promises";
 import { isAbsolute, resolve } from "node:path";
 import { createInterface } from "node:readline";
+import { createCaptor } from "./capture.js";
 import { createDetector } from "./detect.js";
 const DEFAULT_RUNS = 5;
+const DEFAULT_SOLVING_RUNS = 1;
 const DEFAULT_CONCURRENCY = 1;
 const DEFAULT_TIMEOUT_MS = 60_000;
+const DEFAULT_SOLVING_TIMEOUT_MS = 300_000;
 export async function runCases(cases, options) {
-    const byCase = new Map();
+    const routing = new Map();
+    const solving = new Map();
     const jobs = cases.flatMap((evalCase) => {
-        byCase.set(evalCase, []);
-        const count = evalCase.runs ?? options.runs ?? DEFAULT_RUNS;
-        return Array.from({ length: count }, () => evalCase);
+        if (evalCase.tier === "routing")
+            routing.set(evalCase, []);
+        else
+            solving.set(evalCase, []);
+        return Array.from({ length: runsFor(evalCase, options) }, () => evalCase);
     });
     const concurrency = Math.max(1, options.concurrency ?? DEFAULT_CONCURRENCY);
     await forEachLimit(jobs, concurrency, async (evalCase) => {
-        const result = await runOnce(evalCase, options);
-        byCase.get(evalCase)?.push(result);
-        options.onRun?.(evalCase.id, result);
+        if (evalCase.tier === "routing") {
+            const result = await runRouting(evalCase, options);
+            routing.get(evalCase)?.push(result);
+            options.onRun?.(evalCase.id, result);
+        }
+        else {
+            const capture = await runSolving(evalCase, options);
+            solving.get(evalCase)?.push(capture);
+            options.onCapture?.(evalCase.id, capture);
+        }
     });
-    return cases.map((evalCase) => ({ evalCase, runs: byCase.get(evalCase) ?? [] }));
+    return cases.map((evalCase) => evalCase.tier === "routing"
+        ? { tier: "routing", evalCase, runs: routing.get(evalCase) ?? [] }
+        : { tier: "solving", evalCase, captures: solving.get(evalCase) ?? [] });
+}
+function runsFor(evalCase, options) {
+    const fallback = evalCase.tier === "solving" ? DEFAULT_SOLVING_RUNS : DEFAULT_RUNS;
+    return evalCase.runs ?? options.runs ?? fallback;
 }
 function skillsToCollect(evalCase) {
     return "path" in evalCase.expect ? evalCase.expect.path.length : 1;
 }
-async function runOnce(evalCase, options) {
-    const cwd = evalCase.cwd ? resolveCwd(options.cwd, evalCase.cwd) : options.cwd;
-    const args = [
-        "-p",
-        evalCase.prompt,
-        "--output-format",
-        "stream-json",
-        "--verbose",
-        "--include-partial-messages",
-    ];
-    if (options.model)
-        args.push("--model", options.model);
-    const env = { ...process.env };
-    delete env["CLAUDECODE"];
+async function runRouting(evalCase, options) {
     const detector = createDetector(skillsToCollect(evalCase));
-    const child = spawn(options.claudeBin ?? "claude", args, {
+    const reached = await runSession(evalCase, options, {
+        timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+        onLine: (line) => detector.push(line),
+        done: () => detector.done,
+        killOnDone: true,
+    });
+    return detector.result(reached ? "timeout" : "no-skill");
+}
+async function runSolving(evalCase, options) {
+    const captor = createCaptor();
+    const reached = await runSession(evalCase, options, {
+        timeoutMs: options.solvingTimeoutMs ?? DEFAULT_SOLVING_TIMEOUT_MS,
+        onLine: (line) => captor.push(line),
+        done: () => captor.done,
+        killOnDone: false,
+    });
+    const capture = captor.result(reached ? "timeout" : "stream-end");
+    return mergeDiskWrites(capture, evalCase, options.cwd);
+}
+async function runSession(evalCase, options, handlers) {
+    const cwd = evalCase.cwd ? resolveCwd(options.cwd, evalCase.cwd) : options.cwd;
+    const child = spawn(options.claudeBin ?? "claude", buildArgs(evalCase.prompt, options.model), {
         cwd,
-        env,
+        env: scrubbedEnv(),
         stdio: ["ignore", "pipe", "ignore"],
     });
     const spawnFailure = new Promise((_, reject) => {
@@ -50,27 +78,73 @@ async function runOnce(evalCase, options) {
     const timer = setTimeout(() => {
         deadline.reached = true;
         child.kill("SIGKILL");
-    }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+    }, handlers.timeoutMs);
+    const killOnDone = handlers.killOnDone ? () => child.kill("SIGKILL") : undefined;
     try {
-        await Promise.race([drain(child.stdout, () => child.kill("SIGKILL"), detector), spawnFailure]);
+        await Promise.race([
+            drain(child.stdout, handlers.onLine, handlers.done, killOnDone),
+            spawnFailure,
+        ]);
     }
     finally {
         clearTimeout(timer);
         if (child.exitCode === null)
             child.kill("SIGKILL");
     }
-    return detector.result(deadline.reached ? "timeout" : "no-skill");
+    return deadline.reached;
 }
-async function drain(stdout, stop, detector) {
+function buildArgs(prompt, model) {
+    const args = [
+        "-p",
+        prompt,
+        "--output-format",
+        "stream-json",
+        "--verbose",
+        "--include-partial-messages",
+    ];
+    if (model)
+        args.push("--model", model);
+    return args;
+}
+function scrubbedEnv() {
+    const env = { ...process.env };
+    delete env["CLAUDECODE"];
+    return env;
+}
+async function drain(stdout, onLine, done, killOnDone) {
     const lines = createInterface({ input: stdout });
     for await (const line of lines) {
-        detector.push(line);
-        if (detector.done) {
-            stop();
+        onLine(line);
+        if (done()) {
+            killOnDone?.();
             break;
         }
     }
 }
+async function mergeDiskWrites(capture, evalCase, baseCwd) {
+    const declared = declaredWritePaths(evalCase);
+    if (declared.length === 0)
+        return capture;
+    const cwd = evalCase.cwd ? resolveCwd(baseCwd, evalCase.cwd) : baseCwd;
+    const writes = new Map(capture.writes.map((w) => [w.path, w]));
+    for (const path of declared) {
+        const onDisk = await readFileOrNull(resolveCwd(cwd, path));
+        if (onDisk !== null)
+            writes.set(path, { path, content: onDisk });
+    }
+    return { ...capture, writes: [...writes.values()] };
+}
+function declaredWritePaths(evalCase) {
+    return evalCase.assert.flatMap((a) => (a.kind === "wroteFile" ? [a.path] : []));
+}
+async function readFileOrNull(path) {
+    try {
+        return await readFile(path, "utf8");
+    }
+    catch {
+        return null;
+    }
+}
 function resolveCwd(base, caseCwd) {
     return isAbsolute(caseCwd) ? caseCwd : resolve(base, caseCwd);
 }

package/dist/eval/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,cAAc,EAAwB,MAAM,aAAa,CAAC;AAGnE,MAAM,YAAY,GAAG,CAAC,CAAC;AACvB,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAiBlC,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAA4B,EAC5B,OAAsB;IAEtB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAiC,CAAC;IACxD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QACtC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,IAAI,YAAY,CAAC;QAC5D,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,mBAAmB,CAAC,CAAC;IAC5E,MAAM,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;QACvD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACnC,OAAO,CAAC,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,SAAS,eAAe,CAAC,QAAoB;IAC3C,OAAO,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,QAAoB,EAAE,OAAsB;IACjE,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAC/E,MAAM,IAAI,GAAG;QACX,IAAI;QACJ,QAAQ,CAAC,MAAM;QACf,iBAAiB;QACjB,aAAa;QACb,WAAW;QACX,4BAA4B;KAC7B,CAAC;IACF,IAAI,OAAO,CAAC,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAEvD,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC/B,OAAO,GAAG,CAAC,YAAY,CAAC,CAAC;IAEzB,MAAM,QAAQ,GAAG,cAAc,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,QAAQ,EAAE,IAAI,EAAE;QACvD,GAAG;QACH,GAAG;QACH,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC;KACpC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;QACpD,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;QAC5B,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxB,CAAC,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;IAE5C,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;IACjG,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;AACpE,CAAC;AAED,KAAK,UAAU,KAAK,CAClB,MAA6B,EAC7B,IAAgB,EAChB,QAA2C;IAE3C,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC/B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClB,IAAI,EAAE,CAAC;YACP,MAAM;QACR,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,OAAe;IAC/C,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,KAAmB,EACnB,KAAa,EACb,MAAkC;IAElC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC/E,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YAC3B,MAAM,IAAI,CAAC,CAAC;YACZ,IAAI,IAAI,KAAK,SAAS;gBAAE,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC"}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,YAAY,EAAyC,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,cAAc,EAAwB,MAAM,aAAa,CAAC;AAGnE,MAAM,YAAY,GAAG,CAAC,CAAC;AACvB,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAC/B,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,0BAA0B,GAAG,OAAO,CAAC;AA0B3C,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAA4B,EAC5B,OAAsB;IAEtB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAC;IACzD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAgC,CAAC;IACxD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QACtC,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS;YAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;;YACtD,OAAO,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAC/B,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,mBAAmB,CAAC,CAAC;IAC5E,MAAM,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;QACvD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACpC,OAAO,CAAC,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YACpD,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACrC,OAAO,CAAC,SAAS,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAC5B,QAAQ,CAAC,IAAI,KAAK,SAAS;QACzB,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE;QAClE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CACzE,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,QAAoB,EAAE,OAAsB;IAC3D,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,YAAY,CAAC;IACnF,OAAO,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC;AACnD,CAAC;AAED,SAAS,eAAe,CAAC,QAA2B;IAClD,OAAO,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,QAA2B,EAC3B,OAAsB;IAEtB,MAAM,QAAQ,GAAG,cAAc,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE;QAClD,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB;QAClD,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;QACrC,IAAI,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI;QACzB,UAAU,EAAE,IAAI;KACjB,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;AAC3D,CAAC;AAED,KAAK,UAAU,UAAU,CACvB,QAA2B,EAC3B,OAAsB;IAEtB,MAAM,MAAM,GAAG,YAAY,EAAE,CAAC;IAC9B,MAAM,OAAO,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,OAAO,EAAE;QAClD,SAAS,EAAE,OAAO,CAAC,gBAAgB,IAAI,0BAA0B;QACjE,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;QACnC,IAAI,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI;QACvB,UAAU,EAAE,KAAK;KAClB,CAAC,CAAC;IACH,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;IAClE,OAAO,eAAe,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC;AACzD,CAAC;AASD,KAAK,UAAU,UAAU,CACvB,QAAoB,EACpB,OAAsB,EACtB,QAAyB;IAEzB,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAC/E,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,QAAQ,EAAE,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,CAAC,EAAE;QAC5F,GAAG;QACH,GAAG,EAAE,WAAW,EAAE;QAClB,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC;KACpC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;QACpD,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;QAC5B,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxB,CAAC,EAAE,QAAQ,CAAC,SAAS,CAAC,CAAC;IAEvB,MAAM,UAAU,GAAG,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IACjF,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,CAAC;YACjB,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;YAC/D,YAAY;SACb,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,QAAQ,CAAC,OAAO,CAAC;AAC1B,CAAC;AAED,SAAS,SAAS,CAAC,MAAc,EAAE,KAAyB;IAC1D,MAAM,IAAI,GAAG;QACX,IAAI;QACJ,MAAM;QACN,iBAAiB;QACjB,aAAa;QACb,WAAW;QACX,4BAA4B;KAC7B,CAAC;IACF,IAAI,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACvC,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,WAAW;IAClB,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC/B,OAAO,GAAG,CAAC,YAAY,CAAC,CAAC;IACzB,OAAO,GAAG,CAAC;AACb,CAAC;AAED,KAAK,UAAU,KAAK,CAClB,MAA6B,EAC7B,MAA8B,EAC9B,IAAmB,EACnB,UAAuB;IAEvB,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC,CAAC;QACb,IAAI,IAAI,EAAE,EAAE,CAAC;YACX,UAAU,EAAE,EAAE,CAAC;YACf,MAAM;QACR,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,OAAuB,EACvB,QAA2B,EAC3B,OAAe;IAEf,MAAM,QAAQ,GAAG,kBAAkB,CAAC,QAAQ,CAAC,CAAC;IAC9C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IAE1C,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;IACvE,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC/D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;QAC3D,IAAI,MAAM,KAAK,IAAI;YAAE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,EAAE,GAAG,OAAO,EAAE,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAA2B,EAAE,CAAC;AAChF,CAAC;AAED,SAAS,kBAAkB,CAAC,QAA2B;IACrD,OAAO,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAClF,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,IAAY;IACxC,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,OAAe;IAC/C,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,KAAmB,EACnB,KAAa,EACb,MAAkC;IAElC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC/E,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YAC3B,MAAM,IAAI,CAAC,CAAC;YACZ,IAAI,IAAI,KAAK,SAAS;gBAAE,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC"}