@namzu/sdk 62.12.0 → 62.13.0

package/dist/eval/report-stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-stats.d.ts","sourceRoot":"","sources":["../../src/eval/report-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAE,KAAK,SAAS,EAAkB,MAAM,gBAAgB,CAAA;AACnF,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE1E;;;;;;;GAOG;AAEH,MAAM,WAAW,iBAAiB;IACjC,sBAAsB;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,uCAAuC;IACvC,iBAAiB,EAAE,MAAM,CAAA;IACzB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,SAAS,EAAE,aAAa,CAAA;CACxB;AAED,qFAAqF;AACrF,MAAM,WAAW,QAAQ;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAA;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAA;CACpC;AAED,MAAM,WAAW,QAAQ;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,SAAS,EAAE,aAAa,CAAA;IACxB,aAAa,EAAE,iBAAiB,EAAE,CAAA;IAClC,WAAW,EAAE,iBAAiB,CAAA;IAC9B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,YAAY,EAAE,YAAY,CAAC,CAAC,CAAA;CAC/D;AAED,eAAO,MAAM,WAAW,oDAAqD,CAAA;AAE7E,mFAAmF;AACnF,wBAAgB,SAAS,CACxB,OAAO,EAAE,SAAS,UAAU,EAAE,EAC9B,KAAK,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAuB,GAC9C,QAAQ,EAAE,CAYZ;AAED,gEAAgE;AAChE,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,CAO1E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAC3B,IAAI,EAAE,SAAS,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,QAAQ,CAAC,EAAE,GAClD,MAAM,GAAG,SAAS,CAQpB;AAED,8EAA8E;AAC9E,wBAAgB,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,GAAG,QAAQ,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAKzF;AAED,wBAAgB,OAAO,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,GAAG,SAAS,GAAG,IAAI,CAYrE;AAED,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,GAAG,OAAO,CAEnE;AAmCD,wBAAgB,SAAS,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,QAAQ,CAmC5E;AAKD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACvC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,SAAS,EAAE,aAAa,CAAA;CACxB,GAAG,MAAM,EAAE,CAsBX;AAED,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE,CAmBnE"}

package/dist/eval/report-stats.js

@@ -0,0 +1,184 @@
+import { bootstrapCases } from './bootstrap.js';
+export const SPLIT_ORDER = ['development', 'held-out', 'unassigned'];
+/** Group a flat list of runs by case name, in the order the cases first appear. */
+export function groupRuns(results, keyOf = (name) => name) {
+    const groups = new Map();
+    for (const run of results) {
+        const key = keyOf(run.case);
+        let group = groups.get(key);
+        if (!group) {
+            group = { name: run.case, ...(run.split ? { split: run.split } : {}), runs: [] };
+            groups.set(key, group);
+        }
+        group.runs.push(run);
+    }
+    return [...groups].map(([key, g]) => ({ key, ...g }));
+}
+/** The first name that appears more than once, or undefined. */
+export function duplicateName(names) {
+    const seen = new Set();
+    for (const name of names) {
+        if (seen.has(name))
+            return name;
+        seen.add(name);
+    }
+    return undefined;
+}
+export function duplicateMessage(suite, name) {
+    return `suite "${suite}" has the case "${name}" more than once; a case is named by its id, and two cases sharing one would be merged into one in every count and interval. Give each case its own name.`;
+}
+/**
+ * The first case in a list of runs that appears twice for the same repeat: a
+ * second run of a case is a repeat only when it says it is a later one.
+ */
+export function duplicateRun(runs) {
+    const seen = new Set();
+    for (const run of runs) {
+        const key = `${run.case}\u0000${run.repeat ?? 1}`;
+        if (seen.has(key))
+            return run.case;
+        seen.add(key);
+    }
+    return undefined;
+}
+/** Status of a run, tolerating a report old enough to carry only `passed`. */
+export function statusOf(run) {
+    if (run.status === 'passed' || run.status === 'failed' || run.status === 'inconclusive') {
+        return run.status;
+    }
+    return run.passed === true ? 'passed' : 'failed';
+}
+export function tallyOf(runs) {
+    let scoreSum = 0;
+    let passes = 0;
+    let judged = 0;
+    for (const run of runs) {
+        const status = statusOf(run);
+        if (status === 'inconclusive')
+            continue;
+        judged++;
+        scoreSum += run.mean;
+        if (status === 'passed')
+            passes++;
+    }
+    return judged === 0 ? null : { scoreSum, passes, judged };
+}
+/** True when the case has at least one judged run and every run passed. */
+export function passedEveryRun(runs) {
+    return runs.length > 0 && runs.every((run) => statusOf(run) === 'passed');
+}
+function summaryOf(runs) {
+    let passed = 0;
+    let failed = 0;
+    let inconclusive = 0;
+    for (const run of runs) {
+        const status = statusOf(run);
+        if (status === 'passed')
+            passed++;
+        else if (status === 'failed')
+            failed++;
+        else
+            inconclusive++;
+    }
+    return { passed, failed, inconclusive };
+}
+function numbersOver(cases, seed) {
+    const tallies = [];
+    let runs = 0;
+    for (const c of cases) {
+        runs += c.runs.length;
+        const tally = tallyOf(c.runs);
+        if (tally)
+            tallies.push(tally);
+    }
+    const counts = summaryOf(cases.flatMap((c) => c.runs));
+    const judged = counts.passed + counts.failed;
+    const scoreSum = tallies.reduce((sum, t) => sum + t.scoreSum, 0);
+    return {
+        runs,
+        ...counts,
+        passRate: judged === 0 ? null : counts.passed / judged,
+        mean: judged === 0 ? null : scoreSum / judged,
+        intervals: bootstrapCases(tallies, seed),
+    };
+}
+export function statsOver(cases, seed) {
+    const whole = numbersOver(cases, seed);
+    const caseSummaries = cases.map((c) => {
+        const counts = summaryOf(c.runs);
+        const tally = tallyOf(c.runs);
+        return {
+            case: c.name,
+            ...(c.split ? { split: c.split } : {}),
+            repeats: c.runs.length,
+            ...counts,
+            mean: tally ? tally.scoreSum / tally.judged : null,
+            passedEveryRepeat: passedEveryRun(c.runs),
+        };
+    });
+    const everyRepeat = caseSummaries.filter((c) => c.passedEveryRepeat).length;
+    const splits = {};
+    for (const name of SPLIT_ORDER) {
+        const members = cases.filter((c) => (c.split ?? 'unassigned') === name);
+        if (members.length === 0)
+            continue;
+        const part = numbersOver(members, seed);
+        splits[name] = { cases: members.length, ...part };
+    }
+    return {
+        cases: cases.length,
+        ...whole,
+        caseSummaries,
+        reliability: {
+            cases: cases.length,
+            passedEveryRepeat: everyRepeat,
+            fraction: cases.length === 0 ? null : everyRepeat / cases.length,
+        },
+        splits,
+    };
+}
+const percent = (value) => `${(value * 100).toFixed(1)}%`;
+const score = (value) => value.toFixed(3);
+/**
+ * The intervals, as lines a person reads next to the numbers. A number whose
+ * interval is not computed is printed with the reason, never alone.
+ */
+export function describeIntervals(args) {
+    const { intervals } = args;
+    const line = (label, estimate, interval, show) => {
+        if (estimate === null)
+            return `${label}: none, no run was judged`;
+        if (!interval.computed)
+            return `${label} ${show(estimate)}, no interval: ${interval.reason}`;
+        return `${label} ${show(estimate)}, 95% interval ${show(interval.low)} to ${show(interval.high)}`;
+    };
+    const lines = [
+        line('mean score', args.mean, intervals.mean, score),
+        line('pass rate', args.passRate, intervals.passRate, percent),
+    ];
+    if (intervals.mean.computed || intervals.passRate.computed) {
+        lines.push(`intervals: ${intervals.method.kind} (${intervals.method.cases} cases, ${intervals.method.resamples} resamples), seed ${intervals.method.seed}`);
+    }
+    return lines;
+}
+/** The line that says which splits a suite has, and says so plainly when none is held out. */
+export function describeSplits(splits) {
+    const lines = [];
+    // A suite that declares no split has one bucket and nothing to break down.
+    const declared = SPLIT_ORDER.some((name) => name !== 'unassigned' && splits[name]);
+    for (const name of SPLIT_ORDER) {
+        const part = splits[name];
+        if (!part || !declared)
+            continue;
+        const rate = part.passRate === null ? 'no judged run' : `pass rate ${percent(part.passRate)}`;
+        const interval = part.intervals.passRate.computed
+            ? `, 95% interval ${percent(part.intervals.passRate.low)} to ${percent(part.intervals.passRate.high)}`
+            : '';
+        lines.push(`${name}: ${part.cases} case${part.cases === 1 ? '' : 's'}, ${rate}${interval}`);
+    }
+    if (!splits['held-out']) {
+        lines.push('no held-out cases: every number here is measured on cases the suite was developed against');
+    }
+    return lines;
+}
+//# sourceMappingURL=report-stats.js.map

package/dist/eval/report-stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-stats.js","sourceRoot":"","sources":["../../src/eval/report-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsC,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAsDnF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAU,CAAA;AAE7E,mFAAmF;AACnF,MAAM,UAAU,SAAS,CACxB,OAA8B,EAC9B,QAAkC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI;IAEhD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmE,CAAA;IACzF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC3B,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,KAAK,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;YAChF,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACrB,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;AACtD,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,aAAa,CAAC,KAAwB;IACrD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;QAC/B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACf,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,IAAY;IAC3D,OAAO,UAAU,KAAK,mBAAmB,IAAI,2JAA2J,CAAA;AACzM,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC3B,IAAoD;IAEpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAA;QACjD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAA;QAClC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACd,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,QAAQ,CAAC,GAA0C;IAClE,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;QACzF,OAAO,GAAG,CAAC,MAAM,CAAA;IAClB,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAA2B;IAClD,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,KAAK,cAAc;YAAE,SAAQ;QACvC,MAAM,EAAE,CAAA;QACR,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAA;QACpB,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;IAClC,CAAC;IACD,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;AAC1D,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,cAAc,CAAC,IAA2B;IACzD,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,SAAS,CAAC,IAA2B;IAC7C,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;aAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;;YACjC,YAAY,EAAE,CAAA;IACpB,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,CAAA;AACxC,CAAC;AAED,SAAS,WAAW,CAAC,KAA0B,EAAE,IAAY;IAC5D,MAAM,OAAO,GAAgB,EAAE,CAAA;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAA;QACrB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC7B,IAAI,KAAK;YAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC/B,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;IAChE,OAAO;QACN,IAAI;QACJ,GAAG,MAAM;QACT,QAAQ,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM;QACtD,IAAI,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,GAAG,MAAM;QAC7C,SAAS,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC;KACxC,CAAA;AACF,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAA0B,EAAE,IAAY;IACjE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACtC,MAAM,aAAa,GAAwB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1D,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC7B,OAAO;YACN,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;YACtB,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAClD,iBAAiB,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;SACzC,CAAA;IACF,CAAC,CAAC,CAAA;IACF,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAA;IAE3E,MAAM,MAAM,GAAuB,EAAE,CAAA;IACrC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK,IAAI,CAAC,CAAA;QACvE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAClC,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QACvC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,EAAE,CAAA;IAClD,CAAC;IAED,OAAO;QACN,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,GAAG,KAAK;QACR,aAAa;QACb,WAAW,EAAE;YACZ,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,iBAAiB,EAAE,WAAW;YAC9B,QAAQ,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM;SAChE;QACD,MAAM;KACN,CAAA;AACF,CAAC;AAED,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AACzE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAEzD;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAIjC;IACA,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAA;IAC1B,MAAM,IAAI,GAAG,CACZ,KAAa,EACb,QAAuB,EACvB,QAA+B,EAC/B,IAA2B,EAClB,EAAE;QACX,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,GAAG,KAAK,2BAA2B,CAAA;QACjE,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAAE,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,QAAQ,CAAC,MAAM,EAAE,CAAA;QAC5F,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAA;IAClG,CAAC,CAAA;IACD,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;QACpD,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC;KAC7D,CAAA;IACD,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CACT,cAAc,SAAS,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,MAAM,CAAC,KAAK,WAAW,SAAS,CAAC,MAAM,CAAC,SAAS,qBAAqB,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAC/I,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,MAA0B;IACxD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;IAClF,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ;YAAE,SAAQ;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAA;QAC7F,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ;YAChD,CAAC,CAAC,kBAAkB,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACtG,CAAC,CAAC,EAAE,CAAA;QACL,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAA;IAC5F,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CACT,2FAA2F,CAC3F,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC"}

package/dist/eval/run-options.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Settings an operator gives a whole eval run, handed to every
+ * `runExperiment` that executes inside it.
+ *
+ * A suite file calls `runExperiment` itself and returns the report, so
+ * `namzu eval --repeat 5` has no argument to pass the five through. Rather
+ * than ask every suite to forward it, the runner opens a scope with
+ * {@link withEvalRunOptions} and `runExperiment` reads the scope. A suite
+ * that was written before the option existed repeats like any other.
+ *
+ * An option given here wins over the same option in the suite's own config:
+ * the person running the command asked for it.
+ */
+export interface EvalRunOptions {
+    /** Times every case runs. */
+    readonly repeats?: number;
+    /** Seed for the bootstrap. */
+    readonly seed?: number;
+    /** Version of the program that is running the suites, for the disclosure. */
+    readonly harnessVersion?: string;
+    /** Source revision the suites came from, when known. */
+    readonly commit?: string;
+    /** Version of the suite being run, when known. */
+    readonly suiteVersion?: string;
+}
+/** Run `work` with `options` in force for every experiment it starts. */
+export declare function withEvalRunOptions<T>(options: EvalRunOptions, work: () => T): T;
+/** The options in force here, or an empty set outside any scope. */
+export declare function currentEvalRunOptions(): EvalRunOptions;
+//# sourceMappingURL=run-options.d.ts.map

package/dist/eval/run-options.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-options.d.ts","sourceRoot":"","sources":["../../src/eval/run-options.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC9B,6BAA6B;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,8BAA8B;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,6EAA6E;IAC7E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,wDAAwD;IACxD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,kDAAkD;IAClD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC9B;AAcD,yEAAyE;AACzE,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAE/E;AAED,oEAAoE;AACpE,wBAAgB,qBAAqB,IAAI,cAAc,CAEtD"}

package/dist/eval/run-options.js

@@ -0,0 +1,20 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+const STORE_KEY = Symbol.for('namzu.eval.runOptions');
+/**
+ * One store per process, found through a global symbol, so a suite that
+ * resolves its own copy of the SDK still sees the scope the runner opened.
+ */
+function store() {
+    const holder = globalThis;
+    holder[STORE_KEY] ??= new AsyncLocalStorage();
+    return holder[STORE_KEY];
+}
+/** Run `work` with `options` in force for every experiment it starts. */
+export function withEvalRunOptions(options, work) {
+    return store().run(options, work);
+}
+/** The options in force here, or an empty set outside any scope. */
+export function currentEvalRunOptions() {
+    return store().getStore() ?? {};
+}
+//# sourceMappingURL=run-options.js.map

package/dist/eval/run-options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-options.js","sourceRoot":"","sources":["../../src/eval/run-options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AA4BpD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAA;AAErD;;;GAGG;AACH,SAAS,KAAK;IACb,MAAM,MAAM,GAAG,UAAiE,CAAA;IAChF,MAAM,CAAC,SAAS,CAAC,KAAK,IAAI,iBAAiB,EAAkB,CAAA;IAC7D,OAAO,MAAM,CAAC,SAAS,CAAC,CAAA;AACzB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,kBAAkB,CAAI,OAAuB,EAAE,IAAa;IAC3E,OAAO,KAAK,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;AAClC,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,qBAAqB;IACpC,OAAO,KAAK,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAA;AAChC,CAAC"}

package/dist/eval/run-summary.d.ts

@@ -0,0 +1,36 @@
+import type { RunMode } from './disclosure.js';
+import { type RunStats } from './report-stats.js';
+import type { ExperimentReport } from './types.js';
+/** What produced a whole run, across every suite in it. */
+export interface RunDisclosure {
+    readonly harnessVersion: string;
+    readonly sdkVersion: string;
+    readonly generatedAt: string;
+    readonly commit: string;
+    readonly seed: number;
+    /** Repeats per case; the largest across suites when they differ. */
+    readonly repeats: number;
+    readonly suites: number;
+    readonly cases: number;
+    readonly runs: number;
+    /** `mixed` when suites that used a model differ in kind; a suite with no model at all does not count. */
+    readonly mode: RunMode | 'mixed';
+}
+/** The numbers for every case of every suite taken together. */
+export interface RunSummary extends Omit<RunStats, 'caseSummaries' | 'reliability'> {
+    readonly disclosure: RunDisclosure;
+    /** Present only when cases were repeated. */
+    readonly reliability?: RunStats['reliability'];
+}
+export declare function summarizeRun(entries: ReadonlyArray<{
+    suite: string;
+    report: ExperimentReport;
+}>, context: {
+    seed: number;
+    harnessVersion: string;
+    commit: string;
+    generatedAt: string;
+    sdkVersion: string;
+}): RunSummary;
+export declare function formatRunSummary(summary: RunSummary): string;
+//# sourceMappingURL=run-summary.d.ts.map

package/dist/eval/run-summary.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-summary.d.ts","sourceRoot":"","sources":["../../src/eval/run-summary.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EACN,KAAK,QAAQ,EAKb,MAAM,mBAAmB,CAAA;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAElD,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yGAAyG;IACzG,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAAA;CAChC;AAED,gEAAgE;AAChE,MAAM,WAAW,UAAW,SAAQ,IAAI,CAAC,QAAQ,EAAE,eAAe,GAAG,aAAa,CAAC;IAClF,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAA;IAClC,6CAA6C;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAA;CAC9C;AAED,wBAAgB,YAAY,CAC3B,OAAO,EAAE,aAAa,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,CAAC,EACnE,OAAO,EAAE;IACR,IAAI,EAAE,MAAM,CAAA;IACZ,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;CAClB,GACC,UAAU,CAmCZ;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM,CAuB5D"}

package/dist/eval/run-summary.js

@@ -0,0 +1,57 @@
+import { UNKNOWN } from './disclosure.js';
+import { describeIntervals, describeSplits, groupRuns, statsOver, } from './report-stats.js';
+export function summarizeRun(entries, context) {
+    // A case is named by its suite and its id: two suites may reuse a name.
+    const cases = entries.flatMap(({ suite, report }) => groupRuns(report.cases ?? [], (name) => `${suite}\u0000${name}`));
+    const { caseSummaries: _perCase, reliability, ...stats } = statsOver(cases, context.seed);
+    const repeats = Math.max(1, ...entries.map((e) => e.report.repeats ?? 1));
+    // A suite that used no model at all says nothing about whether a model
+    // ran, so it does not make a run "mixed" with the suites that did.
+    const all = entries.map((e) => e.report.disclosure?.mode ?? UNKNOWN);
+    const modes = new Set(all.filter((m) => m !== 'no-model'));
+    const mode = modes.size === 0
+        ? all.length === 0
+            ? UNKNOWN
+            : 'no-model'
+        : modes.size === 1
+            ? [...modes][0]
+            : 'mixed';
+    return {
+        ...stats,
+        ...(repeats > 1 ? { reliability } : {}),
+        disclosure: {
+            harnessVersion: context.harnessVersion,
+            sdkVersion: context.sdkVersion,
+            generatedAt: context.generatedAt,
+            commit: context.commit,
+            seed: context.seed,
+            repeats,
+            suites: entries.length,
+            cases: stats.cases,
+            runs: stats.runs,
+            mode,
+        },
+    };
+}
+export function formatRunSummary(summary) {
+    const d = summary.disclosure;
+    const lines = [
+        `Whole run: ${summary.passed}/${summary.runs} runs passed across ${d.suites} suite${d.suites === 1 ? '' : 's'}, ${summary.cases} case${summary.cases === 1 ? '' : 's'}${d.repeats > 1 ? ` x ${d.repeats} repeats` : ''}`,
+    ];
+    for (const line of describeIntervals({
+        mean: summary.mean,
+        passRate: summary.passRate,
+        intervals: summary.intervals,
+    })) {
+        lines.push(`  ${line}`);
+    }
+    if (summary.reliability) {
+        const r = summary.reliability;
+        lines.push(`  reliability across repeats: ${r.passedEveryRepeat} of ${r.cases} cases passed in every one of ${d.repeats} repeats${r.fraction === null ? '' : ` (${(r.fraction * 100).toFixed(1)}%)`}`);
+    }
+    for (const line of describeSplits(summary.splits))
+        lines.push(`  ${line}`);
+    lines.push(`  produced by: namzu eval ${d.harnessVersion}, sdk ${d.sdkVersion}, commit ${d.commit}, ${d.generatedAt}, ${d.mode}`);
+    return lines.join('\n');
+}
+//# sourceMappingURL=run-summary.js.map

package/dist/eval/run-summary.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-summary.js","sourceRoot":"","sources":["../../src/eval/run-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAEzC,OAAO,EAEN,iBAAiB,EACjB,cAAc,EACd,SAAS,EACT,SAAS,GACT,MAAM,mBAAmB,CAAA;AA0B1B,MAAM,UAAU,YAAY,CAC3B,OAAmE,EACnE,OAMC;IAED,wEAAwE;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CACnD,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,KAAK,SAAS,IAAI,EAAE,CAAC,CAChE,CAAA;IACD,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,GAAG,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;IACzF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAA;IACzE,uEAAuE;IACvE,mEAAmE;IACnE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,IAAI,OAAO,CAAC,CAAA;IACpE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAA;IAC1D,MAAM,IAAI,GACT,KAAK,CAAC,IAAI,KAAK,CAAC;QACf,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC;YACjB,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,UAAU;QACb,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;YACjB,CAAC,CAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAa;YAC5B,CAAC,CAAC,OAAO,CAAA;IACZ,OAAO;QACN,GAAG,KAAK;QACR,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,UAAU,EAAE;YACX,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO;YACP,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI;SACJ;KACD,CAAA;AACF,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAmB;IACnD,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,CAAA;IAC5B,MAAM,KAAK,GAAG;QACb,cAAc,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,uBAAuB,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,KAAK,QAAQ,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE;KACxN,CAAA;IACD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;QACpC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC5B,CAAC,EAAE,CAAC;QACJ,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IACxB,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,OAAO,CAAC,WAAW,CAAA;QAC7B,KAAK,CAAC,IAAI,CACT,iCAAiC,CAAC,CAAC,iBAAiB,OAAO,CAAC,CAAC,KAAK,iCAAiC,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAC1L,CAAA;IACF,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IAC1E,KAAK,CAAC,IAAI,CACT,6BAA6B,CAAC,CAAC,cAAc,SAAS,CAAC,CAAC,UAAU,YAAY,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,IAAI,EAAE,CACrH,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/scorers.d.ts

@@ -1,3 +1,5 @@
+import type { HandBackKind } from '../types/session/hand-back.js';
+import type { TurnOutcomeVerdict } from '../types/session/outcome.js';
 import type { EvalCase, EvalTurn, Score, Scorer } from './types.js';
 /**
  * How closely the turn's tool sequence matched the expected one, as F1 over
@@ -27,6 +29,36 @@ export declare function completionScorer(acceptable?: readonly string[]): Scorer
 export declare function stepBudgetScorer(maxSteps: number): Scorer;
 /** The final text contains every required substring. */
 export declare function containsScorer(...required: string[]): Scorer;
+/**
+ * The turn's outcome record has one of the given verdicts.
+ *
+ * Reads the record the kernel settled from what actually ran, not the answer's
+ * words. A turn with no record scores unavailable: there is nothing to read,
+ * which is not the same as a wrong verdict.
+ */
+export declare function outcomeScorer(verdicts: readonly TurnOutcomeVerdict[]): Scorer;
+/**
+ * Whether the answer's plain check claims agree with the turn's outcome
+ * record: 1 when `compareAnswerToOutcome` found no mismatch, 0 when it found
+ * one, with the claim and what the record holds in the detail.
+ *
+ * Pass `{ expectMismatch: true }` for a case that is built to contradict
+ * itself, so the suite proves the comparison fires as well as that it stays
+ * silent.
+ */
+export declare function claimMismatchScorer(opts?: {
+    expectMismatch?: boolean;
+}): Scorer;
+/**
+ * Score whether the turn was handed back to a person.
+ *
+ * Pass `{ expectKind }` for a case built so the model hands the task back, and
+ * the scorer checks it did, and for that kind; with nothing it checks the turn
+ * was not handed back. A gate: a scripted suite pins the kernel's side of it.
+ */
+export declare function handedBackScorer(opts?: {
+    expectKind?: HandBackKind;
+}): Scorer;
 /**
  * Judge the turn with a caller-supplied predicate.
  *

package/dist/eval/scorers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scorers.d.ts","sourceRoot":"","sources":["../../src/eval/scorers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAyBnE;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,gBAAgB,IAAI,MAAM,CAwCzC;AAED,sEAAsE;AACtE,wBAAgB,gBAAgB,CAC/B,UAAU,GAAE,SAAS,MAAM,EAAmC,GAC5D,MAAM,CAqBR;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAezD;AAED,wDAAwD;AACxD,wBAAgB,cAAc,CAAC,GAAG,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAmB5D;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC3B,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,KAAK,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAChE,MAAM,CAER"}
+{"version":3,"file":"scorers.d.ts","sourceRoot":"","sources":["../../src/eval/scorers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAA;AACjE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AACrE,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAyBnE;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,gBAAgB,IAAI,MAAM,CAwCzC;AAED,sEAAsE;AACtE,wBAAgB,gBAAgB,CAC/B,UAAU,GAAE,SAAS,MAAM,EAAmC,GAC5D,MAAM,CAqBR;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAezD;AAED,wDAAwD;AACxD,wBAAgB,cAAc,CAAC,GAAG,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAmB5D;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,SAAS,kBAAkB,EAAE,GAAG,MAAM,CAoB7E;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,MAAM,CAiBnF;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,YAAY,CAAA;CAAO,GAAG,MAAM,CAejF;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC3B,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,KAAK,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAChE,MAAM,CAER"}

package/dist/eval/scorers.js

@@ -136,6 +136,83 @@ export function containsScorer(...required) {
         },
     };
 }
+/**
+ * The turn's outcome record has one of the given verdicts.
+ *
+ * Reads the record the kernel settled from what actually ran, not the answer's
+ * words. A turn with no record scores unavailable: there is nothing to read,
+ * which is not the same as a wrong verdict.
+ */
+export function outcomeScorer(verdicts) {
+    return {
+        name: 'outcome',
+        // The record either says one of the verdicts or it does not.
+        severity: 'gate',
+        score(turn) {
+            const outcome = turn.outcome;
+            if (!outcome) {
+                return { score: 0, unavailable: true, reason: 'the turn has no outcome record' };
+            }
+            const ok = verdicts.includes(outcome.verdict);
+            return {
+                score: ok ? 1 : 0,
+                reason: ok
+                    ? `verdict ${outcome.verdict}`
+                    : `verdict ${outcome.verdict}, expected one of ${verdicts.join(', ')}`,
+                details: { verdict: outcome.verdict, expected: verdicts },
+            };
+        },
+    };
+}
+/**
+ * Whether the answer's plain check claims agree with the turn's outcome
+ * record: 1 when `compareAnswerToOutcome` found no mismatch, 0 when it found
+ * one, with the claim and what the record holds in the detail.
+ *
+ * Pass `{ expectMismatch: true }` for a case that is built to contradict
+ * itself, so the suite proves the comparison fires as well as that it stays
+ * silent.
+ */
+export function claimMismatchScorer(opts = {}) {
+    const expectMismatch = opts.expectMismatch === true;
+    return {
+        name: 'claim-mismatch',
+        severity: 'gate',
+        score(turn) {
+            const found = turn.claimMismatch;
+            const ok = expectMismatch ? found !== undefined : found === undefined;
+            return {
+                score: ok ? 1 : 0,
+                reason: found
+                    ? `${found.said}, but the record: ${found.recordSays}`
+                    : 'no claim in the answer contradicts the record',
+                details: { claimMismatch: found ?? null, expectMismatch },
+            };
+        },
+    };
+}
+/**
+ * Score whether the turn was handed back to a person.
+ *
+ * Pass `{ expectKind }` for a case built so the model hands the task back, and
+ * the scorer checks it did, and for that kind; with nothing it checks the turn
+ * was not handed back. A gate: a scripted suite pins the kernel's side of it.
+ */
+export function handedBackScorer(opts = {}) {
+    return {
+        name: 'handed-back',
+        severity: 'gate',
+        score(turn) {
+            const found = turn.handedBack;
+            const ok = opts.expectKind === undefined ? found === undefined : found?.kind === opts.expectKind;
+            return {
+                score: ok ? 1 : 0,
+                reason: found ? `handed back (${found.kind}): ${found.blocker}` : 'not handed back',
+                details: { handedBack: found ?? null, expectKind: opts.expectKind ?? null },
+            };
+        },
+    };
+}
 /**
  * Judge the turn with a caller-supplied predicate.
  *

package/dist/eval/scorers.js.map

@@ -1 +1 @@
-{"version":3,"file":"scorers.js","sourceRoot":"","sources":["../../src/eval/scorers.ts"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,SAAS,SAAS,CAAC,CAAoB,EAAE,CAAoB;IAC5D,MAAM,KAAK,GAAe,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,CACnE,IAAI,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAC/B,CAAA;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;YACpB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACzB,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI;gBAAE,SAAQ;YAC3B,GAAG,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QAC1F,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;AACxC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,gBAAgB;IAC/B,OAAO;QACN,IAAI,EAAE,YAAY;QAClB,KAAK,CAAC,IAAc,EAAE,QAAkB;YACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,IAAI,EAAE,CAAA;YAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAA;YAE7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAClD,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,mCAAmC,EAAE,CAAA;YACjE,CAAC;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO;oBACN,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,+BAA+B,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;oBAC7E,OAAO,EAAE,EAAE,MAAM,EAAE;iBACnB,CAAA;YACF,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO;oBACN,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,YAAY,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B;oBACpE,OAAO,EAAE,EAAE,QAAQ,EAAE;iBACrB,CAAA;YACF,CAAC;YAED,MAAM,OAAO,GAAG,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;YAC3C,MAAM,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC,MAAM,CAAA;YACzC,MAAM,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAA;YACxC,MAAM,EAAE,GAAG,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAA;YAEzF,OAAO;gBACN,KAAK,EAAE,EAAE;gBACT,MAAM,EACL,EAAE,KAAK,CAAC;oBACP,CAAC,CAAC,qBAAqB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;oBAC3C,CAAC,CAAC,WAAW,OAAO,IAAI,QAAQ,CAAC,MAAM,uBAAuB,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBACjH,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE;aACzD,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,gBAAgB,CAC/B,aAAgC,CAAC,UAAU,EAAE,gBAAgB,CAAC;IAE9D,OAAO;QACN,IAAI,EAAE,YAAY;QAClB,oEAAoE;QACpE,kEAAkE;QAClE,oEAAoE;QACpE,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAChB,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAA;YAC1D,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YAChF,OAAO;gBACN,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjB,MAAM,EAAE,EAAE;oBACT,CAAC,CAAC,cAAc,IAAI,CAAC,UAAU,EAAE;oBACjC,CAAC,CAAC,cAAc,IAAI,CAAC,UAAU,IAAI,SAAS,qBAAqB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACzF,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE;aACxC,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAChD,OAAO;QACN,IAAI,EAAE,aAAa;QACnB,KAAK,CAAC,IAAc;YACnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;YAC9B,OAAO;gBACN,KAAK,EAAE,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;gBAC1D,MAAM,EACL,IAAI,IAAI,QAAQ;oBACf,CAAC,CAAC,GAAG,IAAI,IAAI,QAAQ,QAAQ;oBAC7B,CAAC,CAAC,gBAAgB,IAAI,oBAAoB,QAAQ,YAAY;gBAChE,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC3B,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,cAAc,CAAC,GAAG,QAAkB;IACnD,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,gEAAgE;QAChE,+DAA+D;QAC/D,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAA;YAC9B,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YACzD,OAAO;gBACN,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM;gBACvF,MAAM,EACL,OAAO,CAAC,MAAM,KAAK,CAAC;oBACnB,CAAC,CAAC,aAAa,QAAQ,CAAC,MAAM,qBAAqB;oBACnD,CAAC,CAAC,YAAY,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACpC,OAAO,EAAE,EAAE,OAAO,EAAE;aACpB,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAC3B,IAAY,EACZ,EAAkE;IAElE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;AAC3B,CAAC"}
+{"version":3,"file":"scorers.js","sourceRoot":"","sources":["../../src/eval/scorers.ts"],"names":[],"mappings":"AAIA;;;;;;GAMG;AACH,SAAS,SAAS,CAAC,CAAoB,EAAE,CAAoB;IAC5D,MAAM,KAAK,GAAe,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,CACnE,IAAI,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAC/B,CAAA;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;YACpB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACzB,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI;gBAAE,SAAQ;YAC3B,GAAG,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QAC1F,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;AACxC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,gBAAgB;IAC/B,OAAO;QACN,IAAI,EAAE,YAAY;QAClB,KAAK,CAAC,IAAc,EAAE,QAAkB;YACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,IAAI,EAAE,CAAA;YAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAA;YAE7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAClD,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,mCAAmC,EAAE,CAAA;YACjE,CAAC;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO;oBACN,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,+BAA+B,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;oBAC7E,OAAO,EAAE,EAAE,MAAM,EAAE;iBACnB,CAAA;YACF,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO;oBACN,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,YAAY,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B;oBACpE,OAAO,EAAE,EAAE,QAAQ,EAAE;iBACrB,CAAA;YACF,CAAC;YAED,MAAM,OAAO,GAAG,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;YAC3C,MAAM,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC,MAAM,CAAA;YACzC,MAAM,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAA;YACxC,MAAM,EAAE,GAAG,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAA;YAEzF,OAAO;gBACN,KAAK,EAAE,EAAE;gBACT,MAAM,EACL,EAAE,KAAK,CAAC;oBACP,CAAC,CAAC,qBAAqB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;oBAC3C,CAAC,CAAC,WAAW,OAAO,IAAI,QAAQ,CAAC,MAAM,uBAAuB,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBACjH,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE;aACzD,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,gBAAgB,CAC/B,aAAgC,CAAC,UAAU,EAAE,gBAAgB,CAAC;IAE9D,OAAO;QACN,IAAI,EAAE,YAAY;QAClB,oEAAoE;QACpE,kEAAkE;QAClE,oEAAoE;QACpE,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAChB,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAA;YAC1D,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YAChF,OAAO;gBACN,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjB,MAAM,EAAE,EAAE;oBACT,CAAC,CAAC,cAAc,IAAI,CAAC,UAAU,EAAE;oBACjC,CAAC,CAAC,cAAc,IAAI,CAAC,UAAU,IAAI,SAAS,qBAAqB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACzF,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE;aACxC,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAChD,OAAO;QACN,IAAI,EAAE,aAAa;QACnB,KAAK,CAAC,IAAc;YACnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;YAC9B,OAAO;gBACN,KAAK,EAAE,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;gBAC1D,MAAM,EACL,IAAI,IAAI,QAAQ;oBACf,CAAC,CAAC,GAAG,IAAI,IAAI,QAAQ,QAAQ;oBAC7B,CAAC,CAAC,gBAAgB,IAAI,oBAAoB,QAAQ,YAAY;gBAChE,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC3B,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,cAAc,CAAC,GAAG,QAAkB;IACnD,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,gEAAgE;QAChE,+DAA+D;QAC/D,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAA;YAC9B,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YACzD,OAAO;gBACN,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM;gBACvF,MAAM,EACL,OAAO,CAAC,MAAM,KAAK,CAAC;oBACnB,CAAC,CAAC,aAAa,QAAQ,CAAC,MAAM,qBAAqB;oBACnD,CAAC,CAAC,YAAY,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACpC,OAAO,EAAE,EAAE,OAAO,EAAE;aACpB,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,QAAuC;IACpE,OAAO;QACN,IAAI,EAAE,SAAS;QACf,6DAA6D;QAC7D,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAA;YAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACd,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,gCAAgC,EAAE,CAAA;YACjF,CAAC;YACD,MAAM,EAAE,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YAC7C,OAAO;gBACN,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjB,MAAM,EAAE,EAAE;oBACT,CAAC,CAAC,WAAW,OAAO,CAAC,OAAO,EAAE;oBAC9B,CAAC,CAAC,WAAW,OAAO,CAAC,OAAO,qBAAqB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACvE,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE;aACzD,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAqC,EAAE;IAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,KAAK,IAAI,CAAA;IACnD,OAAO;QACN,IAAI,EAAE,gBAAgB;QACtB,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAA;YAChC,MAAM,EAAE,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAA;YACrE,OAAO;gBACN,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjB,MAAM,EAAE,KAAK;oBACZ,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,qBAAqB,KAAK,CAAC,UAAU,EAAE;oBACtD,CAAC,CAAC,+CAA+C;gBAClD,OAAO,EAAE,EAAE,aAAa,EAAE,KAAK,IAAI,IAAI,EAAE,cAAc,EAAE;aACzD,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAsC,EAAE;IACxE,OAAO;QACN,IAAI,EAAE,aAAa;QACnB,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAA;YAC7B,MAAM,EAAE,GACP,IAAI,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,KAAK,IAAI,CAAC,UAAU,CAAA;YACtF,OAAO;gBACN,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjB,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,gBAAgB,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,iBAAiB;gBACnF,OAAO,EAAE,EAAE,UAAU,EAAE,KAAK,IAAI,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE;aAC3E,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAC3B,IAAY,EACZ,EAAkE;IAElE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;AAC3B,CAAC"}

package/dist/eval/types.d.ts

@@ -1,5 +1,18 @@
+import type { AnswerClaimMismatch, TurnOutcome } from '../types/session/outcome.js';
 import type { StepResult } from '../types/session/step.js';
+import type { CaseIntervals } from './bootstrap.js';
+import type { ReportDisclosure } from './disclosure.js';
+import type { RepeatReliability, SplitSummary } from './report-stats.js';
 import type { ScoreUncertainty } from './uncertainty.js';
+/**
+ * Which part of a suite a case belongs to.
+ *
+ * `development` cases are the ones the harness was shaped against; a number
+ * measured on them says how well it fits them. `held-out` cases were not
+ * looked at while it was shaped, and only they say how it does on cases it
+ * has not seen.
+ */
+export type EvalSplit = 'development' | 'held-out';
 /**
  * One case in a dataset: an input, and what a good run looks like.
  *
@@ -18,6 +31,11 @@ export interface EvalCase<TInput = unknown> {
     expectedTools?: readonly string[];
     /** Per-case scorer overrides; falls back to the dataset's scorers. */
     scorers?: readonly Scorer[];
+    /**
+     * `development` or `held-out`. Left out, the case is reported as
+     * unassigned. See {@link EvalSplit}.
+     */
+    split?: EvalSplit;
 }
 /** What actually happened, handed to every scorer. */
 export interface EvalTurn {
@@ -34,6 +52,18 @@ export interface EvalTurn {
     durationMs: number;
     /** Set when the turn threw rather than settling. */
     error?: string;
+    /**
+     * What the turn changed and what checked it, from the kernel's own record of
+     * its tool calls. Absent for a turn that never ran the loop.
+     */
+    outcome?: TurnOutcome;
+    /**
+     * A plain claim in the answer that the outcome record contradicts, found by
+     * `compareAnswerToOutcome`. Absent when there is none.
+     */
+    claimMismatch?: AnswerClaimMismatch;
+    /** The model handed the task back to a person and the turn ended there. Absent otherwise. */
+    handedBack?: import('../types/session/hand-back.js').HandBack;
 }
 /**
  * A judgement about one turn.
@@ -136,6 +166,10 @@ export interface CaseResult {
      * mattered. Absent when no gate missed.
      */
     failedGates?: readonly string[];
+    /** Which run of this case this is, from 1. Set on every run of a report that has `repeats`. */
+    repeat?: number;
+    /** The case's declared split, copied here so a report can be cut by it. */
+    split?: EvalSplit;
 }
 export interface ExperimentReport {
     name: string;
@@ -169,5 +203,45 @@ export interface ExperimentReport {
      */
     byScorer: Record<string, number>;
     durationMs: number;
+    /**
+     * Times each case ran. When it is above 1, `cases` holds every run of
+     * every case, in case order, each marked with its `repeat`, and `passed`,
+     * `failed` and `inconclusive` count runs. Absent in a report written before
+     * repeats existed, which is read as 1.
+     */
+    repeats?: number;
+    /** Seed of the bootstrap behind {@link intervals}. */
+    seed?: number;
+    /**
+     * Passed runs over judged runs. An inconclusive run is in neither number,
+     * so it cannot pull the rate either way.
+     */
+    passRate?: number;
+    /**
+     * 95% intervals for {@link mean} and {@link passRate}, by a bootstrap over
+     * cases. Absent from a report that predates it.
+     */
+    intervals?: CaseIntervals;
+    /** Per case: how many of its repeats passed. Present when `repeats` is above 1. */
+    caseSummaries?: readonly CaseRepeatSummary[];
+    /** Reliability across repeats. Present when `repeats` is above 1. */
+    reliability?: RepeatReliability;
+    /** Numbers per split. Keys appear only for splits that have cases. */
+    splits?: Partial<Record<EvalSplit | 'unassigned', SplitSummary>>;
+    /** What produced this report. */
+    disclosure?: ReportDisclosure;
+}
+/** How the repeats of one case went. */
+export interface CaseRepeatSummary {
+    case: string;
+    split?: EvalSplit;
+    repeats: number;
+    passed: number;
+    failed: number;
+    inconclusive: number;
+    /** Mean score over the judged repeats; null when none was judged. */
+    mean: number | null;
+    /** Every repeat passed. An inconclusive repeat means this is false. */
+    passedEveryRepeat: boolean;
 }
 //# sourceMappingURL=types.d.ts.map