@namzu/sdk 62.12.0 → 62.14.0

package/dist/eval/experiment.js

@@ -1,4 +1,9 @@
+import { EvalUsageError, validateSeed } from './bootstrap.js';
+import { buildDisclosure, describeDisclosure } from './disclosure.js';
+import { describeIntervals, describeSplits, duplicateMessage, duplicateName, groupRuns, statsOver, } from './report-stats.js';
+import { currentEvalRunOptions } from './run-options.js';
 import { describeUncertainty, uncertaintyOf } from './uncertainty.js';
+const MAX_REPEATS = 100;
 const MAX_CASE_TIMEOUT_MS = 2_147_483_647;
 /**
  * Run a dataset and score it.
@@ -17,25 +22,42 @@ export async function runExperiment(config) {
     const threshold = config.passThreshold ?? 1;
     const concurrency = Math.max(1, config.concurrency ?? 1);
     const timeoutMs = validateCaseTimeoutMs(config.timeoutMs);
-    const results = new Array(config.cases.length);
+    const options = currentEvalRunOptions();
+    const repeats = validateRepeats(options.repeats ?? config.repeats);
+    const seed = validateSeed(options.seed ?? config.seed);
+    for (const evalCase of config.cases)
+        validateSplit(evalCase);
+    const alike = duplicateName(config.cases.map((c) => c.name));
+    if (alike !== undefined)
+        throw new Error(duplicateMessage(config.name, alike));
+    // Case-major, so a case's repeats sit together in the report.
+    const total = config.cases.length * repeats;
+    const results = new Array(total);
     let cursor = 0;
     const worker = async () => {
         for (;;) {
             const index = cursor++;
-            const evalCase = config.cases[index];
-            if (!evalCase)
+            if (index >= total)
                 return;
+            const evalCase = config.cases[Math.floor(index / repeats)];
             const result = await evaluateCase(config, evalCase, threshold, timeoutMs);
+            result.repeat = (index % repeats) + 1;
+            if (evalCase.split)
+                result.split = evalCase.split;
             results[index] = result;
             config.onCaseFinish?.(result);
         }
     };
-    await Promise.all(Array.from({ length: Math.min(concurrency, config.cases.length) }, worker));
+    await Promise.all(Array.from({ length: Math.min(concurrency, total) }, worker));
     const settled = results.filter(Boolean);
     // An inconclusive case has no mean to contribute; averaging its zero in
     // would report a suite-wide regression caused by the harness.
     const scored = settled.filter((r) => r.status !== 'inconclusive');
     const mean = scored.length === 0 ? 0 : scored.reduce((sum, r) => sum + r.mean, 0) / scored.length;
+    const stats = statsOver(groupRuns(settled), seed);
+    // One number per CASE, so a case's repeats are not counted as independent
+    // evidence about the suite.
+    const caseMeans = stats.caseSummaries.flatMap((c) => (c.mean === null ? [] : [c.mean]));
     return {
         name: config.name,
         cases: settled,
@@ -43,14 +65,35 @@ export async function runExperiment(config) {
         // Over the same cases the mean is over. Computing spread across a
         // different denominator than the average it qualifies would produce
         // an interval that does not belong to the number beside it.
-        uncertainty: uncertaintyOf(scored.map((r) => r.mean)),
+        uncertainty: uncertaintyOf(caseMeans),
         passed: settled.filter((r) => r.status === 'passed').length,
         failed: settled.filter((r) => r.status === 'failed').length,
         inconclusive: settled.filter((r) => r.status === 'inconclusive').length,
         byScorer: meanByScorer(settled),
         durationMs: Date.now() - startedAt,
+        repeats,
+        seed,
+        ...(stats.passRate === null ? {} : { passRate: stats.passRate }),
+        intervals: stats.intervals,
+        ...(repeats > 1 ? { caseSummaries: stats.caseSummaries, reliability: stats.reliability } : {}),
+        splits: stats.splits,
+        disclosure: buildDisclosure(config.disclosure, { cases: config.cases.length, repeats, seed }, options, new Date().toISOString()),
     };
 }
+function validateRepeats(value) {
+    if (value === undefined)
+        return 1;
+    if (!Number.isInteger(value) || value < 1 || value > MAX_REPEATS) {
+        throw new EvalUsageError(`repeats must be an integer from 1 to ${MAX_REPEATS}, or omitted; received ${String(value)}`);
+    }
+    return value;
+}
+function validateSplit(evalCase) {
+    const split = evalCase.split;
+    if (split !== undefined && split !== 'development' && split !== 'held-out') {
+        throw new RangeError(`case "${evalCase.name}" declares split ${JSON.stringify(split)}; a split is "development" or "held-out", or left out`);
+    }
+}
 function validateCaseTimeoutMs(value) {
     if (value === undefined)
         return undefined;
@@ -263,17 +306,46 @@ function derivedUncertainty(report) {
     return uncertaintyOf(report.cases.filter((c) => c.status !== 'inconclusive').map((c) => c.mean));
 }
 export function formatReport(report) {
+    const repeats = report.repeats ?? 1;
+    const caseCount = repeats > 1 ? report.cases.length / repeats : report.cases.length;
     const lines = [
-        `${report.name}: ${report.passed}/${report.cases.length} passed (mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`,
-        // On its own line and always printed, including when the interval is
-        // undefined. A mean printed alone is the thing that has been
-        // over-read: two experiments three points apart look like a difference, and
-        // at the n a hand-built suite has they are usually the same experiment
-        // twice. Computing the interval and not showing it would leave the
-        // reader exactly where they started.
-        `  ${describeUncertainty(report.mean, report.uncertainty ?? derivedUncertainty(report))}`,
-        '',
+        repeats > 1
+            ? `${report.name}: ${report.passed}/${report.cases.length} runs passed (${caseCount} cases x ${repeats} repeats, mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`
+            : `${report.name}: ${report.passed}/${report.cases.length} passed (mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`,
     ];
+    if (report.intervals) {
+        // The interval sits next to the number it belongs to. A mean printed
+        // alone is the thing that has been over-read: two experiments three
+        // points apart look like a difference, and at the n a hand-built
+        // suite has they are usually the same experiment twice.
+        for (const line of describeIntervals({
+            mean: report.cases.some((c) => c.status !== 'inconclusive') ? report.mean : null,
+            passRate: report.passRate ?? null,
+            intervals: report.intervals,
+        })) {
+            lines.push(`  ${line}`);
+        }
+    }
+    else {
+        // A report from before bootstrap intervals, or one built by hand: the
+        // older interval, still never omitted.
+        lines.push(`  ${describeUncertainty(report.mean, report.uncertainty ?? derivedUncertainty(report))}`);
+    }
+    if (report.reliability && repeats > 1) {
+        const r = report.reliability;
+        lines.push(`  reliability across repeats: ${r.passedEveryRepeat} of ${r.cases} cases passed in every one of ${repeats} repeats${r.fraction === null ? '' : ` (${(r.fraction * 100).toFixed(1)}%)`}`);
+        for (const c of report.caseSummaries ?? []) {
+            if (!c.passedEveryRepeat)
+                lines.push(`    ${c.case}: ${c.passed} of ${c.repeats} passed`);
+        }
+    }
+    if (report.splits) {
+        for (const line of describeSplits(report.splits))
+            lines.push(`  ${line}`);
+    }
+    if (report.disclosure)
+        lines.push(`  ${describeDisclosure(report.disclosure)}`);
+    lines.push('');
     // Surfaced above the failures, because an inconclusive case means the
     // harness is broken and every number below it is measured over less
     // evidence than it looks like.
@@ -287,7 +359,7 @@ export function formatReport(report) {
     if (failures.length > 0) {
         lines.push('', 'Failures:');
         for (const failure of failures) {
-            lines.push(`  ✗ ${failure.case} (${failure.mean.toFixed(2)})`);
+            lines.push(`  ✗ ${failure.case}${repeats > 1 && failure.repeat ? ` [repeat ${failure.repeat} of ${repeats}]` : ''} (${failure.mean.toFixed(2)})`);
             if (failure.failedGates && failure.failedGates.length > 0) {
                 // First line under the case, because it says WHY this failed.
                 // A mean of 0.75 next to four scores leaves the reader

package/dist/eval/experiment.js.map

@@ -1 +1 @@
-{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErE,MAAM,mBAAmB,GAAG,aAAa,CAAA;AAmCzC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,MAAgC;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAA;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAA;IACxD,MAAM,SAAS,GAAG,qBAAqB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IAEzD,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;IAC5D,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACxC,SAAS,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAA;YACtB,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YACpC,IAAI,CAAC,QAAQ;gBAAE,OAAM;YAErB,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;YACzE,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAA;YACvB,MAAM,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAA;QAC9B,CAAC;IACF,CAAC,CAAA;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAA;IAE7F,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACvC,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IAEjG,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,OAAO;QACd,IAAI;QACJ,kEAAkE;QAClE,oEAAoE;QACpE,4DAA4D;QAC5D,WAAW,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QACrD,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,MAAM;QACvE,QAAQ,EAAE,YAAY,CAAC,OAAO,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;KAClC,CAAA;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAyB;IACvD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACzC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,mBAAmB,EAAE,CAAC;QAC1E,MAAM,IAAI,UAAU,CACnB,0CAA0C,mBAAmB,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CACtG,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAQD,6EAA6E;AAC7E,SAAS,gBAAgB,CAAC,SAA6B;IACtD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,IAAI,KAAgD,CAAA;IACpD,IAAI,MAAkC,CAAA;IACtC,IAAI,OAAiC,CAAA;IAErC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YAChD,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAChD,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACrE,CAAC,CAAC,CAAA;QACF,qEAAqE;QACrE,gEAAgE;QAChE,yDAAyD;QACzD,KAAK,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC3B,KAAK,GAAG,UAAU,CACjB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAC,EACxE,SAAS,CACT,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,KAAK,CAAC,IAAI,CAAI,IAAgB;YAC7B,IAAI,CAAC,MAAM;gBAAE,OAAO,MAAM,IAAI,CAAA;YAC9B,qEAAqE;YACrE,mEAAmE;YACnE,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;YACzB,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;QAC1C,CAAC;QACD,OAAO;YACN,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,IAAI,OAAO;gBAAE,UAAU,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACrE,CAAC;KACD,CAAA;AACF,CAAC;AAED,KAAK,UAAU,YAAY,CAC1B,MAAgC,EAChC,QAA0B,EAC1B,SAAiB,EACjB,SAA6B;IAE7B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAA;IAC5C,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAC1D,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAA;QAClD,MAAM,MAAM,GAA0B,EAAE,CAAA;QAExC,iEAAiE;QACjE,iEAAiE;QACjE,iDAAiD;QACjD,iEAAiE;QACjE,0DAA0D;QAC1D,6DAA6D;QAC7D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;QAC9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CACd,0BAA0B,MAAM,CAAC,IAAI,eAAe,QAAQ,CAAC,IAAI,wKAAwK,CACzO,CAAA;YACF,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QACxE,CAAC;QAED,iEAAiE;QACjE,+DAA+D;QAC/D,2DAA2D;QAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;aAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;QAExF,iEAAiE;QACjE,8DAA8D;QAC9D,2DAA2D;QAC3D,gEAAgE;QAChE,2DAA2D;QAC3D,8DAA8D;QAC9D,8DAA8D;QAC9D,uBAAuB;QACvB,MAAM,WAAW,GAAG,OAAO;aACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC;aACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACb,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YAC5B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,OAAO,KAAK,CAAA;YACtD,OAAO,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC,CAAA;QAChD,CAAC,CAAC;aACD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEpB,MAAM,MAAM,GACX,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YACpD,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;gBACvB,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,IAAI,IAAI,SAAS;oBAClB,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,QAAQ,CAAA;QACd,OAAO;YACN,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI;YACJ,MAAM;YACN,IAAI;YACJ,MAAM;YACN,MAAM,EAAE,MAAM,KAAK,QAAQ;YAC3B,8DAA8D;YAC9D,6DAA6D;YAC7D,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAA;IACF,CAAC;YAAS,CAAC;QACV,QAAQ,CAAC,OAAO,EAAE,CAAA;IACnB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,WAAW,CACzB,MAAgC,EAChC,QAA0B,EAC1B,QAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;QAClE,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,OAAO;YACN,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,EAAE;YACb,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,4DAA4D;YAC5D,6DAA6D;YAC7D,sBAAsB;YACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACvD,CAAA;IACF,CAAC;AACF,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,SAAS,CACvB,MAAc,EACd,IAAc,EACd,QAAkB,EAClB,QAAsB;IAEtB,sEAAsE;IACtE,qEAAqE;IACrE,2DAA2D;IAC3D,uEAAuE;IACvE,wEAAwE;IACxE,0EAA0E;IAC1E,qDAAqD;IACrD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAA;IAC1F,CAAC;IAED,IAAI,CAAC;QACJ,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QAChC,qEAAqE;QACrE,oEAAoE;QACpE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;QACxF,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,oEAAoE;QACpE,6DAA6D;QAC7D,kEAAkE;QAClE,qEAAqE;QACrE,kEAAkE;QAClE,yCAAyC;QACzC,OAAO;YACN,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,WAAW,MAAM,CAAC,IAAI,gCAC7B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAChD,EAAE;SACF,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,OAA8B;IACnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4C,CAAA;IAChE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,SAAQ;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;YACtD,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAA;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACtB,CAAC;IACF,CAAC;IACD,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,oEAAoE;IACpE,sEAAsE;IACtE,qBAAqB;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,KAAK,CAAA;IACzC,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH;;;;;;;;;;;;GAYG;AACH,SAAS,kBAAkB,CAAC,MAAwB;IACnD,OAAO,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;AACjG,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAwB;IACpD,MAAM,KAAK,GAAa;QACvB,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;QAC3H,qEAAqE;QACrE,6DAA6D;QAC7D,4EAA4E;QAC5E,uEAAuE;QACvE,mEAAmE;QACnE,qCAAqC;QACrC,KAAK,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,WAAW,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,EAAE;QACzF,EAAE;KACF,CAAA;IAED,sEAAsE;IACtE,oEAAoE;IACpE,+BAA+B;IAC/B,IAAI,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACT,KAAK,MAAM,CAAC,YAAY,QAAQ,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,yDAAyD,EAC7H,EAAE,CACF,CAAA;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAA;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,OAAO,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAA;YAC9D,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3D,8DAA8D;gBAC9D,uDAAuD;gBACvD,+BAA+B;gBAC/B,KAAK,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACnE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;oBAChC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,kBAAkB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC1D,CAAC;qBAAM,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC7C,CAAC;YACF,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAA;QACzE,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACxE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;YAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}
+{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/eval/experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAA;AAC7D,OAAO,EAAmB,eAAe,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACtF,OAAO,EACN,iBAAiB,EACjB,cAAc,EACd,gBAAgB,EAChB,aAAa,EACb,SAAS,EACT,SAAS,GACT,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAA;AAWxD,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAErE,MAAM,WAAW,GAAG,GAAG,CAAA;AAEvB,MAAM,mBAAmB,GAAG,aAAa,CAAA;AAyDzC;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,MAAgC;IAEhC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,aAAa,IAAI,CAAC,CAAA;IAC3C,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,IAAI,CAAC,CAAC,CAAA;IACxD,MAAM,SAAS,GAAG,qBAAqB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IACzD,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAA;IACvC,MAAM,OAAO,GAAG,eAAe,CAAC,OAAO,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAC,CAAA;IAClE,MAAM,IAAI,GAAG,YAAY,CAAC,OAAO,CAAC,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,CAAA;IACtD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,KAAK;QAAE,aAAa,CAAC,QAAQ,CAAC,CAAA;IAC5D,MAAM,KAAK,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,IAAI,KAAK,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,gBAAgB,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAA;IAE9E,8DAA8D;IAC9D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAA;IAC3C,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,KAAK,CAAC,CAAA;IAC9C,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,MAAM,GAAG,KAAK,IAAmB,EAAE;QACxC,SAAS,CAAC;YACT,MAAM,KAAK,GAAG,MAAM,EAAE,CAAA;YACtB,IAAI,KAAK,IAAI,KAAK;gBAAE,OAAM;YAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,CAAqB,CAAA;YAE9E,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC,CAAA;YACzE,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;YACrC,IAAI,QAAQ,CAAC,KAAK;gBAAE,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAA;YACjD,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAA;YACvB,MAAM,CAAC,YAAY,EAAE,CAAC,MAAM,CAAC,CAAA;QAC9B,CAAC;IACF,CAAC,CAAA;IAED,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC,CAAA;IAE/E,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACvC,wEAAwE;IACxE,8DAA8D;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACjE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;IAEjG,MAAM,KAAK,GAAG,SAAS,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,CAAA;IACjD,0EAA0E;IAC1E,4BAA4B;IAC5B,MAAM,SAAS,GAAG,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAEvF,OAAO;QACN,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,KAAK,EAAE,OAAO;QACd,IAAI;QACJ,kEAAkE;QAClE,oEAAoE;QACpE,4DAA4D;QAC5D,WAAW,EAAE,aAAa,CAAC,SAAS,CAAC;QACrC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,MAAM;QAC3D,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,MAAM;QACvE,QAAQ,EAAE,YAAY,CAAC,OAAO,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;QAClC,OAAO;QACP,IAAI;QACJ,GAAG,CAAC,KAAK,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC;QAChE,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,KAAK,CAAC,aAAa,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,UAAU,EAAE,eAAe,CAC1B,MAAM,CAAC,UAAU,EACjB,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,EAC7C,OAAO,EACP,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CACxB;KACD,CAAA;AACF,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB;IACjD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,CAAC,CAAA;IACjC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,WAAW,EAAE,CAAC;QAClE,MAAM,IAAI,cAAc,CACvB,wCAAwC,WAAW,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CAC5F,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,aAAa,CAAC,QAA2B;IACjD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAgB,CAAA;IACvC,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,aAAa,IAAI,KAAK,KAAK,UAAU,EAAE,CAAC;QAC5E,MAAM,IAAI,UAAU,CACnB,SAAS,QAAQ,CAAC,IAAI,oBAAoB,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,uDAAuD,CACtH,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAyB;IACvD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAA;IACzC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,mBAAmB,EAAE,CAAC;QAC1E,MAAM,IAAI,UAAU,CACnB,0CAA0C,mBAAmB,0BAA0B,MAAM,CAAC,KAAK,CAAC,EAAE,CACtG,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAQD,6EAA6E;AAC7E,SAAS,gBAAgB,CAAC,SAA6B;IACtD,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAA;IACxC,IAAI,KAAgD,CAAA;IACpD,IAAI,MAAkC,CAAA;IACtC,IAAI,OAAiC,CAAA;IAErC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC9C,MAAM,GAAG,IAAI,OAAO,CAAQ,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE;YAChD,OAAO,GAAG,GAAG,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;YAChD,UAAU,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAA;QACrE,CAAC,CAAC,CAAA;QACF,qEAAqE;QACrE,gEAAgE;QAChE,yDAAyD;QACzD,KAAK,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;QAC3B,KAAK,GAAG,UAAU,CACjB,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,wBAAwB,SAAS,IAAI,CAAC,CAAC,EACxE,SAAS,CACT,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,KAAK,CAAC,IAAI,CAAI,IAAgB;YAC7B,IAAI,CAAC,MAAM;gBAAE,OAAO,MAAM,IAAI,CAAA;YAC9B,qEAAqE;YACrE,mEAAmE;YACnE,KAAK,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;YACzB,OAAO,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;QAC1C,CAAC;QACD,OAAO;YACN,YAAY,CAAC,KAAK,CAAC,CAAA;YACnB,IAAI,OAAO;gBAAE,UAAU,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACrE,CAAC;KACD,CAAA;AACF,CAAC;AAED,KAAK,UAAU,YAAY,CAC1B,MAAgC,EAChC,QAA0B,EAC1B,SAAiB,EACjB,SAA6B;IAE7B,MAAM,QAAQ,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAA;IAC5C,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QAC1D,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,MAAM,CAAC,OAAO,CAAA;QAClD,MAAM,MAAM,GAA0B,EAAE,CAAA;QAExC,iEAAiE;QACjE,iEAAiE;QACjE,iDAAiD;QACjD,iEAAiE;QACjE,0DAA0D;QAC1D,6DAA6D;QAC7D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;QAC9B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CACd,0BAA0B,MAAM,CAAC,IAAI,eAAe,QAAQ,CAAC,IAAI,wKAAwK,CACzO,CAAA;YACF,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACrB,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;QACxE,CAAC;QAED,iEAAiE;QACjE,+DAA+D;QAC/D,2DAA2D;QAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;aAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,IAAI,CAAC;aACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAA;QAExF,iEAAiE;QACjE,8DAA8D;QAC9D,2DAA2D;QAC3D,gEAAgE;QAChE,2DAA2D;QAC3D,8DAA8D;QAC9D,8DAA8D;QAC9D,uBAAuB;QACvB,MAAM,WAAW,GAAG,OAAO;aACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC;aACpC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;YACb,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;YAC5B,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,OAAO,KAAK,CAAA;YACtD,OAAO,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,SAAS,CAAC,CAAA;QAChD,CAAC,CAAC;aACD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAEpB,MAAM,MAAM,GACX,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YACpD,CAAC,CAAC,cAAc;YAChB,CAAC,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC;gBACvB,CAAC,CAAC,QAAQ;gBACV,CAAC,CAAC,IAAI,IAAI,SAAS;oBAClB,CAAC,CAAC,QAAQ;oBACV,CAAC,CAAC,QAAQ,CAAA;QACd,OAAO;YACN,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,IAAI;YACJ,MAAM;YACN,IAAI;YACJ,MAAM;YACN,MAAM,EAAE,MAAM,KAAK,QAAQ;YAC3B,8DAA8D;YAC9D,6DAA6D;YAC7D,GAAG,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClD,CAAA;IACF,CAAC;YAAS,CAAC;QACV,QAAQ,CAAC,OAAO,EAAE,CAAA;IACnB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,WAAW,CACzB,MAAgC,EAChC,QAA0B,EAC1B,QAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,CAAC;QACJ,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAA;QAClE,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,OAAO;YACN,MAAM,EAAE,IAAI;YACZ,KAAK,EAAE,EAAE;YACT,SAAS,EAAE,EAAE;YACb,WAAW,EAAE,CAAC;YACd,YAAY,EAAE,CAAC;YACf,4DAA4D;YAC5D,6DAA6D;YAC7D,sBAAsB;YACtB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;YAClC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;SACvD,CAAA;IACF,CAAC;AACF,CAAC;AAED,kEAAkE;AAClE,KAAK,UAAU,SAAS,CACvB,MAAc,EACd,IAAc,EACd,QAAkB,EAClB,QAAsB;IAEtB,sEAAsE;IACtE,qEAAqE;IACrE,2DAA2D;IAC3D,uEAAuE;IACvE,wEAAwE;IACxE,0EAA0E;IAC1E,qDAAqD;IACrD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,EAAE,CAAA;IAC1F,CAAC;IAED,IAAI,CAAC;QACJ,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,CAAA;QAChC,qEAAqE;QACrE,oEAAoE;QACpE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAA;QACxF,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACd,oEAAoE;QACpE,6DAA6D;QAC7D,kEAAkE;QAClE,qEAAqE;QACrE,kEAAkE;QAClE,yCAAyC;QACzC,OAAO;YACN,KAAK,EAAE,CAAC;YACR,WAAW,EAAE,IAAI;YACjB,MAAM,EAAE,WAAW,MAAM,CAAC,IAAI,gCAC7B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAChD,EAAE;SACF,CAAA;IACF,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,OAA8B;IACnD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA4C,CAAA;IAChE,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAC3D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI;gBAAE,SAAQ;YACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAA;YACtD,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,CAAA;YAC1B,KAAK,CAAC,KAAK,EAAE,CAAA;YACb,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;QACtB,CAAC;IACF,CAAC;IACD,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,oEAAoE;IACpE,sEAAsE;IACtE,qBAAqB;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC7C,IAAI,KAAK,GAAG,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,KAAK,CAAA;IACzC,CAAC;IACD,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH;;;;;;;;;;;;GAYG;AACH,SAAS,kBAAkB,CAAC,MAAwB;IACnD,OAAO,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;AACjG,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAwB;IACpD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAA;IACnC,MAAM,SAAS,GAAG,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAA;IACnF,MAAM,KAAK,GAAa;QACvB,OAAO,GAAG,CAAC;YACV,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,SAAS,YAAY,OAAO,kBAAkB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;YAC3K,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,iBAAiB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,MAAM,CAAC,UAAU,IAAI;KAC9H,CAAA;IACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,qEAAqE;QACrE,oEAAoE;QACpE,iEAAiE;QACjE,wDAAwD;QACxD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;YACpC,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI;YAChF,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,IAAI;YACjC,SAAS,EAAE,MAAM,CAAC,SAAS;SAC3B,CAAC,EAAE,CAAC;YACJ,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;SAAM,CAAC;QACP,sEAAsE;QACtE,uCAAuC;QACvC,KAAK,CAAC,IAAI,CACT,KAAK,mBAAmB,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,WAAW,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAC,EAAE,CACzF,CAAA;IACF,CAAC;IACD,IAAI,MAAM,CAAC,WAAW,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,MAAM,CAAC,WAAW,CAAA;QAC5B,KAAK,CAAC,IAAI,CACT,iCAAiC,CAAC,CAAC,iBAAiB,OAAO,CAAC,CAAC,KAAK,iCAAiC,OAAO,WAAW,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CACxL,CAAA;QACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,aAAa,IAAI,EAAE,EAAE,CAAC;YAC5C,IAAI,CAAC,CAAC,CAAC,iBAAiB;gBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,OAAO,SAAS,CAAC,CAAA;QAC1F,CAAC;IACF,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IAC1E,CAAC;IACD,IAAI,MAAM,CAAC,UAAU;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,kBAAkB,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAA;IAC/E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAEd,sEAAsE;IACtE,oEAAoE;IACpE,+BAA+B;IAC/B,IAAI,MAAM,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CACT,KAAK,MAAM,CAAC,YAAY,QAAQ,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,yDAAyD,EAC7H,EAAE,CACF,CAAA;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAA;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,CAAA;QAC3B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CACT,OAAO,OAAO,CAAC,IAAI,GAAG,OAAO,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,OAAO,CAAC,MAAM,OAAO,OAAO,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CACrI,CAAA;YACD,IAAI,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC3D,8DAA8D;gBAC9D,uDAAuD;gBACvD,+BAA+B;gBAC/B,KAAK,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YACnE,CAAC;YACD,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC5D,IAAI,KAAK,CAAC,WAAW,KAAK,IAAI,EAAE,CAAC;oBAChC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,kBAAkB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC1D,CAAC;qBAAM,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;oBAC5B,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;gBAC7C,CAAC;YACF,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK;gBAAE,KAAK,CAAC,IAAI,CAAC,gBAAgB,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAA;QACzE,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,cAAc,CAAC,CAAA;IACxE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,EAAE,EAAE,aAAa,CAAC,CAAA;QAC7B,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;YAC9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzD,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;YAC7C,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/from-turn.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"from-turn.d.ts","sourceRoot":"","sources":["../../src/eval/from-turn.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAA;AACpD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAE1C;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,QAAQ,CAczF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACtC,SAAS,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,GACtC,OAAO,CAAC,QAAQ,CAAC,CAKnB"}
+{"version":3,"file":"from-turn.d.ts","sourceRoot":"","sources":["../../src/eval/from-turn.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,0BAA0B,CAAA;AACpD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AAE1C;;;;;;;;GAQG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,QAAQ,CAkBzF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,iBAAiB,CACtC,SAAS,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC,GACtC,OAAO,CAAC,QAAQ,CAAC,CAKnB"}

package/dist/eval/from-turn.js

@@ -1,3 +1,4 @@
+import { compareAnswerToOutcome } from '../runtime/query/turn/answer-claims.js';
 /**
  * Project a finished `Turn` into the shape scorers consume.
  *
@@ -9,6 +10,7 @@
  */
 export function evalTurnFromTurn(turn, opts = {}) {
     const steps = turn.steps ?? [];
+    const claimMismatch = compareAnswerToOutcome(turn.result, turn.outcome);
     return {
         output: turn.result ?? null,
         ...(turn.structuredOutput !== undefined ? { structuredOutput: turn.structuredOutput } : {}),
@@ -20,6 +22,9 @@ export function evalTurnFromTurn(turn, opts = {}) {
         totalCostUsd: turn.costInfo.totalCost,
         durationMs: opts.durationMs ?? (turn.endedAt ?? Date.now()) - turn.startedAt,
         ...(turn.lastError ? { error: turn.lastError } : {}),
+        ...(turn.outcome ? { outcome: turn.outcome } : {}),
+        ...(claimMismatch ? { claimMismatch } : {}),
+        ...(turn.handedBack ? { handedBack: turn.handedBack } : {}),
     };
 }
 /**

package/dist/eval/from-turn.js.map

@@ -1 +1 @@
-{"version":3,"file":"from-turn.js","sourceRoot":"","sources":["../../src/eval/from-turn.ts"],"names":[],"mappings":"AAGA;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAU,EAAE,OAAgC,EAAE;IAC9E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAA;IAC9B,OAAO;QACN,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;QAC3B,GAAG,CAAC,IAAI,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3F,KAAK;QACL,yDAAyD;QACzD,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxE,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,WAAW;QACxC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;QACrC,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS;QAC5E,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpD,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,SAAwC;IAExC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;IACjC,OAAO,CAAC,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;IAChD,OAAO,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC,CAAA;AAC5E,CAAC"}
+{"version":3,"file":"from-turn.js","sourceRoot":"","sources":["../../src/eval/from-turn.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,sBAAsB,EAAE,MAAM,wCAAwC,CAAA;AAI/E;;;;;;;;GAQG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAU,EAAE,OAAgC,EAAE;IAC9E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,EAAE,CAAA;IAC9B,MAAM,aAAa,GAAG,sBAAsB,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;IACvE,OAAO;QACN,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,IAAI;QAC3B,GAAG,CAAC,IAAI,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,IAAI,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3F,KAAK;QACL,yDAAyD;QACzD,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACxE,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3D,WAAW,EAAE,IAAI,CAAC,UAAU,CAAC,WAAW;QACxC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,SAAS;QACrC,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS;QAC5E,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACpD,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClD,GAAG,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC3D,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,SAAwC;IAExC,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,IAAI,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;IACjC,OAAO,CAAC,IAAI,CAAC,IAAI;QAAE,IAAI,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,CAAA;IAChD,OAAO,gBAAgB,CAAC,IAAI,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE,CAAC,CAAA;AAC5E,CAAC"}

package/dist/eval/index.d.ts

@@ -6,8 +6,19 @@ export { formatReport, runExperiment } from './experiment.js';
 export type { ExperimentConfig } from './experiment.js';
 export { judgeScorer } from './judge.js';
 export type { JudgeScorerConfig } from './judge.js';
-export { completionScorer, containsScorer, customScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
+export { claimMismatchScorer, handedBackScorer, completionScorer, containsScorer, customScorer, outcomeScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
+export { BOOTSTRAP_RESAMPLES, DEFAULT_EVAL_SEED, EvalUsageError, INTERVAL_LEVEL, MIN_CASES_FOR_INTERVAL, bootstrapCases, bootstrapPairedDifference, createSeededRandom, exactSignTestP, isEvalUsageError, } from './bootstrap.js';
+export type { BootstrapMethod, CaseIntervals, CaseTally, DifferenceIntervals, Interval, PairedTally, } from './bootstrap.js';
+export type { ReportDisclosure, RoleDisclosure, RoutingDisclosure, RunMode, SuiteFacts, } from './disclosure.js';
+export { sdkVersion } from './disclosure.js';
+export { currentEvalRunOptions, withEvalRunOptions } from './run-options.js';
+export type { EvalRunOptions } from './run-options.js';
+export { formatRunSummary, summarizeRun } from './run-summary.js';
+export type { RunDisclosure, RunSummary } from './run-summary.js';
+export type { RepeatReliability, SplitSummary } from './report-stats.js';
+export { compareEvalReports, formatEvalComparison, loadEvalReportFile } from './compare.js';
+export type { CaseRef, DisclosureComparison, DisclosureDifference, DisclosureDimension, DisclosureUnknown, EvalComparison, Flips, LoadedReportFile, LoadedSuite, SideNumbers, } from './compare.js';
 export { describeUncertainty, uncertaintyOf } from './uncertainty.js';
 export type { ScoreUncertainty } from './uncertainty.js';
-export type { CaseResult, CaseStatus, EvalCase, EvalTurn, ExperimentReport, Score, Scorer, } from './types.js';
+export type { CaseRepeatSummary, CaseResult, EvalSplit, CaseStatus, EvalCase, EvalTurn, ExperimentReport, Score, Scorer, } from './types.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AACxF,YAAY,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAA;AAC5F,YAAY,EACX,YAAY,EACZ,kBAAkB,EAClB,wBAAwB,EACxB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,aAAa,GACb,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC7D,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AACxC,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EACN,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AACrE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACxD,YAAY,EACX,UAAU,EACV,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,MAAM,GACN,MAAM,YAAY,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AACxF,YAAY,EAAE,qBAAqB,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAA;AAC5F,YAAY,EACX,YAAY,EACZ,kBAAkB,EAClB,wBAAwB,EACxB,qBAAqB,EACrB,qBAAqB,EACrB,iBAAiB,EACjB,aAAa,GACb,MAAM,2BAA2B,CAAA;AAClC,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC7D,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AACxC,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAA;AACnD,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,yBAAyB,EACzB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,GAChB,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACX,eAAe,EACf,aAAa,EACb,SAAS,EACT,mBAAmB,EACnB,QAAQ,EACR,WAAW,GACX,MAAM,gBAAgB,CAAA;AACvB,YAAY,EACX,gBAAgB,EAChB,cAAc,EACd,iBAAiB,EACjB,OAAO,EACP,UAAU,GACV,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAC5E,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACtD,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AACjE,YAAY,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAA;AACjE,YAAY,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AACxE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAC3F,YAAY,EACX,OAAO,EACP,oBAAoB,EACpB,oBAAoB,EACpB,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,KAAK,EACL,gBAAgB,EAChB,WAAW,EACX,WAAW,GACX,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AACrE,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AACxD,YAAY,EACX,iBAAiB,EACjB,UAAU,EACV,SAAS,EACT,UAAU,EACV,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAChB,KAAK,EACL,MAAM,GACN,MAAM,YAAY,CAAA"}

package/dist/eval/index.js

@@ -2,6 +2,11 @@ export { evalTurnFromQuery, evalTurnFromTurn } from './from-turn.js';
 export { compareHarnessTrials, reviewHarnessCandidate } from './harness-verification.js';
 export { formatReport, runExperiment } from './experiment.js';
 export { judgeScorer } from './judge.js';
-export { completionScorer, containsScorer, customScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
+export { claimMismatchScorer, handedBackScorer, completionScorer, containsScorer, customScorer, outcomeScorer, stepBudgetScorer, trajectoryScorer, } from './scorers.js';
+export { BOOTSTRAP_RESAMPLES, DEFAULT_EVAL_SEED, EvalUsageError, INTERVAL_LEVEL, MIN_CASES_FOR_INTERVAL, bootstrapCases, bootstrapPairedDifference, createSeededRandom, exactSignTestP, isEvalUsageError, } from './bootstrap.js';
+export { sdkVersion } from './disclosure.js';
+export { currentEvalRunOptions, withEvalRunOptions } from './run-options.js';
+export { formatRunSummary, summarizeRun } from './run-summary.js';
+export { compareEvalReports, formatEvalComparison, loadEvalReportFile } from './compare.js';
 export { describeUncertainty, uncertaintyOf } from './uncertainty.js';
 //# sourceMappingURL=index.js.map

package/dist/eval/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AAWxF,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,OAAO,EACN,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAA;AACpE,OAAO,EAAE,oBAAoB,EAAE,sBAAsB,EAAE,MAAM,2BAA2B,CAAA;AAWxF,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAE7D,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAA;AAExC,OAAO,EACN,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,gBAAgB,GAChB,MAAM,cAAc,CAAA;AACrB,OAAO,EACN,mBAAmB,EACnB,iBAAiB,EACjB,cAAc,EACd,cAAc,EACd,sBAAsB,EACtB,cAAc,EACd,yBAAyB,EACzB,kBAAkB,EAClB,cAAc,EACd,gBAAgB,GAChB,MAAM,gBAAgB,CAAA;AAgBvB,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC5C,OAAO,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAA;AAE5E,OAAO,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAA;AAGjE,OAAO,EAAE,kBAAkB,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,cAAc,CAAA;AAa3F,OAAO,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA"}

package/dist/eval/report-stats.d.ts

@@ -0,0 +1,77 @@
+import { type CaseIntervals, type CaseTally } from './bootstrap.js';
+import type { CaseRepeatSummary, CaseResult, EvalSplit } from './types.js';
+/**
+ * The numbers a report carries about a set of runs, whether that set is one
+ * suite, one split of it, or every suite in a run.
+ *
+ * Cases are the unit. A case with several repeats is one case with several
+ * runs: the pass rate pools the runs, the intervals resample the cases, and
+ * reliability asks of each case whether it held up every time.
+ */
+export interface RepeatReliability {
+    /** Cases that ran. */
+    cases: number;
+    /** Cases whose every repeat passed. */
+    passedEveryRepeat: number;
+    /** `passedEveryRepeat / cases`; null when no case ran. */
+    fraction: number | null;
+}
+export interface SplitSummary {
+    cases: number;
+    runs: number;
+    passed: number;
+    failed: number;
+    inconclusive: number;
+    passRate: number | null;
+    mean: number | null;
+    intervals: CaseIntervals;
+}
+/** The runs of one case, with the key that pairs it with the same case elsewhere. */
+export interface CaseRuns {
+    readonly key: string;
+    readonly name: string;
+    readonly split?: EvalSplit;
+    readonly runs: readonly CaseResult[];
+}
+export interface RunStats {
+    cases: number;
+    runs: number;
+    passed: number;
+    failed: number;
+    inconclusive: number;
+    passRate: number | null;
+    mean: number | null;
+    intervals: CaseIntervals;
+    caseSummaries: CaseRepeatSummary[];
+    reliability: RepeatReliability;
+    splits: Partial<Record<EvalSplit | 'unassigned', SplitSummary>>;
+}
+export declare const SPLIT_ORDER: readonly ["development", "held-out", "unassigned"];
+/** Group a flat list of runs by case name, in the order the cases first appear. */
+export declare function groupRuns(results: readonly CaseResult[], keyOf?: (name: string) => string): CaseRuns[];
+/** The first name that appears more than once, or undefined. */
+export declare function duplicateName(names: readonly string[]): string | undefined;
+export declare function duplicateMessage(suite: string, name: string): string;
+/**
+ * The first case in a list of runs that appears twice for the same repeat: a
+ * second run of a case is a repeat only when it says it is a later one.
+ */
+export declare function duplicateRun(runs: readonly Pick<CaseResult, 'case' | 'repeat'>[]): string | undefined;
+/** Status of a run, tolerating a report old enough to carry only `passed`. */
+export declare function statusOf(run: Pick<CaseResult, 'status' | 'passed'>): CaseResult['status'];
+export declare function tallyOf(runs: readonly CaseResult[]): CaseTally | null;
+/** True when the case has at least one judged run and every run passed. */
+export declare function passedEveryRun(runs: readonly CaseResult[]): boolean;
+export declare function statsOver(cases: readonly CaseRuns[], seed: number): RunStats;
+/**
+ * The intervals, as lines a person reads next to the numbers. A number whose
+ * interval is not computed is printed with the reason, never alone.
+ */
+export declare function describeIntervals(args: {
+    mean: number | null;
+    passRate: number | null;
+    intervals: CaseIntervals;
+}): string[];
+/** The line that says which splits a suite has, and says so plainly when none is held out. */
+export declare function describeSplits(splits: RunStats['splits']): string[];
+//# sourceMappingURL=report-stats.d.ts.map

package/dist/eval/report-stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-stats.d.ts","sourceRoot":"","sources":["../../src/eval/report-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,aAAa,EAAE,KAAK,SAAS,EAAkB,MAAM,gBAAgB,CAAA;AACnF,OAAO,KAAK,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAE1E;;;;;;;GAOG;AAEH,MAAM,WAAW,iBAAiB;IACjC,sBAAsB;IACtB,KAAK,EAAE,MAAM,CAAA;IACb,uCAAuC;IACvC,iBAAiB,EAAE,MAAM,CAAA;IACzB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;CACvB;AAED,MAAM,WAAW,YAAY;IAC5B,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,SAAS,EAAE,aAAa,CAAA;CACxB;AAED,qFAAqF;AACrF,MAAM,WAAW,QAAQ;IACxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,CAAA;IAC1B,QAAQ,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,CAAA;CACpC;AAED,MAAM,WAAW,QAAQ;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,SAAS,EAAE,aAAa,CAAA;IACxB,aAAa,EAAE,iBAAiB,EAAE,CAAA;IAClC,WAAW,EAAE,iBAAiB,CAAA;IAC9B,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,YAAY,EAAE,YAAY,CAAC,CAAC,CAAA;CAC/D;AAED,eAAO,MAAM,WAAW,oDAAqD,CAAA;AAE7E,mFAAmF;AACnF,wBAAgB,SAAS,CACxB,OAAO,EAAE,SAAS,UAAU,EAAE,EAC9B,KAAK,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAuB,GAC9C,QAAQ,EAAE,CAYZ;AAED,gEAAgE;AAChE,wBAAgB,aAAa,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,GAAG,SAAS,CAO1E;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAC3B,IAAI,EAAE,SAAS,IAAI,CAAC,UAAU,EAAE,MAAM,GAAG,QAAQ,CAAC,EAAE,GAClD,MAAM,GAAG,SAAS,CAQpB;AAED,8EAA8E;AAC9E,wBAAgB,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,GAAG,QAAQ,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAKzF;AAED,wBAAgB,OAAO,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,GAAG,SAAS,GAAG,IAAI,CAYrE;AAED,2EAA2E;AAC3E,wBAAgB,cAAc,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,GAAG,OAAO,CAEnE;AAmCD,wBAAgB,SAAS,CAAC,KAAK,EAAE,SAAS,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,QAAQ,CAmC5E;AAKD;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE;IACvC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAA;IACvB,SAAS,EAAE,aAAa,CAAA;CACxB,GAAG,MAAM,EAAE,CAsBX;AAED,8FAA8F;AAC9F,wBAAgB,cAAc,CAAC,MAAM,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,MAAM,EAAE,CAmBnE"}

package/dist/eval/report-stats.js

@@ -0,0 +1,184 @@
+import { bootstrapCases } from './bootstrap.js';
+export const SPLIT_ORDER = ['development', 'held-out', 'unassigned'];
+/** Group a flat list of runs by case name, in the order the cases first appear. */
+export function groupRuns(results, keyOf = (name) => name) {
+    const groups = new Map();
+    for (const run of results) {
+        const key = keyOf(run.case);
+        let group = groups.get(key);
+        if (!group) {
+            group = { name: run.case, ...(run.split ? { split: run.split } : {}), runs: [] };
+            groups.set(key, group);
+        }
+        group.runs.push(run);
+    }
+    return [...groups].map(([key, g]) => ({ key, ...g }));
+}
+/** The first name that appears more than once, or undefined. */
+export function duplicateName(names) {
+    const seen = new Set();
+    for (const name of names) {
+        if (seen.has(name))
+            return name;
+        seen.add(name);
+    }
+    return undefined;
+}
+export function duplicateMessage(suite, name) {
+    return `suite "${suite}" has the case "${name}" more than once; a case is named by its id, and two cases sharing one would be merged into one in every count and interval. Give each case its own name.`;
+}
+/**
+ * The first case in a list of runs that appears twice for the same repeat: a
+ * second run of a case is a repeat only when it says it is a later one.
+ */
+export function duplicateRun(runs) {
+    const seen = new Set();
+    for (const run of runs) {
+        const key = `${run.case}\u0000${run.repeat ?? 1}`;
+        if (seen.has(key))
+            return run.case;
+        seen.add(key);
+    }
+    return undefined;
+}
+/** Status of a run, tolerating a report old enough to carry only `passed`. */
+export function statusOf(run) {
+    if (run.status === 'passed' || run.status === 'failed' || run.status === 'inconclusive') {
+        return run.status;
+    }
+    return run.passed === true ? 'passed' : 'failed';
+}
+export function tallyOf(runs) {
+    let scoreSum = 0;
+    let passes = 0;
+    let judged = 0;
+    for (const run of runs) {
+        const status = statusOf(run);
+        if (status === 'inconclusive')
+            continue;
+        judged++;
+        scoreSum += run.mean;
+        if (status === 'passed')
+            passes++;
+    }
+    return judged === 0 ? null : { scoreSum, passes, judged };
+}
+/** True when the case has at least one judged run and every run passed. */
+export function passedEveryRun(runs) {
+    return runs.length > 0 && runs.every((run) => statusOf(run) === 'passed');
+}
+function summaryOf(runs) {
+    let passed = 0;
+    let failed = 0;
+    let inconclusive = 0;
+    for (const run of runs) {
+        const status = statusOf(run);
+        if (status === 'passed')
+            passed++;
+        else if (status === 'failed')
+            failed++;
+        else
+            inconclusive++;
+    }
+    return { passed, failed, inconclusive };
+}
+function numbersOver(cases, seed) {
+    const tallies = [];
+    let runs = 0;
+    for (const c of cases) {
+        runs += c.runs.length;
+        const tally = tallyOf(c.runs);
+        if (tally)
+            tallies.push(tally);
+    }
+    const counts = summaryOf(cases.flatMap((c) => c.runs));
+    const judged = counts.passed + counts.failed;
+    const scoreSum = tallies.reduce((sum, t) => sum + t.scoreSum, 0);
+    return {
+        runs,
+        ...counts,
+        passRate: judged === 0 ? null : counts.passed / judged,
+        mean: judged === 0 ? null : scoreSum / judged,
+        intervals: bootstrapCases(tallies, seed),
+    };
+}
+export function statsOver(cases, seed) {
+    const whole = numbersOver(cases, seed);
+    const caseSummaries = cases.map((c) => {
+        const counts = summaryOf(c.runs);
+        const tally = tallyOf(c.runs);
+        return {
+            case: c.name,
+            ...(c.split ? { split: c.split } : {}),
+            repeats: c.runs.length,
+            ...counts,
+            mean: tally ? tally.scoreSum / tally.judged : null,
+            passedEveryRepeat: passedEveryRun(c.runs),
+        };
+    });
+    const everyRepeat = caseSummaries.filter((c) => c.passedEveryRepeat).length;
+    const splits = {};
+    for (const name of SPLIT_ORDER) {
+        const members = cases.filter((c) => (c.split ?? 'unassigned') === name);
+        if (members.length === 0)
+            continue;
+        const part = numbersOver(members, seed);
+        splits[name] = { cases: members.length, ...part };
+    }
+    return {
+        cases: cases.length,
+        ...whole,
+        caseSummaries,
+        reliability: {
+            cases: cases.length,
+            passedEveryRepeat: everyRepeat,
+            fraction: cases.length === 0 ? null : everyRepeat / cases.length,
+        },
+        splits,
+    };
+}
+const percent = (value) => `${(value * 100).toFixed(1)}%`;
+const score = (value) => value.toFixed(3);
+/**
+ * The intervals, as lines a person reads next to the numbers. A number whose
+ * interval is not computed is printed with the reason, never alone.
+ */
+export function describeIntervals(args) {
+    const { intervals } = args;
+    const line = (label, estimate, interval, show) => {
+        if (estimate === null)
+            return `${label}: none, no run was judged`;
+        if (!interval.computed)
+            return `${label} ${show(estimate)}, no interval: ${interval.reason}`;
+        return `${label} ${show(estimate)}, 95% interval ${show(interval.low)} to ${show(interval.high)}`;
+    };
+    const lines = [
+        line('mean score', args.mean, intervals.mean, score),
+        line('pass rate', args.passRate, intervals.passRate, percent),
+    ];
+    if (intervals.mean.computed || intervals.passRate.computed) {
+        lines.push(`intervals: ${intervals.method.kind} (${intervals.method.cases} cases, ${intervals.method.resamples} resamples), seed ${intervals.method.seed}`);
+    }
+    return lines;
+}
+/** The line that says which splits a suite has, and says so plainly when none is held out. */
+export function describeSplits(splits) {
+    const lines = [];
+    // A suite that declares no split has one bucket and nothing to break down.
+    const declared = SPLIT_ORDER.some((name) => name !== 'unassigned' && splits[name]);
+    for (const name of SPLIT_ORDER) {
+        const part = splits[name];
+        if (!part || !declared)
+            continue;
+        const rate = part.passRate === null ? 'no judged run' : `pass rate ${percent(part.passRate)}`;
+        const interval = part.intervals.passRate.computed
+            ? `, 95% interval ${percent(part.intervals.passRate.low)} to ${percent(part.intervals.passRate.high)}`
+            : '';
+        lines.push(`${name}: ${part.cases} case${part.cases === 1 ? '' : 's'}, ${rate}${interval}`);
+    }
+    if (!splits['held-out']) {
+        lines.push('no held-out cases: every number here is measured on cases the suite was developed against');
+    }
+    return lines;
+}
+//# sourceMappingURL=report-stats.js.map

package/dist/eval/report-stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-stats.js","sourceRoot":"","sources":["../../src/eval/report-stats.ts"],"names":[],"mappings":"AAAA,OAAO,EAAsC,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAsDnF,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,aAAa,EAAE,UAAU,EAAE,YAAY,CAAU,CAAA;AAE7E,mFAAmF;AACnF,MAAM,UAAU,SAAS,CACxB,OAA8B,EAC9B,QAAkC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI;IAEhD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAmE,CAAA;IACzF,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC3B,IAAI,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC3B,IAAI,CAAC,KAAK,EAAE,CAAC;YACZ,KAAK,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;YAChF,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IACrB,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;AACtD,CAAC;AAED,gEAAgE;AAChE,MAAM,UAAU,aAAa,CAAC,KAAwB;IACrD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;QAC/B,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IACf,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa,EAAE,IAAY;IAC3D,OAAO,UAAU,KAAK,mBAAmB,IAAI,2JAA2J,CAAA;AACzM,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAC3B,IAAoD;IAEpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,IAAI,SAAS,GAAG,CAAC,MAAM,IAAI,CAAC,EAAE,CAAA;QACjD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,GAAG,CAAC,IAAI,CAAA;QAClC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACd,CAAC;IACD,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,QAAQ,CAAC,GAA0C;IAClE,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,IAAI,GAAG,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;QACzF,OAAO,GAAG,CAAC,MAAM,CAAA;IAClB,CAAC;IACD,OAAO,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAA;AACjD,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,IAA2B;IAClD,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,KAAK,cAAc;YAAE,SAAQ;QACvC,MAAM,EAAE,CAAA;QACR,QAAQ,IAAI,GAAG,CAAC,IAAI,CAAA;QACpB,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;IAClC,CAAC;IACD,OAAO,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;AAC1D,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,cAAc,CAAC,IAA2B;IACzD,OAAO,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,QAAQ,CAAC,CAAA;AAC1E,CAAC;AAED,SAAS,SAAS,CAAC,IAA2B;IAC7C,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAA;QAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;aAC5B,IAAI,MAAM,KAAK,QAAQ;YAAE,MAAM,EAAE,CAAA;;YACjC,YAAY,EAAE,CAAA;IACpB,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,CAAA;AACxC,CAAC;AAED,SAAS,WAAW,CAAC,KAA0B,EAAE,IAAY;IAC5D,MAAM,OAAO,GAAgB,EAAE,CAAA;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAA;QACrB,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC7B,IAAI,KAAK;YAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IAC/B,CAAC;IACD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IACtD,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAA;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAA;IAChE,OAAO;QACN,IAAI;QACJ,GAAG,MAAM;QACT,QAAQ,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,MAAM;QACtD,IAAI,EAAE,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,GAAG,MAAM;QAC7C,SAAS,EAAE,cAAc,CAAC,OAAO,EAAE,IAAI,CAAC;KACxC,CAAA;AACF,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAA0B,EAAE,IAAY;IACjE,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;IACtC,MAAM,aAAa,GAAwB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAC1D,MAAM,MAAM,GAAG,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAChC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAA;QAC7B,OAAO;YACN,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM;YACtB,GAAG,MAAM;YACT,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;YAClD,iBAAiB,EAAE,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC;SACzC,CAAA;IACF,CAAC,CAAC,CAAA;IACF,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAA;IAE3E,MAAM,MAAM,GAAuB,EAAE,CAAA;IACrC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK,IAAI,CAAC,CAAA;QACvE,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,SAAQ;QAClC,MAAM,IAAI,GAAG,WAAW,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QACvC,MAAM,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,IAAI,EAAE,CAAA;IAClD,CAAC;IAED,OAAO;QACN,KAAK,EAAE,KAAK,CAAC,MAAM;QACnB,GAAG,KAAK;QACR,aAAa;QACb,WAAW,EAAE;YACZ,KAAK,EAAE,KAAK,CAAC,MAAM;YACnB,iBAAiB,EAAE,WAAW;YAC9B,QAAQ,EAAE,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,GAAG,KAAK,CAAC,MAAM;SAChE;QACD,MAAM;KACN,CAAA;AACF,CAAC;AAED,MAAM,OAAO,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AACzE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;AAEzD;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAIjC;IACA,MAAM,EAAE,SAAS,EAAE,GAAG,IAAI,CAAA;IAC1B,MAAM,IAAI,GAAG,CACZ,KAAa,EACb,QAAuB,EACvB,QAA+B,EAC/B,IAA2B,EAClB,EAAE;QACX,IAAI,QAAQ,KAAK,IAAI;YAAE,OAAO,GAAG,KAAK,2BAA2B,CAAA;QACjE,IAAI,CAAC,QAAQ,CAAC,QAAQ;YAAE,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,QAAQ,CAAC,MAAM,EAAE,CAAA;QAC5F,OAAO,GAAG,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAA;IAClG,CAAC,CAAA;IACD,MAAM,KAAK,GAAG;QACb,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC;QACpD,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC;KAC7D,CAAA;IACD,IAAI,SAAS,CAAC,IAAI,CAAC,QAAQ,IAAI,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC5D,KAAK,CAAC,IAAI,CACT,cAAc,SAAS,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,MAAM,CAAC,KAAK,WAAW,SAAS,CAAC,MAAM,CAAC,SAAS,qBAAqB,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAC/I,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8FAA8F;AAC9F,MAAM,UAAU,cAAc,CAAC,MAA0B;IACxD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,2EAA2E;IAC3E,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,YAAY,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,CAAA;IAClF,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,CAAC,IAAI,IAAI,CAAC,QAAQ;YAAE,SAAQ;QAChC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,aAAa,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAA;QAC7F,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ;YAChD,CAAC,CAAC,kBAAkB,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;YACtG,CAAC,CAAC,EAAE,CAAA;QACL,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,KAAK,IAAI,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,IAAI,GAAG,QAAQ,EAAE,CAAC,CAAA;IAC5F,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CACT,2FAA2F,CAC3F,CAAA;IACF,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC"}

package/dist/eval/run-options.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Settings an operator gives a whole eval run, handed to every
+ * `runExperiment` that executes inside it.
+ *
+ * A suite file calls `runExperiment` itself and returns the report, so
+ * `namzu eval --repeat 5` has no argument to pass the five through. Rather
+ * than ask every suite to forward it, the runner opens a scope with
+ * {@link withEvalRunOptions} and `runExperiment` reads the scope. A suite
+ * that was written before the option existed repeats like any other.
+ *
+ * An option given here wins over the same option in the suite's own config:
+ * the person running the command asked for it.
+ */
+export interface EvalRunOptions {
+    /** Times every case runs. */
+    readonly repeats?: number;
+    /** Seed for the bootstrap. */
+    readonly seed?: number;
+    /** Version of the program that is running the suites, for the disclosure. */
+    readonly harnessVersion?: string;
+    /** Source revision the suites came from, when known. */
+    readonly commit?: string;
+    /** Version of the suite being run, when known. */
+    readonly suiteVersion?: string;
+}
+/** Run `work` with `options` in force for every experiment it starts. */
+export declare function withEvalRunOptions<T>(options: EvalRunOptions, work: () => T): T;
+/** The options in force here, or an empty set outside any scope. */
+export declare function currentEvalRunOptions(): EvalRunOptions;
+//# sourceMappingURL=run-options.d.ts.map

package/dist/eval/run-options.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-options.d.ts","sourceRoot":"","sources":["../../src/eval/run-options.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,cAAc;IAC9B,6BAA6B;IAC7B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;IACzB,8BAA8B;IAC9B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;IACtB,6EAA6E;IAC7E,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAA;IAChC,wDAAwD;IACxD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,kDAAkD;IAClD,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC9B;AAcD,yEAAyE;AACzE,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,OAAO,EAAE,cAAc,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,CAE/E;AAED,oEAAoE;AACpE,wBAAgB,qBAAqB,IAAI,cAAc,CAEtD"}