@namzu/sdk 62.12.0 → 62.14.0

package/dist/eval/run-options.js

@@ -0,0 +1,20 @@
+import { AsyncLocalStorage } from 'node:async_hooks';
+const STORE_KEY = Symbol.for('namzu.eval.runOptions');
+/**
+ * One store per process, found through a global symbol, so a suite that
+ * resolves its own copy of the SDK still sees the scope the runner opened.
+ */
+function store() {
+    const holder = globalThis;
+    holder[STORE_KEY] ??= new AsyncLocalStorage();
+    return holder[STORE_KEY];
+}
+/** Run `work` with `options` in force for every experiment it starts. */
+export function withEvalRunOptions(options, work) {
+    return store().run(options, work);
+}
+/** The options in force here, or an empty set outside any scope. */
+export function currentEvalRunOptions() {
+    return store().getStore() ?? {};
+}
+//# sourceMappingURL=run-options.js.map

package/dist/eval/run-options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-options.js","sourceRoot":"","sources":["../../src/eval/run-options.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAA;AA4BpD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAA;AAErD;;;GAGG;AACH,SAAS,KAAK;IACb,MAAM,MAAM,GAAG,UAAiE,CAAA;IAChF,MAAM,CAAC,SAAS,CAAC,KAAK,IAAI,iBAAiB,EAAkB,CAAA;IAC7D,OAAO,MAAM,CAAC,SAAS,CAAC,CAAA;AACzB,CAAC;AAED,yEAAyE;AACzE,MAAM,UAAU,kBAAkB,CAAI,OAAuB,EAAE,IAAa;IAC3E,OAAO,KAAK,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;AAClC,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,qBAAqB;IACpC,OAAO,KAAK,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAA;AAChC,CAAC"}

package/dist/eval/run-summary.d.ts

@@ -0,0 +1,36 @@
+import type { RunMode } from './disclosure.js';
+import { type RunStats } from './report-stats.js';
+import type { ExperimentReport } from './types.js';
+/** What produced a whole run, across every suite in it. */
+export interface RunDisclosure {
+    readonly harnessVersion: string;
+    readonly sdkVersion: string;
+    readonly generatedAt: string;
+    readonly commit: string;
+    readonly seed: number;
+    /** Repeats per case; the largest across suites when they differ. */
+    readonly repeats: number;
+    readonly suites: number;
+    readonly cases: number;
+    readonly runs: number;
+    /** `mixed` when suites that used a model differ in kind; a suite with no model at all does not count. */
+    readonly mode: RunMode | 'mixed';
+}
+/** The numbers for every case of every suite taken together. */
+export interface RunSummary extends Omit<RunStats, 'caseSummaries' | 'reliability'> {
+    readonly disclosure: RunDisclosure;
+    /** Present only when cases were repeated. */
+    readonly reliability?: RunStats['reliability'];
+}
+export declare function summarizeRun(entries: ReadonlyArray<{
+    suite: string;
+    report: ExperimentReport;
+}>, context: {
+    seed: number;
+    harnessVersion: string;
+    commit: string;
+    generatedAt: string;
+    sdkVersion: string;
+}): RunSummary;
+export declare function formatRunSummary(summary: RunSummary): string;
+//# sourceMappingURL=run-summary.d.ts.map

package/dist/eval/run-summary.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-summary.d.ts","sourceRoot":"","sources":["../../src/eval/run-summary.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAC9C,OAAO,EACN,KAAK,QAAQ,EAKb,MAAM,mBAAmB,CAAA;AAC1B,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAElD,2DAA2D;AAC3D,MAAM,WAAW,aAAa;IAC7B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;IAC5B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,oEAAoE;IACpE,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,yGAAyG;IACzG,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,OAAO,CAAA;CAChC;AAED,gEAAgE;AAChE,MAAM,WAAW,UAAW,SAAQ,IAAI,CAAC,QAAQ,EAAE,eAAe,GAAG,aAAa,CAAC;IAClF,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAA;IAClC,6CAA6C;IAC7C,QAAQ,CAAC,WAAW,CAAC,EAAE,QAAQ,CAAC,aAAa,CAAC,CAAA;CAC9C;AAED,wBAAgB,YAAY,CAC3B,OAAO,EAAE,aAAa,CAAC;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,CAAC,EACnE,OAAO,EAAE;IACR,IAAI,EAAE,MAAM,CAAA;IACZ,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,MAAM,CAAA;IACd,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;CAClB,GACC,UAAU,CAmCZ;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,UAAU,GAAG,MAAM,CAuB5D"}

package/dist/eval/run-summary.js

@@ -0,0 +1,57 @@
+import { UNKNOWN } from './disclosure.js';
+import { describeIntervals, describeSplits, groupRuns, statsOver, } from './report-stats.js';
+export function summarizeRun(entries, context) {
+    // A case is named by its suite and its id: two suites may reuse a name.
+    const cases = entries.flatMap(({ suite, report }) => groupRuns(report.cases ?? [], (name) => `${suite}\u0000${name}`));
+    const { caseSummaries: _perCase, reliability, ...stats } = statsOver(cases, context.seed);
+    const repeats = Math.max(1, ...entries.map((e) => e.report.repeats ?? 1));
+    // A suite that used no model at all says nothing about whether a model
+    // ran, so it does not make a run "mixed" with the suites that did.
+    const all = entries.map((e) => e.report.disclosure?.mode ?? UNKNOWN);
+    const modes = new Set(all.filter((m) => m !== 'no-model'));
+    const mode = modes.size === 0
+        ? all.length === 0
+            ? UNKNOWN
+            : 'no-model'
+        : modes.size === 1
+            ? [...modes][0]
+            : 'mixed';
+    return {
+        ...stats,
+        ...(repeats > 1 ? { reliability } : {}),
+        disclosure: {
+            harnessVersion: context.harnessVersion,
+            sdkVersion: context.sdkVersion,
+            generatedAt: context.generatedAt,
+            commit: context.commit,
+            seed: context.seed,
+            repeats,
+            suites: entries.length,
+            cases: stats.cases,
+            runs: stats.runs,
+            mode,
+        },
+    };
+}
+export function formatRunSummary(summary) {
+    const d = summary.disclosure;
+    const lines = [
+        `Whole run: ${summary.passed}/${summary.runs} runs passed across ${d.suites} suite${d.suites === 1 ? '' : 's'}, ${summary.cases} case${summary.cases === 1 ? '' : 's'}${d.repeats > 1 ? ` x ${d.repeats} repeats` : ''}`,
+    ];
+    for (const line of describeIntervals({
+        mean: summary.mean,
+        passRate: summary.passRate,
+        intervals: summary.intervals,
+    })) {
+        lines.push(`  ${line}`);
+    }
+    if (summary.reliability) {
+        const r = summary.reliability;
+        lines.push(`  reliability across repeats: ${r.passedEveryRepeat} of ${r.cases} cases passed in every one of ${d.repeats} repeats${r.fraction === null ? '' : ` (${(r.fraction * 100).toFixed(1)}%)`}`);
+    }
+    for (const line of describeSplits(summary.splits))
+        lines.push(`  ${line}`);
+    lines.push(`  produced by: namzu eval ${d.harnessVersion}, sdk ${d.sdkVersion}, commit ${d.commit}, ${d.generatedAt}, ${d.mode}`);
+    return lines.join('\n');
+}
+//# sourceMappingURL=run-summary.js.map

package/dist/eval/run-summary.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"run-summary.js","sourceRoot":"","sources":["../../src/eval/run-summary.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAEzC,OAAO,EAEN,iBAAiB,EACjB,cAAc,EACd,SAAS,EACT,SAAS,GACT,MAAM,mBAAmB,CAAA;AA0B1B,MAAM,UAAU,YAAY,CAC3B,OAAmE,EACnE,OAMC;IAED,wEAAwE;IACxE,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CACnD,SAAS,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,KAAK,SAAS,IAAI,EAAE,CAAC,CAChE,CAAA;IACD,MAAM,EAAE,aAAa,EAAE,QAAQ,EAAE,WAAW,EAAE,GAAG,KAAK,EAAE,GAAG,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;IACzF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAA;IACzE,uEAAuE;IACvE,mEAAmE;IACnE,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,IAAI,IAAI,OAAO,CAAC,CAAA;IACpE,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,UAAU,CAAC,CAAC,CAAA;IAC1D,MAAM,IAAI,GACT,KAAK,CAAC,IAAI,KAAK,CAAC;QACf,CAAC,CAAC,GAAG,CAAC,MAAM,KAAK,CAAC;YACjB,CAAC,CAAC,OAAO;YACT,CAAC,CAAC,UAAU;QACb,CAAC,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC;YACjB,CAAC,CAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAa;YAC5B,CAAC,CAAC,OAAO,CAAA;IACZ,OAAO;QACN,GAAG,KAAK;QACR,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvC,UAAU,EAAE;YACX,cAAc,EAAE,OAAO,CAAC,cAAc;YACtC,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,OAAO;YACP,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,IAAI;SACJ;KACD,CAAA;AACF,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAmB;IACnD,MAAM,CAAC,GAAG,OAAO,CAAC,UAAU,CAAA;IAC5B,MAAM,KAAK,GAAG;QACb,cAAc,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,IAAI,uBAAuB,CAAC,CAAC,MAAM,SAAS,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,OAAO,CAAC,KAAK,QAAQ,OAAO,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,EAAE;KACxN,CAAA;IACD,KAAK,MAAM,IAAI,IAAI,iBAAiB,CAAC;QACpC,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ;QAC1B,SAAS,EAAE,OAAO,CAAC,SAAS;KAC5B,CAAC,EAAE,CAAC;QACJ,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IACxB,CAAC;IACD,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,OAAO,CAAC,WAAW,CAAA;QAC7B,KAAK,CAAC,IAAI,CACT,iCAAiC,CAAC,CAAC,iBAAiB,OAAO,CAAC,CAAC,KAAK,iCAAiC,CAAC,CAAC,OAAO,WAAW,CAAC,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAC1L,CAAA;IACF,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,cAAc,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC,CAAA;IAC1E,KAAK,CAAC,IAAI,CACT,6BAA6B,CAAC,CAAC,cAAc,SAAS,CAAC,CAAC,UAAU,YAAY,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,IAAI,EAAE,CACrH,CAAA;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC"}

package/dist/eval/scorers.d.ts

@@ -1,3 +1,5 @@
+import type { HandBackKind } from '../types/session/hand-back.js';
+import type { TurnOutcomeVerdict } from '../types/session/outcome.js';
 import type { EvalCase, EvalTurn, Score, Scorer } from './types.js';
 /**
  * How closely the turn's tool sequence matched the expected one, as F1 over
@@ -27,6 +29,36 @@ export declare function completionScorer(acceptable?: readonly string[]): Scorer
 export declare function stepBudgetScorer(maxSteps: number): Scorer;
 /** The final text contains every required substring. */
 export declare function containsScorer(...required: string[]): Scorer;
+/**
+ * The turn's outcome record has one of the given verdicts.
+ *
+ * Reads the record the kernel settled from what actually ran, not the answer's
+ * words. A turn with no record scores unavailable: there is nothing to read,
+ * which is not the same as a wrong verdict.
+ */
+export declare function outcomeScorer(verdicts: readonly TurnOutcomeVerdict[]): Scorer;
+/**
+ * Whether the answer's plain check claims agree with the turn's outcome
+ * record: 1 when `compareAnswerToOutcome` found no mismatch, 0 when it found
+ * one, with the claim and what the record holds in the detail.
+ *
+ * Pass `{ expectMismatch: true }` for a case that is built to contradict
+ * itself, so the suite proves the comparison fires as well as that it stays
+ * silent.
+ */
+export declare function claimMismatchScorer(opts?: {
+    expectMismatch?: boolean;
+}): Scorer;
+/**
+ * Score whether the turn was handed back to a person.
+ *
+ * Pass `{ expectKind }` for a case built so the model hands the task back, and
+ * the scorer checks it did, and for that kind; with nothing it checks the turn
+ * was not handed back. A gate: a scripted suite pins the kernel's side of it.
+ */
+export declare function handedBackScorer(opts?: {
+    expectKind?: HandBackKind;
+}): Scorer;
 /**
  * Judge the turn with a caller-supplied predicate.
  *

package/dist/eval/scorers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"scorers.d.ts","sourceRoot":"","sources":["../../src/eval/scorers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAyBnE;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,gBAAgB,IAAI,MAAM,CAwCzC;AAED,sEAAsE;AACtE,wBAAgB,gBAAgB,CAC/B,UAAU,GAAE,SAAS,MAAM,EAAmC,GAC5D,MAAM,CAqBR;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAezD;AAED,wDAAwD;AACxD,wBAAgB,cAAc,CAAC,GAAG,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAmB5D;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC3B,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,KAAK,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAChE,MAAM,CAER"}
+{"version":3,"file":"scorers.d.ts","sourceRoot":"","sources":["../../src/eval/scorers.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAA;AACjE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AACrE,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,YAAY,CAAA;AAyBnE;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,gBAAgB,IAAI,MAAM,CAwCzC;AAED,sEAAsE;AACtE,wBAAgB,gBAAgB,CAC/B,UAAU,GAAE,SAAS,MAAM,EAAmC,GAC5D,MAAM,CAqBR;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAezD;AAED,wDAAwD;AACxD,wBAAgB,cAAc,CAAC,GAAG,QAAQ,EAAE,MAAM,EAAE,GAAG,MAAM,CAmB5D;AAED;;;;;;GAMG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,SAAS,kBAAkB,EAAE,GAAG,MAAM,CAoB7E;AAED;;;;;;;;GAQG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,GAAE;IAAE,cAAc,CAAC,EAAE,OAAO,CAAA;CAAO,GAAG,MAAM,CAiBnF;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,GAAE;IAAE,UAAU,CAAC,EAAE,YAAY,CAAA;CAAO,GAAG,MAAM,CAejF;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAC3B,IAAI,EAAE,MAAM,EACZ,EAAE,EAAE,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,KAAK,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAChE,MAAM,CAER"}

package/dist/eval/scorers.js

@@ -136,6 +136,83 @@ export function containsScorer(...required) {
         },
     };
 }
+/**
+ * The turn's outcome record has one of the given verdicts.
+ *
+ * Reads the record the kernel settled from what actually ran, not the answer's
+ * words. A turn with no record scores unavailable: there is nothing to read,
+ * which is not the same as a wrong verdict.
+ */
+export function outcomeScorer(verdicts) {
+    return {
+        name: 'outcome',
+        // The record either says one of the verdicts or it does not.
+        severity: 'gate',
+        score(turn) {
+            const outcome = turn.outcome;
+            if (!outcome) {
+                return { score: 0, unavailable: true, reason: 'the turn has no outcome record' };
+            }
+            const ok = verdicts.includes(outcome.verdict);
+            return {
+                score: ok ? 1 : 0,
+                reason: ok
+                    ? `verdict ${outcome.verdict}`
+                    : `verdict ${outcome.verdict}, expected one of ${verdicts.join(', ')}`,
+                details: { verdict: outcome.verdict, expected: verdicts },
+            };
+        },
+    };
+}
+/**
+ * Whether the answer's plain check claims agree with the turn's outcome
+ * record: 1 when `compareAnswerToOutcome` found no mismatch, 0 when it found
+ * one, with the claim and what the record holds in the detail.
+ *
+ * Pass `{ expectMismatch: true }` for a case that is built to contradict
+ * itself, so the suite proves the comparison fires as well as that it stays
+ * silent.
+ */
+export function claimMismatchScorer(opts = {}) {
+    const expectMismatch = opts.expectMismatch === true;
+    return {
+        name: 'claim-mismatch',
+        severity: 'gate',
+        score(turn) {
+            const found = turn.claimMismatch;
+            const ok = expectMismatch ? found !== undefined : found === undefined;
+            return {
+                score: ok ? 1 : 0,
+                reason: found
+                    ? `${found.said}, but the record: ${found.recordSays}`
+                    : 'no claim in the answer contradicts the record',
+                details: { claimMismatch: found ?? null, expectMismatch },
+            };
+        },
+    };
+}
+/**
+ * Score whether the turn was handed back to a person.
+ *
+ * Pass `{ expectKind }` for a case built so the model hands the task back, and
+ * the scorer checks it did, and for that kind; with nothing it checks the turn
+ * was not handed back. A gate: a scripted suite pins the kernel's side of it.
+ */
+export function handedBackScorer(opts = {}) {
+    return {
+        name: 'handed-back',
+        severity: 'gate',
+        score(turn) {
+            const found = turn.handedBack;
+            const ok = opts.expectKind === undefined ? found === undefined : found?.kind === opts.expectKind;
+            return {
+                score: ok ? 1 : 0,
+                reason: found ? `handed back (${found.kind}): ${found.blocker}` : 'not handed back',
+                details: { handedBack: found ?? null, expectKind: opts.expectKind ?? null },
+            };
+        },
+    };
+}
 /**
  * Judge the turn with a caller-supplied predicate.
  *

package/dist/eval/scorers.js.map

@@ -1 +1 @@
-{"version":3,"file":"scorers.js","sourceRoot":"","sources":["../../src/eval/scorers.ts"],"names":[],"mappings":"AAEA;;;;;;GAMG;AACH,SAAS,SAAS,CAAC,CAAoB,EAAE,CAAoB;IAC5D,MAAM,KAAK,GAAe,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,CACnE,IAAI,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAC/B,CAAA;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;YACpB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACzB,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI;gBAAE,SAAQ;YAC3B,GAAG,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QAC1F,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;AACxC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,gBAAgB;IAC/B,OAAO;QACN,IAAI,EAAE,YAAY;QAClB,KAAK,CAAC,IAAc,EAAE,QAAkB;YACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,IAAI,EAAE,CAAA;YAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAA;YAE7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAClD,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,mCAAmC,EAAE,CAAA;YACjE,CAAC;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO;oBACN,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,+BAA+B,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;oBAC7E,OAAO,EAAE,EAAE,MAAM,EAAE;iBACnB,CAAA;YACF,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO;oBACN,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,YAAY,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B;oBACpE,OAAO,EAAE,EAAE,QAAQ,EAAE;iBACrB,CAAA;YACF,CAAC;YAED,MAAM,OAAO,GAAG,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;YAC3C,MAAM,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC,MAAM,CAAA;YACzC,MAAM,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAA;YACxC,MAAM,EAAE,GAAG,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAA;YAEzF,OAAO;gBACN,KAAK,EAAE,EAAE;gBACT,MAAM,EACL,EAAE,KAAK,CAAC;oBACP,CAAC,CAAC,qBAAqB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;oBAC3C,CAAC,CAAC,WAAW,OAAO,IAAI,QAAQ,CAAC,MAAM,uBAAuB,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBACjH,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE;aACzD,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,gBAAgB,CAC/B,aAAgC,CAAC,UAAU,EAAE,gBAAgB,CAAC;IAE9D,OAAO;QACN,IAAI,EAAE,YAAY;QAClB,oEAAoE;QACpE,kEAAkE;QAClE,oEAAoE;QACpE,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAChB,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAA;YAC1D,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YAChF,OAAO;gBACN,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjB,MAAM,EAAE,EAAE;oBACT,CAAC,CAAC,cAAc,IAAI,CAAC,UAAU,EAAE;oBACjC,CAAC,CAAC,cAAc,IAAI,CAAC,UAAU,IAAI,SAAS,qBAAqB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACzF,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE;aACxC,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAChD,OAAO;QACN,IAAI,EAAE,aAAa;QACnB,KAAK,CAAC,IAAc;YACnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;YAC9B,OAAO;gBACN,KAAK,EAAE,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;gBAC1D,MAAM,EACL,IAAI,IAAI,QAAQ;oBACf,CAAC,CAAC,GAAG,IAAI,IAAI,QAAQ,QAAQ;oBAC7B,CAAC,CAAC,gBAAgB,IAAI,oBAAoB,QAAQ,YAAY;gBAChE,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC3B,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,cAAc,CAAC,GAAG,QAAkB;IACnD,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,gEAAgE;QAChE,+DAA+D;QAC/D,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAA;YAC9B,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YACzD,OAAO;gBACN,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM;gBACvF,MAAM,EACL,OAAO,CAAC,MAAM,KAAK,CAAC;oBACnB,CAAC,CAAC,aAAa,QAAQ,CAAC,MAAM,qBAAqB;oBACnD,CAAC,CAAC,YAAY,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACpC,OAAO,EAAE,EAAE,OAAO,EAAE;aACpB,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAC3B,IAAY,EACZ,EAAkE;IAElE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;AAC3B,CAAC"}
+{"version":3,"file":"scorers.js","sourceRoot":"","sources":["../../src/eval/scorers.ts"],"names":[],"mappings":"AAIA;;;;;;GAMG;AACH,SAAS,SAAS,CAAC,CAAoB,EAAE,CAAoB;IAC5D,MAAM,KAAK,GAAe,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,CACnE,IAAI,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAC/B,CAAA;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;YACpB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACzB,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI;gBAAE,SAAQ;YAC3B,GAAG,CAAC,CAAC,CAAC;gBACL,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QAC1F,CAAC;IACF,CAAC;IACD,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;AACxC,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,gBAAgB;IAC/B,OAAO;QACN,IAAI,EAAE,YAAY;QAClB,KAAK,CAAC,IAAc,EAAE,QAAkB;YACvC,MAAM,QAAQ,GAAG,QAAQ,CAAC,aAAa,IAAI,EAAE,CAAA;YAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAA;YAE7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAClD,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,mCAAmC,EAAE,CAAA;YACjE,CAAC;YACD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC3B,OAAO;oBACN,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,+BAA+B,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;oBAC7E,OAAO,EAAE,EAAE,MAAM,EAAE;iBACnB,CAAA;YACF,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACzB,OAAO;oBACN,KAAK,EAAE,CAAC;oBACR,MAAM,EAAE,YAAY,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,4BAA4B;oBACpE,OAAO,EAAE,EAAE,QAAQ,EAAE;iBACrB,CAAA;YACF,CAAC;YAED,MAAM,OAAO,GAAG,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;YAC3C,MAAM,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC,MAAM,CAAA;YACzC,MAAM,MAAM,GAAG,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAA;YACxC,MAAM,EAAE,GAAG,SAAS,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,SAAS,GAAG,MAAM,CAAC,CAAA;YAEzF,OAAO;gBACN,KAAK,EAAE,EAAE;gBACT,MAAM,EACL,EAAE,KAAK,CAAC;oBACP,CAAC,CAAC,qBAAqB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;oBAC3C,CAAC,CAAC,WAAW,OAAO,IAAI,QAAQ,CAAC,MAAM,uBAAuB,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;gBACjH,OAAO,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE;aACzD,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,gBAAgB,CAC/B,aAAgC,CAAC,UAAU,EAAE,gBAAgB,CAAC;IAE9D,OAAO;QACN,IAAI,EAAE,YAAY;QAClB,oEAAoE;QACpE,kEAAkE;QAClE,oEAAoE;QACpE,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;gBAChB,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,gBAAgB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAA;YAC1D,CAAC;YACD,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YAChF,OAAO;gBACN,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjB,MAAM,EAAE,EAAE;oBACT,CAAC,CAAC,cAAc,IAAI,CAAC,UAAU,EAAE;oBACjC,CAAC,CAAC,cAAc,IAAI,CAAC,UAAU,IAAI,SAAS,qBAAqB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACzF,OAAO,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE;aACxC,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAChD,OAAO;QACN,IAAI,EAAE,aAAa;QACnB,KAAK,CAAC,IAAc;YACnB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;YAC9B,OAAO;gBACN,KAAK,EAAE,IAAI,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,IAAI,CAAC;gBAC1D,MAAM,EACL,IAAI,IAAI,QAAQ;oBACf,CAAC,CAAC,GAAG,IAAI,IAAI,QAAQ,QAAQ;oBAC7B,CAAC,CAAC,gBAAgB,IAAI,oBAAoB,QAAQ,YAAY;gBAChE,OAAO,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;aAC3B,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,cAAc,CAAC,GAAG,QAAkB;IACnD,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,gEAAgE;QAChE,+DAA+D;QAC/D,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAA;YAC9B,MAAM,OAAO,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YACzD,OAAO;gBACN,KAAK,EAAE,QAAQ,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM;gBACvF,MAAM,EACL,OAAO,CAAC,MAAM,KAAK,CAAC;oBACnB,CAAC,CAAC,aAAa,QAAQ,CAAC,MAAM,qBAAqB;oBACnD,CAAC,CAAC,YAAY,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACpC,OAAO,EAAE,EAAE,OAAO,EAAE;aACpB,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAAC,QAAuC;IACpE,OAAO;QACN,IAAI,EAAE,SAAS;QACf,6DAA6D;QAC7D,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAA;YAC5B,IAAI,CAAC,OAAO,EAAE,CAAC;gBACd,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,EAAE,gCAAgC,EAAE,CAAA;YACjF,CAAC;YACD,MAAM,EAAE,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;YAC7C,OAAO;gBACN,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjB,MAAM,EAAE,EAAE;oBACT,CAAC,CAAC,WAAW,OAAO,CAAC,OAAO,EAAE;oBAC9B,CAAC,CAAC,WAAW,OAAO,CAAC,OAAO,qBAAqB,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;gBACvE,OAAO,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE;aACzD,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAqC,EAAE;IAC1E,MAAM,cAAc,GAAG,IAAI,CAAC,cAAc,KAAK,IAAI,CAAA;IACnD,OAAO;QACN,IAAI,EAAE,gBAAgB;QACtB,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,MAAM,KAAK,GAAG,IAAI,CAAC,aAAa,CAAA;YAChC,MAAM,EAAE,GAAG,cAAc,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAA;YACrE,OAAO;gBACN,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjB,MAAM,EAAE,KAAK;oBACZ,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,qBAAqB,KAAK,CAAC,UAAU,EAAE;oBACtD,CAAC,CAAC,+CAA+C;gBAClD,OAAO,EAAE,EAAE,aAAa,EAAE,KAAK,IAAI,IAAI,EAAE,cAAc,EAAE;aACzD,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAsC,EAAE;IACxE,OAAO;QACN,IAAI,EAAE,aAAa;QACnB,QAAQ,EAAE,MAAM;QAChB,KAAK,CAAC,IAAc;YACnB,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAA;YAC7B,MAAM,EAAE,GACP,IAAI,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,EAAE,IAAI,KAAK,IAAI,CAAC,UAAU,CAAA;YACtF,OAAO;gBACN,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACjB,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,gBAAgB,KAAK,CAAC,IAAI,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,iBAAiB;gBACnF,OAAO,EAAE,EAAE,UAAU,EAAE,KAAK,IAAI,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,IAAI,IAAI,EAAE;aAC3E,CAAA;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAC3B,IAAY,EACZ,EAAkE;IAElE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,CAAA;AAC3B,CAAC"}

package/dist/eval/types.d.ts

@@ -1,5 +1,18 @@
+import type { AnswerClaimMismatch, TurnOutcome } from '../types/session/outcome.js';
 import type { StepResult } from '../types/session/step.js';
+import type { CaseIntervals } from './bootstrap.js';
+import type { ReportDisclosure } from './disclosure.js';
+import type { RepeatReliability, SplitSummary } from './report-stats.js';
 import type { ScoreUncertainty } from './uncertainty.js';
+/**
+ * Which part of a suite a case belongs to.
+ *
+ * `development` cases are the ones the harness was shaped against; a number
+ * measured on them says how well it fits them. `held-out` cases were not
+ * looked at while it was shaped, and only they say how it does on cases it
+ * has not seen.
+ */
+export type EvalSplit = 'development' | 'held-out';
 /**
  * One case in a dataset: an input, and what a good run looks like.
  *
@@ -18,6 +31,11 @@ export interface EvalCase<TInput = unknown> {
     expectedTools?: readonly string[];
     /** Per-case scorer overrides; falls back to the dataset's scorers. */
     scorers?: readonly Scorer[];
+    /**
+     * `development` or `held-out`. Left out, the case is reported as
+     * unassigned. See {@link EvalSplit}.
+     */
+    split?: EvalSplit;
 }
 /** What actually happened, handed to every scorer. */
 export interface EvalTurn {
@@ -34,6 +52,18 @@ export interface EvalTurn {
     durationMs: number;
     /** Set when the turn threw rather than settling. */
     error?: string;
+    /**
+     * What the turn changed and what checked it, from the kernel's own record of
+     * its tool calls. Absent for a turn that never ran the loop.
+     */
+    outcome?: TurnOutcome;
+    /**
+     * A plain claim in the answer that the outcome record contradicts, found by
+     * `compareAnswerToOutcome`. Absent when there is none.
+     */
+    claimMismatch?: AnswerClaimMismatch;
+    /** The model handed the task back to a person and the turn ended there. Absent otherwise. */
+    handedBack?: import('../types/session/hand-back.js').HandBack;
 }
 /**
  * A judgement about one turn.
@@ -136,6 +166,10 @@ export interface CaseResult {
      * mattered. Absent when no gate missed.
      */
     failedGates?: readonly string[];
+    /** Which run of this case this is, from 1. Set on every run of a report that has `repeats`. */
+    repeat?: number;
+    /** The case's declared split, copied here so a report can be cut by it. */
+    split?: EvalSplit;
 }
 export interface ExperimentReport {
     name: string;
@@ -169,5 +203,45 @@ export interface ExperimentReport {
      */
     byScorer: Record<string, number>;
     durationMs: number;
+    /**
+     * Times each case ran. When it is above 1, `cases` holds every run of
+     * every case, in case order, each marked with its `repeat`, and `passed`,
+     * `failed` and `inconclusive` count runs. Absent in a report written before
+     * repeats existed, which is read as 1.
+     */
+    repeats?: number;
+    /** Seed of the bootstrap behind {@link intervals}. */
+    seed?: number;
+    /**
+     * Passed runs over judged runs. An inconclusive run is in neither number,
+     * so it cannot pull the rate either way.
+     */
+    passRate?: number;
+    /**
+     * 95% intervals for {@link mean} and {@link passRate}, by a bootstrap over
+     * cases. Absent from a report that predates it.
+     */
+    intervals?: CaseIntervals;
+    /** Per case: how many of its repeats passed. Present when `repeats` is above 1. */
+    caseSummaries?: readonly CaseRepeatSummary[];
+    /** Reliability across repeats. Present when `repeats` is above 1. */
+    reliability?: RepeatReliability;
+    /** Numbers per split. Keys appear only for splits that have cases. */
+    splits?: Partial<Record<EvalSplit | 'unassigned', SplitSummary>>;
+    /** What produced this report. */
+    disclosure?: ReportDisclosure;
+}
+/** How the repeats of one case went. */
+export interface CaseRepeatSummary {
+    case: string;
+    split?: EvalSplit;
+    repeats: number;
+    passed: number;
+    failed: number;
+    inconclusive: number;
+    /** Mean score over the judged repeats; null when none was judged. */
+    mean: number | null;
+    /** Every repeat passed. An inconclusive repeat means this is false. */
+    passedEveryRepeat: boolean;
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/eval/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAExD;;;;;GAKG;AACH,MAAM,WAAW,QAAQ,CAAC,MAAM,GAAG,OAAO;IACzC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,uEAAuE;IACvE,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB;;;OAGG;IACH,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACjC,sEAAsE;IACtE,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CAC3B;AAED,sDAAsD;AACtD,MAAM,WAAW,QAAQ;IACxB,wDAAwD;IACxD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,4EAA4E;IAC5E,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAC1B,KAAK,EAAE,SAAS,UAAU,EAAE,CAAA;IAC5B,wDAAwD;IACxD,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,KAAK;IACrB,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,2EAA2E;IAC3E,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACjC;;;;;;;;;;;;;;;;OAgBG;IACH,WAAW,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,MAAM,CAAA;AAE5C,MAAM,WAAW,MAAM;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;IACvF,kDAAkD;IAClD,QAAQ,CAAC,EAAE,cAAc,CAAA;IACzB;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED;;;;;GAKG;AACH,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,QAAQ,GAAG,cAAc,CAAA;AAE7D,MAAM,WAAW,UAAU;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,QAAQ,CAAA;IACd,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC7B,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,UAAU,CAAA;IAClB,wCAAwC;IACxC,MAAM,EAAE,OAAO,CAAA;IACf;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CAC/B;AAED,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,SAAS,UAAU,EAAE,CAAA;IAC5B,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAA;IACZ;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,EAAE,gBAAgB,CAAA;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAA;IACpB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,UAAU,EAAE,MAAM,CAAA;CAClB"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,mBAAmB,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAA;AACnF,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AAC1D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAA;AACnD,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAA;AACvD,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AACxE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAExD;;;;;;;GAOG;AACH,MAAM,MAAM,SAAS,GAAG,aAAa,GAAG,UAAU,CAAA;AAElD;;;;;GAKG;AACH,MAAM,WAAW,QAAQ,CAAC,MAAM,GAAG,OAAO;IACzC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,uEAAuE;IACvE,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB;;;OAGG;IACH,aAAa,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACjC,sEAAsE;IACtE,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC3B;;;OAGG;IACH,KAAK,CAAC,EAAE,SAAS,CAAA;CACjB;AAED,sDAAsD;AACtD,MAAM,WAAW,QAAQ;IACxB,wDAAwD;IACxD,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,4EAA4E;IAC5E,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAC1B,KAAK,EAAE,SAAS,UAAU,EAAE,CAAA;IAC5B,wDAAwD;IACxD,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,YAAY,EAAE,MAAM,CAAA;IACpB,UAAU,EAAE,MAAM,CAAA;IAClB,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;OAGG;IACH,OAAO,CAAC,EAAE,WAAW,CAAA;IACrB;;;OAGG;IACH,aAAa,CAAC,EAAE,mBAAmB,CAAA;IACnC,6FAA6F;IAC7F,UAAU,CAAC,EAAE,OAAO,+BAA+B,EAAE,QAAQ,CAAA;CAC7D;AAED;;;;;;;GAOG;AACH,MAAM,WAAW,KAAK;IACrB,iEAAiE;IACjE,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,2EAA2E;IAC3E,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IACjC;;;;;;;;;;;;;;;;OAgBG;IACH,WAAW,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,MAAM,cAAc,GAAG,MAAM,GAAG,MAAM,CAAA;AAE5C,MAAM,WAAW,MAAM;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ;;;;;OAKG;IACH,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAA;IACvF,kDAAkD;IAClD,QAAQ,CAAC,EAAE,cAAc,CAAA;IACzB;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;CAClB;AAED;;;;;GAKG;AACH,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,QAAQ,GAAG,cAAc,CAAA;AAE7D,MAAM,WAAW,UAAU;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,QAAQ,CAAA;IACd,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAA;IAC7B,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,UAAU,CAAA;IAClB,wCAAwC;IACxC,MAAM,EAAE,OAAO,CAAA;IACf;;;;;;OAMG;IACH,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC/B,+FAA+F;IAC/F,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,2EAA2E;IAC3E,KAAK,CAAC,EAAE,SAAS,CAAA;CACjB;AAED,MAAM,WAAW,gBAAgB;IAChC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,SAAS,UAAU,EAAE,CAAA;IAC5B,qDAAqD;IACrD,IAAI,EAAE,MAAM,CAAA;IACZ;;;;;;;;;;;;;;OAcG;IACH,WAAW,CAAC,EAAE,gBAAgB,CAAA;IAC9B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,uDAAuD;IACvD,YAAY,EAAE,MAAM,CAAA;IACpB;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,UAAU,EAAE,MAAM,CAAA;IAClB;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,sDAAsD;IACtD,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;OAGG;IACH,SAAS,CAAC,EAAE,aAAa,CAAA;IACzB,mFAAmF;IACnF,aAAa,CAAC,EAAE,SAAS,iBAAiB,EAAE,CAAA;IAC5C,qEAAqE;IACrE,WAAW,CAAC,EAAE,iBAAiB,CAAA;IAC/B,sEAAsE;IACtE,MAAM,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,YAAY,EAAE,YAAY,CAAC,CAAC,CAAA;IAChE,iCAAiC;IACjC,UAAU,CAAC,EAAE,gBAAgB,CAAA;CAC7B;AAED,wCAAwC;AACxC,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,EAAE,SAAS,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,MAAM,CAAA;IACd,YAAY,EAAE,MAAM,CAAA;IACpB,qEAAqE;IACrE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,uEAAuE;IACvE,iBAAiB,EAAE,OAAO,CAAA;CAC1B"}

package/dist/manager/session/turn-recorder.d.ts

@@ -8,6 +8,7 @@ import type { ProviderErrorInfo } from '../../types/provider/index.js';
 import type { AuditEvent, AuditEventInput } from '../../types/session/audit.js';
 import type { TurnRecorderConfig } from '../../types/session/config.js';
 import { type SessionEvent } from '../../types/session/events.js';
+import type { HandBack } from '../../types/session/hand-back.js';
 import type { ProjectId, TopicId } from '../../types/session/ids.js';
 import type { TurnOutcome } from '../../types/session/outcome.js';
 import type { StepResult } from '../../types/session/step.js';
@@ -135,6 +136,8 @@ export declare class TurnRecorder {
     setAbandonedJobIds(jobIds: readonly string[]): void;
     /** The turn's outcome record, set once as the loop ends; absent means none was kept. */
     setOutcome(outcome: TurnOutcome | undefined): void;
+    /** The model handed the task back; set once, as the turn ends. */
+    setHandedBack(handBack: HandBack | undefined): void;
     setSteps(steps: readonly StepResult[]): void;
     /** Record that a provider chain advanced, so the turn stops naming a member that did not serve. */
     setServingProvider(providerId: string): void;

package/dist/manager/session/turn-recorder.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"turn-recorder.d.ts","sourceRoot":"","sources":["../../../src/manager/session/turn-recorder.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AACrE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAA;AAE7E,OAAO,EACN,KAAK,YAAY,EACjB,KAAK,UAAU,EACf,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAI1B,MAAM,kCAAkC,CAAA;AAGzC,OAAO,EACN,KAAK,QAAQ,EACb,KAAK,UAAU,EAKf,MAAM,6BAA6B,CAAA;AAEpC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAA;AACtF,OAAO,KAAK,EAAoB,OAAO,EAAE,MAAM,8BAA8B,CAAA;AAC7E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAA;AACtE,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAA;AAC/E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAA;AACvE,OAAO,EACN,KAAK,YAAY,EAGjB,MAAM,+BAA+B,CAAA;AACtC,OAAO,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAA;AAEjE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAA;AAC7D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oCAAoC,CAAA;AACpE,OAAO,EACN,KAAK,kBAAkB,EACvB,KAAK,MAAM,EACX,KAAK,IAAI,EACT,KAAK,kBAAkB,EACvB,KAAK,cAAc,EAEnB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,MAAM,6BAA6B,CAAA;AAYpC,yDAAyD;AACzD,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAA;CAClC;AAED,iFAAiF;AACjF,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAA;CAC9B;AA6BD,8FAA8F;AAC9F,MAAM,WAAW,uBAAuB;IACvC,oGAAoG;IACpG,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAA;IAC7B,qFAAqF;IACrF,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;IAC3B,uDAAuD;IACvD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,uFAAuF;IACvF,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,4DAA4D;IAC5D,QAAQ,CAAC,OAAO,EAAE;QACjB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;QACpB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;QACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;QAC3B,2EAA2E;QAC3E,QAAQ,CAAC,UAAU,CAAC,EAAE,cAAc,CAAA;KACpC,CAAA;CACD;AAED,yFAAyF;AACzF,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,oFAAoF;IACpF,QAAQ,CAAC,kBAAkB,CAAC,EAAE,OAAO,CAAA;CACrC;AAED,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB,QAAa,CAAA;AA0DnD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,YAAY;;IACxB,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAA;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,kBAAkB,CAAA;IACpC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAA;gBAkDZ,MAAM,EAAE,kBAAkB;IA+CtC,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,SAAS,IAAI,SAAS,CAEzB;IAED,IAAI,eAAe,IAAI,SAAS,GAAG,SAAS,CAE3C;IAED,IAAI,YAAY,IAAI,MAAM,GAAG,SAAS,CAErC;IAED,IAAI,eAAe,IAAI,sBAAsB,GAAG,SAAS,CAExD;IAED,iFAAiF;IACjF,IAAI,KAAK,IAAI,YAAY,GAAG,SAAS,CAEpC;IAED,yDAAyD;IACzD,IAAI,aAAa,IAAI,SAAS,GAAG,SAAS,CAEzC;IAED,qDAAqD;IACrD,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,yEAAyE;IACzE,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,+DAA+D;IAC/D,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,8CAA8C;IAC9C,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAID,IAAI,MAAM,8DAET;IAED,IAAI,UAAU,2BAEb;IAED,IAAI,QAAQ,IAAI,OAAO,EAAE,CAExB;IAED,IAAI,UAAU,IAAI,UAAU,CAI3B;IAED,IAAI,QAAQ,IAAI,QAAQ,CAQvB;IAED,IAAI,gBAAgB,IAAI,MAAM,CAE7B;IAED,6BAA6B;IAC7B,IAAI,IAAI,IAAI,IAAI,CAEf;IAED,OAAO,IAAI,QAAQ,CAAC,IAAI,CAAC;IAmBzB,WAAW,IAAI,IAAI;IAInB,aAAa,CAAC,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI;IAO5C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,iBAAiB,GAAG,IAAI;IASlE,aAAa,IAAI,IAAI;IAOrB,aAAa,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIvC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,iBAAiB,GAAG,IAAI;IAKpE,kBAAkB,IAAI,MAAM;IAK5B;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAMzD,+DAA+D;IAC/D,IAAI,YAAY,IAAI,gBAAgB,CAEnC;IAED,iFAAiF;IACjF,qBAAqB,IAAI,IAAI;IAI7B,2EAA2E;IAC3E,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAKzC,mBAAmB,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IAKrD,kBAAkB,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IAKnD,wFAAwF;IACxF,UAAU,CAAC,OAAO,EAAE,WAAW,GAAG,SAAS,GAAG,IAAI;IAKlD,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,GAAG,IAAI;IAI5C,mGAAmG;IACnG,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAI5C,uFAAuF;IACvF,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAOD,eAAe,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,GAAG,IAAI;IASlE;;;;OAIG;IACH,eAAe,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,GAAG,IAAI;IAMlE,+FAA+F;IAC/F,qBAAqB,IAAI,IAAI;IAK7B,IAAI,gBAAgB,IAAI,MAAM,GAAG,SAAS,CAEzC;IAED,IAAI,sBAAsB,IAAI,MAAM,GAAG,SAAS,CAE/C;IAED,wFAAwF;IACxF,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,GAAG,IAAI;IAQxF,qEAAqE;IACrE,iBAAiB,IAAI,MAAM;IA6B3B,4EAA4E;IAC5E,UAAU,CAAC,MAAM,EAAE,cAAc,CAAC,QAAQ,CAAC,GAAG,kBAAkB;IAqBhE;;;OAGG;IACG,IAAI,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IA2ExE;;;;OAIG;IACH,gBAAgB,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,qBAAqB,KAAK,IAAI,GAAG,MAAM,IAAI;IAoD/E,0DAA0D;IACpD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IA0B9B,mEAAmE;IAC7D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5B,qGAAqG;IAC/F,IAAI,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAM5C;;;OAGG;IACG,KAAK,CAAC,KAAK,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;IA6BjE,6EAA6E;IACvE,MAAM,CAAC,gBAAgB,EAAE,MAAM,EAAE,kBAAkB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAmB7F;;;;OAIG;IACH,IAAI,aAAa,IAAI,MAAM,CAE1B;IAoDD;;;OAGG;IACH,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,eAAe,CAAC;IAKlF;;;;;;OAMG;IACH,qBAAqB,CACpB,KAAK,EAAE,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GACxC,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IAWvC;;;;OAIG;IACG,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IA4D5E;;;;;;;;;;;;;OAaG;IACH,uBAAuB,CACtB,KAAK,EAAE,OAAO,CACb,YAAY,EACZ;QACC,IAAI,EAAE,uBAAuB,GAAG,wBAAwB,GAAG,qBAAqB,CAAA;KAChF,CACD,EACD,QAAQ,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IAyEvC;;;;OAIG;IACH,WAAW,CACV,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE;QACR,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAA;QAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAA;QAC5B;;;;;WAKG;QACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KACtB,GACJ,IAAI;IA6BP,oFAAoF;IACpF,eAAe,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,GAAG,IAAI;IAKnD;;;;OAIG;IACH,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,SAAS,GAAG,SAAS;IAIrD,0EAA0E;IAC1E,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAyQrC;;;;;OAKG;IACG,WAAW,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC;IAsD9D;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAY9B,uDAAuD;IACvD,IAAI,UAAU,IAAI,OAAO,CAExB;CACD;AAED,oEAAoE;AACpE,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,CAAC,YAAY,CAAC,GAAG,kBAAkB,CAkB/F;AAED,8EAA8E;AAC9E,wBAAgB,UAAU,CACzB,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,MAAM,GAAG,SAAS,GACxB,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAiBrC;AAED,8FAA8F;AAC9F,wBAAsB,iBAAiB,CACtC,GAAG,EAAE,UAAU,EACf,MAAM,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAQ7B;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CACtC,GAAG,EAAE,UAAU,EACf,OAAO,GAAE;IAAE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5C,OAAO,CAAC,eAAe,EAAE,CAAC,CAkB5B;AA2CD,sFAAsF;AACtF,MAAM,WAAW,oBAAoB;IACpC,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAChD;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;CACnE;AAQD;;;;;;;;;;GAUG;AACH,wBAAsB,wBAAwB,CAC7C,GAAG,EAAE,UAAU,GACb,OAAO,CAAC,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAE1C;AAED;;;;GAIG;AACH,wBAAsB,4BAA4B,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAiDjG;AAED;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CA6B3E"}
+{"version":3,"file":"turn-recorder.d.ts","sourceRoot":"","sources":["../../../src/manager/session/turn-recorder.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AACrE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAA;AAE7E,OAAO,EACN,KAAK,YAAY,EACjB,KAAK,UAAU,EACf,KAAK,eAAe,EACpB,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAI1B,MAAM,kCAAkC,CAAA;AAGzC,OAAO,EACN,KAAK,QAAQ,EACb,KAAK,UAAU,EAKf,MAAM,6BAA6B,CAAA;AAEpC,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,0BAA0B,CAAA;AACtF,OAAO,KAAK,EAAoB,OAAO,EAAE,MAAM,8BAA8B,CAAA;AAC7E,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAA;AACtE,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAA;AAC/E,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAA;AACvE,OAAO,EACN,KAAK,YAAY,EAGjB,MAAM,+BAA+B,CAAA;AACtC,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAA;AAChE,OAAO,KAAK,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAA;AAEjE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAA;AAC7D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,oCAAoC,CAAA;AACpE,OAAO,EACN,KAAK,kBAAkB,EACvB,KAAK,MAAM,EACX,KAAK,IAAI,EACT,KAAK,kBAAkB,EACvB,KAAK,cAAc,EAEnB,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,MAAM,6BAA6B,CAAA;AAYpC,yDAAyD;AACzD,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,CAAA;CAClC;AAED,iFAAiF;AACjF,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB,+EAA+E;IAC/E,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAA;CAC9B;AA6BD,8FAA8F;AAC9F,MAAM,WAAW,uBAAuB;IACvC,oGAAoG;IACpG,QAAQ,CAAC,KAAK,CAAC,EAAE,YAAY,CAAA;IAC7B,qFAAqF;IACrF,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;IAC3B,uDAAuD;IACvD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,uFAAuF;IACvF,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;IAC5B,4DAA4D;IAC5D,QAAQ,CAAC,OAAO,EAAE;QACjB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;QACpB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;QACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;QAC3B,2EAA2E;QAC3E,QAAQ,CAAC,UAAU,CAAC,EAAE,cAAc,CAAA;KACpC,CAAA;CACD;AAED,yFAAyF;AACzF,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,oFAAoF;IACpF,QAAQ,CAAC,kBAAkB,CAAC,EAAE,OAAO,CAAA;CACrC;AAED,4EAA4E;AAC5E,eAAO,MAAM,yBAAyB,QAAa,CAAA;AA0DnD;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,qBAAa,YAAY;;IACxB,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAA;IAC7B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,kBAAkB,CAAA;IACpC,QAAQ,CAAC,GAAG,EAAE,UAAU,CAAA;gBAkDZ,MAAM,EAAE,kBAAkB;IA+CtC,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,QAAQ,IAAI,QAAQ,CAEvB;IAED,IAAI,SAAS,IAAI,SAAS,CAEzB;IAED,IAAI,eAAe,IAAI,SAAS,GAAG,SAAS,CAE3C;IAED,IAAI,YAAY,IAAI,MAAM,GAAG,SAAS,CAErC;IAED,IAAI,eAAe,IAAI,sBAAsB,GAAG,SAAS,CAExD;IAED,iFAAiF;IACjF,IAAI,KAAK,IAAI,YAAY,GAAG,SAAS,CAEpC;IAED,yDAAyD;IACzD,IAAI,aAAa,IAAI,SAAS,GAAG,SAAS,CAEzC;IAED,qDAAqD;IACrD,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,yEAAyE;IACzE,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,+DAA+D;IAC/D,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAED,8CAA8C;IAC9C,IAAI,QAAQ,IAAI,OAAO,CAEtB;IAID,IAAI,MAAM,8DAET;IAED,IAAI,UAAU,2BAEb;IAED,IAAI,QAAQ,IAAI,OAAO,EAAE,CAExB;IAED,IAAI,UAAU,IAAI,UAAU,CAI3B;IAED,IAAI,QAAQ,IAAI,QAAQ,CAQvB;IAED,IAAI,gBAAgB,IAAI,MAAM,CAE7B;IAED,6BAA6B;IAC7B,IAAI,IAAI,IAAI,IAAI,CAEf;IAED,OAAO,IAAI,QAAQ,CAAC,IAAI,CAAC;IAmBzB,WAAW,IAAI,IAAI;IAInB,aAAa,CAAC,UAAU,CAAC,EAAE,UAAU,GAAG,IAAI;IAO5C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,iBAAiB,GAAG,IAAI;IASlE,aAAa,IAAI,IAAI;IAOrB,aAAa,CAAC,MAAM,EAAE,UAAU,GAAG,IAAI;IAIvC,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,aAAa,CAAC,EAAE,iBAAiB,GAAG,IAAI;IAKpE,kBAAkB,IAAI,MAAM;IAK5B;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,GAAG,IAAI;IAMzD,+DAA+D;IAC/D,IAAI,YAAY,IAAI,gBAAgB,CAEnC;IAED,iFAAiF;IACjF,qBAAqB,IAAI,IAAI;IAI7B,2EAA2E;IAC3E,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,IAAI;IAKzC,mBAAmB,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IAKrD,kBAAkB,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,IAAI;IAKnD,wFAAwF;IACxF,UAAU,CAAC,OAAO,EAAE,WAAW,GAAG,SAAS,GAAG,IAAI;IAKlD,kEAAkE;IAClE,aAAa,CAAC,QAAQ,EAAE,QAAQ,GAAG,SAAS,GAAG,IAAI;IAKnD,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,GAAG,IAAI;IAI5C,mGAAmG;IACnG,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAI5C,uFAAuF;IACvF,IAAI,iBAAiB,IAAI,MAAM,CAE9B;IAOD,eAAe,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,GAAG,IAAI;IASlE;;;;OAIG;IACH,eAAe,CAAC,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,GAAG,IAAI;IAMlE,+FAA+F;IAC/F,qBAAqB,IAAI,IAAI;IAK7B,IAAI,gBAAgB,IAAI,MAAM,GAAG,SAAS,CAEzC;IAED,IAAI,sBAAsB,IAAI,MAAM,GAAG,SAAS,CAE/C;IAED,wFAAwF;IACxF,YAAY,CAAC,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,GAAG,IAAI;IAQxF,qEAAqE;IACrE,iBAAiB,IAAI,MAAM;IA6B3B,4EAA4E;IAC5E,UAAU,CAAC,MAAM,EAAE,cAAc,CAAC,QAAQ,CAAC,GAAG,kBAAkB;IA4BhE;;;OAGG;IACG,IAAI,CAAC,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IA2ExE;;;;OAIG;IACH,gBAAgB,CAAC,QAAQ,EAAE,CAAC,MAAM,EAAE,qBAAqB,KAAK,IAAI,GAAG,MAAM,IAAI;IAoD/E,0DAA0D;IACpD,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IA0B9B,mEAAmE;IAC7D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAW5B,qGAAqG;IAC/F,IAAI,IAAI,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC;IAM5C;;;OAGG;IACG,KAAK,CAAC,KAAK,GAAE,cAAmB,GAAG,OAAO,CAAC,eAAe,CAAC;IA6BjE,6EAA6E;IACvE,MAAM,CAAC,gBAAgB,EAAE,MAAM,EAAE,kBAAkB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAmB7F;;;;OAIG;IACH,IAAI,aAAa,IAAI,MAAM,CAE1B;IAoDD;;;OAGG;IACH,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,eAAe,CAAC;IAKlF;;;;;;OAMG;IACH,qBAAqB,CACpB,KAAK,EAAE,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,GACxC,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IAWvC;;;;OAIG;IACG,WAAW,CAAC,KAAK,EAAE,YAAY,GAAG,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IA4D5E;;;;;;;;;;;;;OAaG;IACH,uBAAuB,CACtB,KAAK,EAAE,OAAO,CACb,YAAY,EACZ;QACC,IAAI,EAAE,uBAAuB,GAAG,wBAAwB,GAAG,qBAAqB,CAAA;KAChF,CACD,EACD,QAAQ,CAAC,EAAE,UAAU,GACnB,OAAO,CAAC,eAAe,GAAG,SAAS,CAAC;IAyEvC;;;;OAIG;IACH,WAAW,CACV,OAAO,EAAE,OAAO,EAChB,OAAO,GAAE;QACR,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,CAAA;QAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,OAAO,CAAA;QAC5B;;;;;WAKG;QACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAA;KACtB,GACJ,IAAI;IA6BP,oFAAoF;IACpF,eAAe,CAAC,QAAQ,EAAE,SAAS,OAAO,EAAE,GAAG,IAAI;IAKnD;;;;OAIG;IACH,YAAY,CAAC,OAAO,EAAE,OAAO,GAAG,SAAS,GAAG,SAAS;IAIrD,0EAA0E;IAC1E,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAyQrC;;;;;OAKG;IACG,WAAW,CAAC,KAAK,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC;IAsD9D;;;;OAIG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAY9B,uDAAuD;IACvD,IAAI,UAAU,IAAI,OAAO,CAExB;CACD;AAED,oEAAoE;AACpE,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,kBAAkB,CAAC,YAAY,CAAC,GAAG,kBAAkB,CAkB/F;AAED,8EAA8E;AAC9E,wBAAgB,UAAU,CACzB,KAAK,EAAE,YAAY,EACnB,MAAM,EAAE,MAAM,GAAG,SAAS,GACxB,UAAU,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAuBrC;AAED,8FAA8F;AAC9F,wBAAsB,iBAAiB,CACtC,GAAG,EAAE,UAAU,EACf,MAAM,EAAE,MAAM,GACZ,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAQ7B;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CACtC,GAAG,EAAE,UAAU,EACf,OAAO,GAAE;IAAE,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5C,OAAO,CAAC,eAAe,EAAE,CAAC,CAkB5B;AA2CD,sFAAsF;AACtF,MAAM,WAAW,oBAAoB;IACpC,0DAA0D;IAC1D,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAChD;;;;;OAKG;IACH,gBAAgB,CAAC,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;CACnE;AAQD;;;;;;;;;;GAUG;AACH,wBAAsB,wBAAwB,CAC7C,GAAG,EAAE,UAAU,GACb,OAAO,CAAC,WAAW,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC,CAE1C;AAED;;;;GAIG;AACH,wBAAsB,4BAA4B,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAiDjG;AAED;;;;GAIG;AACH,wBAAsB,cAAc,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC,CA6B3E"}

package/dist/manager/session/turn-recorder.js

@@ -3,6 +3,7 @@ import { AUTO_CONTINUATION_USER_MESSAGE, REASONING_CUT_USER_MESSAGE, } from '../
 import { EMPTY_TOKEN_USAGE } from '../../constants/limits.js';
 import { NAMZU } from '../../constants/telemetry/index.js';
 import { resolveModelPricing } from '../../pricing/index.js';
+import { compareAnswerToOutcome } from '../../runtime/query/turn/answer-claims.js';
 import { foldSessionLog } from '../../store/session-log/incremental.js';
 import { SessionMessageFold, turnHolder, withMessageId, } from '../../store/session-log/index.js';
 import { getActiveSpanContext } from '../../telemetry/runtime-accessors.js';
@@ -379,6 +380,13 @@ export class TurnRecorder {
         else
             this.#turn.outcome = outcome;
     }
+    /** The model handed the task back; set once, as the turn ends. */
+    setHandedBack(handBack) {
+        if (handBack === undefined)
+            delete this.#turn.handedBack;
+        else
+            this.#turn.handedBack = handBack;
+    }
     setSteps(steps) {
         this.#turn.steps = steps;
     }
@@ -466,6 +474,11 @@ export class TurnRecorder {
     settlement(status) {
         const turn = this.getTurn();
         const resultMessageId = this.#lastAssistantEntry()?.id;
+        // A task handed back says what blocks it in the model's words; those are not
+        // an answer to read claims out of.
+        const claimMismatch = turn.handedBack
+            ? undefined
+            : compareAnswerToOutcome(turn.result, turn.outcome);
         return {
             status,
             iterations: turn.currentIteration,
@@ -479,6 +492,8 @@ export class TurnRecorder {
             abandonedTaskIds: [...(turn.abandonedTaskIds ?? [])],
             abandonedJobIds: [...(turn.abandonedJobIds ?? [])],
             ...(turn.outcome ? { outcome: turn.outcome } : {}),
+            ...(claimMismatch ? { claimMismatch } : {}),
+            ...(turn.handedBack ? { handedBack: turn.handedBack } : {}),
         };
     }
     // ─── the log ───────────────────────────────────────────────────────────
@@ -1372,9 +1387,10 @@ export function snapshotTurnConfig(config) {
 export function eventDraft(event, turnId) {
     const { sessionId: _sessionId, turnId: _turnId, lineage: _lineage, v: _v, seq: _seq, generation: _generation, ...payload } = event;
     const draft = JSON.parse(JSON.stringify(payload));
-    // The outcome rides the live settlement only; the log keeps it in its own record.
+    // The outcome, the claim mismatch and the hand-back ride the live settlement
+    // only; the log keeps the outcome in its own record, the other two nowhere.
     if (draft.settlement !== undefined && typeof draft.settlement === 'object') {
-        const { outcome: _outcome, ...settlement } = draft.settlement;
+        const { outcome: _outcome, claimMismatch: _claimMismatch, handedBack: _handedBack, ...settlement } = draft.settlement;
         draft.settlement = settlement;
     }
     return { ...draft, ...(turnId ? { turnId } : {}) };