@jinn-network/client 0.1.9 → 0.2.0

package/dist/eval/contested-band.d.ts

@@ -0,0 +1,18 @@
+export interface Band {
+    lo: number;
+    hi: number;
+}
+export declare function inContestedBand(passRate: number, band: Band): boolean;
+export interface BlindScreenFidelity {
+    agentSha: string;
+    emptyLoadout: boolean;
+    noCorpusTools: boolean;
+    /** Hash of the host skill directory at screen time. */
+    hostSkillDirHash: string;
+    /** The reference hash of an empty skill directory; hostSkillDirHash MUST equal this. */
+    emptyHostSkillDirHash: string;
+}
+export declare class BlindScreenViolation extends Error {
+    constructor(reason: string);
+}
+export declare function assertBlindScreen(f: BlindScreenFidelity): void;

package/dist/eval/contested-band.js

@@ -0,0 +1,19 @@
+export function inContestedBand(passRate, band) {
+    return passRate >= band.lo && passRate <= band.hi;
+}
+export class BlindScreenViolation extends Error {
+    constructor(reason) {
+        super(`blind-screen fidelity violated: ${reason}`);
+        this.name = 'BlindScreenViolation';
+    }
+}
+export function assertBlindScreen(f) {
+    if (!f.emptyLoadout)
+        throw new BlindScreenViolation('screening run did not have an empty skill loadout');
+    if (!f.noCorpusTools)
+        throw new BlindScreenViolation('screening run had a live corpus tool surface');
+    if (f.hostSkillDirHash !== f.emptyHostSkillDirHash) {
+        throw new BlindScreenViolation(`host skill dir was not empty (${f.hostSkillDirHash} != ${f.emptyHostSkillDirHash})`);
+    }
+}
+//# sourceMappingURL=contested-band.js.map

package/dist/eval/contested-band.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"contested-band.js","sourceRoot":"","sources":["../../src/eval/contested-band.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,eAAe,CAAC,QAAgB,EAAE,IAAU;IAC1D,OAAO,QAAQ,IAAI,IAAI,CAAC,EAAE,IAAI,QAAQ,IAAI,IAAI,CAAC,EAAE,CAAC;AACpD,CAAC;AAYD,MAAM,OAAO,oBAAqB,SAAQ,KAAK;IAC7C,YAAY,MAAc;QACxB,KAAK,CAAC,mCAAmC,MAAM,EAAE,CAAC,CAAC;QACnD,IAAI,CAAC,IAAI,GAAG,sBAAsB,CAAC;IACrC,CAAC;CACF;AAED,MAAM,UAAU,iBAAiB,CAAC,CAAsB;IACtD,IAAI,CAAC,CAAC,CAAC,YAAY;QAAE,MAAM,IAAI,oBAAoB,CAAC,mDAAmD,CAAC,CAAC;IACzG,IAAI,CAAC,CAAC,CAAC,aAAa;QAAE,MAAM,IAAI,oBAAoB,CAAC,8CAA8C,CAAC,CAAC;IACrG,IAAI,CAAC,CAAC,gBAAgB,KAAK,CAAC,CAAC,qBAAqB,EAAE,CAAC;QACnD,MAAM,IAAI,oBAAoB,CAAC,iCAAiC,CAAC,CAAC,gBAAgB,OAAO,CAAC,CAAC,qBAAqB,GAAG,CAAC,CAAC;IACvH,CAAC;AACH,CAAC"}

package/dist/eval/corpus-index.d.ts

@@ -0,0 +1,21 @@
+export interface CorpusRecord {
+    id: string;
+    repos: string[];
+    instanceIdsReferenced: string[];
+    text: string;
+}
+export interface CorpusIndexRecord {
+    id: string;
+    repos: string[];
+    instanceIds: string[];
+    sketch: number[];
+}
+export interface CorpusDerivedIndex {
+    repos: string[];
+    instanceIds: string[];
+    records: CorpusIndexRecord[];
+}
+export declare function tokenize(text: string): string[];
+export declare function minhashSketch(tokens: string[], numHashes?: number): number[];
+export declare function buildCorpusIndex(records: CorpusRecord[]): CorpusDerivedIndex;
+export declare function corpusSnapshotCid(index: CorpusDerivedIndex): `sha256:${string}`;

package/dist/eval/corpus-index.js

@@ -0,0 +1,55 @@
+import { createHash } from 'node:crypto';
+import { canonicalJson } from '../harnesses/engine/canonical-json.js';
+export function tokenize(text) {
+    const m = text.toLowerCase().match(/[a-z0-9_]+/g);
+    return m ?? [];
+}
+/** 32-bit FNV-1a — stable, dependency-free, good enough for MinHash bucketing. */
+function fnv1a(s, seed) {
+    let h = (2166136261 ^ seed) >>> 0;
+    for (let i = 0; i < s.length; i++) {
+        h ^= s.charCodeAt(i);
+        h = Math.imul(h, 16777619) >>> 0;
+    }
+    return h >>> 0;
+}
+export function minhashSketch(tokens, numHashes = 64) {
+    const uniq = [...new Set(tokens)];
+    const out = [];
+    for (let k = 0; k < numHashes; k++) {
+        let min = 0xffffffff;
+        for (const t of uniq) {
+            const h = fnv1a(t, k * 0x9e3779b1);
+            if (h < min)
+                min = h;
+        }
+        out.push(uniq.length === 0 ? 0 : min);
+    }
+    return out;
+}
+export function buildCorpusIndex(records) {
+    const repos = new Set();
+    const instanceIds = new Set();
+    const indexed = [];
+    for (const r of records) {
+        r.repos.forEach((x) => repos.add(x));
+        r.instanceIdsReferenced.forEach((x) => instanceIds.add(x));
+        indexed.push({
+            id: r.id,
+            repos: [...r.repos].sort(),
+            instanceIds: [...r.instanceIdsReferenced].sort(),
+            sketch: minhashSketch(tokenize(r.text)),
+        });
+    }
+    // Deterministic total order: primary by id, tiebreak by canonical content so
+    // duplicate ids never make the cid input-order-dependent (spec §4.3).
+    indexed.sort((a, b) => a.id.localeCompare(b.id) || JSON.stringify(a).localeCompare(JSON.stringify(b)));
+    return { repos: [...repos].sort(), instanceIds: [...instanceIds].sort(), records: indexed };
+}
+export function corpusSnapshotCid(index) {
+    // RFC 8785 canonical JSON (key-sorted, cross-language reproducible) so a third
+    // party can recompute this Merkle root to re-check the disjointness proof (spec §4.3).
+    // Arrays are already fully sorted by buildCorpusIndex; canonicalJson sorts object keys.
+    return `sha256:${createHash('sha256').update(canonicalJson(index)).digest('hex')}`;
+}
+//# sourceMappingURL=corpus-index.js.map

package/dist/eval/corpus-index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"corpus-index.js","sourceRoot":"","sources":["../../src/eval/corpus-index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AAgBtE,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAClD,OAAO,CAAC,IAAI,EAAE,CAAC;AACjB,CAAC;AAED,kFAAkF;AAClF,SAAS,KAAK,CAAC,CAAS,EAAE,IAAY;IACpC,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAgB,EAAE,SAAS,GAAG,EAAE;IAC5D,MAAM,IAAI,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IAClC,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,IAAI,GAAG,GAAG,UAAU,CAAC;QACrB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;YACnC,IAAI,CAAC,GAAG,GAAG;gBAAE,GAAG,GAAG,CAAC,CAAC;QACvB,CAAC;QACD,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAuB;IACtD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;IACtC,MAAM,OAAO,GAAwB,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrC,CAAC,CAAC,qBAAqB,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3D,OAAO,CAAC,IAAI,CAAC;YACX,EAAE,EAAE,CAAC,CAAC,EAAE;YACR,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE;YAC1B,WAAW,EAAE,CAAC,GAAG,CAAC,CAAC,qBAAqB,CAAC,CAAC,IAAI,EAAE;YAChD,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;SACxC,CAAC,CAAC;IACL,CAAC;IACD,6EAA6E;IAC7E,sEAAsE;IACtE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvG,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9F,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,KAAyB;IACzD,+EAA+E;IAC/E,uFAAuF;IACvF,wFAAwF;IACxF,OAAO,UAAU,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;AACrF,CAAC"}

package/dist/eval/disjointness.d.ts

@@ -0,0 +1,26 @@
+import { type CorpusDerivedIndex } from './corpus-index.js';
+export interface SlateTaskForDisjointness {
+    instance_id: string;
+    repo: string;
+    /** Distinctive tokens from the gold patch (changed paths/symbols/PR id). */
+    goldPatchTokens: string[];
+}
+export interface AxisCheck {
+    verdict: 'pass' | 'fail';
+    flaggedPairs: Array<[string, string]>;
+}
+export interface DisjointnessResult {
+    instance: AxisCheck;
+    repo: AxisCheck;
+    lexical: AxisCheck;
+}
+export declare class CorpusContaminationError extends Error {
+    readonly result: DisjointnessResult;
+    constructor(result: DisjointnessResult);
+}
+export declare function checkCorpusDisjoint(slate: SlateTaskForDisjointness[], index: CorpusDerivedIndex, opts?: {
+    lexicalJaccardThreshold?: number;
+}): DisjointnessResult;
+export declare function assertCorpusDisjoint(slate: SlateTaskForDisjointness[], index: CorpusDerivedIndex, opts?: {
+    lexicalJaccardThreshold?: number;
+}): DisjointnessResult;

package/dist/eval/disjointness.js

@@ -0,0 +1,67 @@
+import { minhashSketch } from './corpus-index.js';
+export class CorpusContaminationError extends Error {
+    result;
+    constructor(result) {
+        const fails = ['instance', 'repo', 'lexical'].filter((a) => result[a].verdict === 'fail');
+        super(`corpus contamination on axes [${fails.join(', ')}]: ${JSON.stringify(fails.flatMap((a) => result[a].flaggedPairs))}`);
+        this.result = result;
+        this.name = 'CorpusContaminationError';
+    }
+}
+/** MinHash Jaccard estimate between two equal-length sketches. An all-zero
+ *  sketch is corpus-index's empty-token-set sentinel; a set with no tokens has
+ *  no meaningful overlap, so return 0 rather than letting the sentinel zeros
+ *  count as matches (which would false-positive a degenerate gold patch or an
+ *  empty corpus record into a contamination flag). */
+function sketchJaccard(a, b) {
+    if (a.length === 0)
+        return 0;
+    if (a.length !== b.length) {
+        throw new Error(`sketchJaccard: sketch length mismatch (${a.length} vs ${b.length})`);
+    }
+    if (a.every((x) => x === 0) || b.every((x) => x === 0))
+        return 0;
+    let same = 0;
+    for (let i = 0; i < a.length; i++)
+        if (a[i] === b[i])
+            same++;
+    return same / a.length;
+}
+export function checkCorpusDisjoint(slate, index, opts = {}) {
+    const threshold = opts.lexicalJaccardThreshold ?? 0.15;
+    const corpusRepos = new Set(index.repos);
+    const corpusInstances = new Set(index.instanceIds);
+    const instance = { verdict: 'pass', flaggedPairs: [] };
+    const repo = { verdict: 'pass', flaggedPairs: [] };
+    const lexical = { verdict: 'pass', flaggedPairs: [] };
+    for (const task of slate) {
+        if (corpusInstances.has(task.instance_id)) {
+            instance.verdict = 'fail';
+            for (const rec of index.records)
+                if (rec.instanceIds.includes(task.instance_id))
+                    instance.flaggedPairs.push([task.instance_id, rec.id]);
+        }
+        if (corpusRepos.has(task.repo)) {
+            repo.verdict = 'fail';
+            for (const rec of index.records)
+                if (rec.repos.includes(task.repo))
+                    repo.flaggedPairs.push([task.instance_id, rec.id]);
+        }
+        const goldSketch = minhashSketch(task.goldPatchTokens);
+        for (const rec of index.records) {
+            if (sketchJaccard(goldSketch, rec.sketch) >= threshold) {
+                lexical.verdict = 'fail';
+                lexical.flaggedPairs.push([task.instance_id, rec.id]);
+            }
+        }
+    }
+    return { instance, repo, lexical };
+}
+export function assertCorpusDisjoint(slate, index, opts = {}) {
+    const result = checkCorpusDisjoint(slate, index, opts);
+    if (result.instance.verdict === 'fail' || result.repo.verdict === 'fail' || result.lexical.verdict === 'fail') {
+        throw new CorpusContaminationError(result);
+    }
+    return result;
+}
+//# sourceMappingURL=disjointness.js.map

package/dist/eval/disjointness.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"disjointness.js","sourceRoot":"","sources":["../../src/eval/disjointness.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAA2B,MAAM,mBAAmB,CAAC;AAY3E,MAAM,OAAO,wBAAyB,SAAQ,KAAK;IACrB;IAA5B,YAA4B,MAA0B;QACpD,MAAM,KAAK,GAAI,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,CAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,MAAM,CAAC,CAAC;QACrG,KAAK,CAAC,iCAAiC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,SAAS,CACzE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAC7C,EAAE,CAAC,CAAC;QAJqB,WAAM,GAAN,MAAM,CAAoB;QAKpD,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAC;IACzC,CAAC;CACF;AAED;;;;sDAIsD;AACtD,SAAS,aAAa,CAAC,CAAW,EAAE,CAAW;IAC7C,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7B,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACxF,CAAC;IACD,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IACjE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAAE,IAAI,EAAE,CAAC;IAC7D,OAAO,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC;AACzB,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,KAAiC,EACjC,KAAyB,EACzB,OAA6C,EAAE;IAE/C,MAAM,SAAS,GAAG,IAAI,CAAC,uBAAuB,IAAI,IAAI,CAAC;IACvD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAEnD,MAAM,QAAQ,GAAc,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;IAClE,MAAM,IAAI,GAAc,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;IAC9D,MAAM,OAAO,GAAc,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE,EAAE,CAAC;IAEjE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YAC1C,QAAQ,CAAC,OAAO,GAAG,MAAM,CAAC;YAC1B,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,OAAO;gBAAE,IAAI,GAAG,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC;oBAAE,QAAQ,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC1I,CAAC;QACD,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;YACtB,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,OAAO;gBAAE,IAAI,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACzH,CAAC;QACD,MAAM,UAAU,GAAG,aAAa,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YAChC,IAAI,aAAa,CAAC,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;gBACvD,OAAO,CAAC,OAAO,GAAG,MAAM,CAAC;gBACzB,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,WAAW,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,KAAiC,EACjC,KAAyB,EACzB,OAA6C,EAAE;IAE/C,MAAM,MAAM,GAAG,mBAAmB,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,CAAC,CAAC;IACvD,IAAI,MAAM,CAAC,QAAQ,CAAC,OAAO,KAAK,MAAM,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,KAAK,MAAM,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,KAAK,MAAM,EAAE,CAAC;QAC9G,MAAM,IAAI,wBAAwB,CAAC,MAAM,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/eval/mcnemar-power.d.ts

@@ -0,0 +1,14 @@
+/** Acklam's rational approximation to the inverse normal CDF (|err| < 1.15e-9). */
+export declare function zFor(p: number): number;
+export interface SampleSizeOpts {
+    alpha?: number;
+    power?: number;
+}
+/**
+ * Connor (1987) total task-PAIRS N for McNemar's test.
+ * pb = P(B passes, A fails); pc = P(A passes, B fails).
+ */
+export declare function mcnemarSampleSize(pb: number, pc: number, opts?: SampleSizeOpts): {
+    pairs: number;
+    discordant: number;
+};

package/dist/eval/mcnemar-power.js

@@ -0,0 +1,40 @@
+/** Acklam's rational approximation to the inverse normal CDF (|err| < 1.15e-9). */
+export function zFor(p) {
+    if (p <= 0 || p >= 1)
+        throw new Error('zFor expects 0 < p < 1');
+    const a = [-3.969683028665376e+01, 2.209460984245205e+02, -2.759285104469687e+02, 1.383577518672690e+02, -3.066479806614716e+01, 2.506628277459239e+00];
+    const b = [-5.447609879822406e+01, 1.615858368580409e+02, -1.556989798598866e+02, 6.680131188771972e+01, -1.328068155288572e+01];
+    const c = [-7.784894002430293e-03, -3.223964580411365e-01, -2.400758277161838e+00, -2.549732539343734e+00, 4.374664141464968e+00, 2.938163982698783e+00];
+    const d = [7.784695709041462e-03, 3.224671290700398e-01, 2.445134137142996e+00, 3.754408661907416e+00];
+    const pl = 0.02425;
+    let q, r;
+    if (p < pl) {
+        q = Math.sqrt(-2 * Math.log(p));
+        return (((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
+    }
+    if (p > 1 - pl) {
+        q = Math.sqrt(-2 * Math.log(1 - p));
+        return -(((((c[0] * q + c[1]) * q + c[2]) * q + c[3]) * q + c[4]) * q + c[5]) / ((((d[0] * q + d[1]) * q + d[2]) * q + d[3]) * q + 1);
+    }
+    q = p - 0.5;
+    r = q * q;
+    return (((((a[0] * r + a[1]) * r + a[2]) * r + a[3]) * r + a[4]) * r + a[5]) * q / (((((b[0] * r + b[1]) * r + b[2]) * r + b[3]) * r + b[4]) * r + 1);
+}
+/**
+ * Connor (1987) total task-PAIRS N for McNemar's test.
+ * pb = P(B passes, A fails); pc = P(A passes, B fails).
+ */
+export function mcnemarSampleSize(pb, pc, opts = {}) {
+    const alpha = opts.alpha ?? 0.05;
+    const power = opts.power ?? 0.8;
+    const diff = pb - pc;
+    if (diff <= 0)
+        throw new Error('mcnemarSampleSize expects pb > pc (a positive effect)');
+    const pd = pb + pc;
+    const za = zFor(1 - alpha / 2);
+    const zb = zFor(power);
+    const m = Math.pow(za * Math.sqrt(pd) + zb * Math.sqrt(pd - diff * diff), 2) / (diff * diff);
+    const pairs = m / pd;
+    return { pairs: Math.ceil(pairs), discordant: Math.ceil(m) };
+}
+//# sourceMappingURL=mcnemar-power.js.map

package/dist/eval/mcnemar-power.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mcnemar-power.js","sourceRoot":"","sources":["../../src/eval/mcnemar-power.ts"],"names":[],"mappings":"AAAA,mFAAmF;AACnF,MAAM,UAAU,IAAI,CAAC,CAAS;IAC5B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAChE,MAAM,CAAC,GAAG,CAAC,CAAC,qBAAqB,EAAE,qBAAqB,EAAE,CAAC,qBAAqB,EAAE,qBAAqB,EAAE,CAAC,qBAAqB,EAAE,qBAAqB,CAAC,CAAC;IACxJ,MAAM,CAAC,GAAG,CAAC,CAAC,qBAAqB,EAAE,qBAAqB,EAAE,CAAC,qBAAqB,EAAE,qBAAqB,EAAE,CAAC,qBAAqB,CAAC,CAAC;IACjI,MAAM,CAAC,GAAG,CAAC,CAAC,qBAAqB,EAAE,CAAC,qBAAqB,EAAE,CAAC,qBAAqB,EAAE,CAAC,qBAAqB,EAAE,qBAAqB,EAAE,qBAAqB,CAAC,CAAC;IACzJ,MAAM,CAAC,GAAG,CAAC,qBAAqB,EAAE,qBAAqB,EAAE,qBAAqB,EAAE,qBAAqB,CAAC,CAAC;IACvG,MAAM,EAAE,GAAG,OAAO,CAAC;IACnB,IAAI,CAAS,EAAE,CAAS,CAAC;IACzB,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;QAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC;IAAC,CAAC;IAC5J,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC;QAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC;IAAC,CAAC;IACrK,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAC,CAAC,GAAC,CAAC,CAAC,CAAC;AACzH,CAAC;AAID;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,EAAU,EAAE,EAAU,EAAE,OAAuB,EAAE;IACjF,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;IACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,GAAG,CAAC;IAChC,MAAM,IAAI,GAAG,EAAE,GAAG,EAAE,CAAC;IACrB,IAAI,IAAI,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IACxF,MAAM,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;IACnB,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC;IAC/B,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IACvB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IAC7F,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,CAAC;IACrB,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/D,CAAC"}

package/dist/eval/orchestrator.d.ts

@@ -18,7 +18,6 @@
  * multi-run averaging.
  */
 import type { Harness, HarnessContext } from '../harnesses/types.js';
-import type { SweRebenchV2Evaluator } from '../harnesses/impls/swe-rebench-v2-evaluator/index.js';
 import type { HarnessCheckpointManifest } from '../vendor/@jinn-network/sdk/dist/checkpoint.js';
 import type { LoadedHeldOutSlate } from '../solver-types/_swe-rebench-v2-held-out-slate.js';
 import type { ResolvedSlateTask } from './resolve-slate-tasks.js';
@@ -96,10 +95,43 @@ export type RunHarnessOnceForEval = (params: {
         patch: string;
     };
 }>;
+/**
+ * The grade verdict the orchestrator consumes. It reads exactly two fields:
+ * `passed_match` (→ per-task pass/fail) and `test_log` (→ recorded excerpt).
+ * This is the swe-rebench-v2 verdict surface; the repo-native JinnRepoEvaluator
+ * is projected onto it by the adapter in eval.ts. Anything unscorable is
+ * signalled by THROWING (the orchestrator's catch records `unscorable`) — never
+ * by a verdict flag — so both backends agree on the unscorable contract.
+ */
+export interface EvalGradeVerdict {
+    passed_match: boolean;
+    test_log: string;
+}
+/**
+ * Minimal structural evaluator the orchestrator depends on. `SweRebenchV2Evaluator`
+ * satisfies it directly (its verdict is a superset of `EvalGradeVerdict`); the
+ * jinn-repo path supplies an adapter in eval.ts that wraps `JinnRepoEvaluator`,
+ * translating the call and projecting its `{ passed, unscorable, logExcerpt }`
+ * result onto this verdict (throwing on unscorable). `task`/`solutionPayload`/`row`
+ * are the swe-rebench-v2 grade call; the orchestrator passes them verbatim and
+ * the backend adapter interprets them.
+ */
+export interface SolutionEvaluator {
+    grade(args: {
+        task: {
+            instance_id: string;
+        };
+        solutionPayload: {
+            schemaVersion: string;
+            patch: string;
+        };
+        row?: unknown;
+    }): Promise<EvalGradeVerdict>;
+}
 export interface EvalOrchestratorDeps {
     harness: Harness;
     fetchImplStateDirToLocal(implStateDirCid: string, targetDir: string): Promise<string>;
-    evaluator: SweRebenchV2Evaluator;
+    evaluator: SolutionEvaluator;
     runHarnessOnce: RunHarnessOnceForEval;
     store: {
         recordEvalResult(args: EvalResultRecord): void;
@@ -151,11 +183,40 @@ export interface EvalRunResult {
     /** What was actually graded (local impl-state), not the checkpoint identity. */
     evaluated: EvalProvenance;
 }
+/**
+ * One slate instance normalised for the backend-agnostic run/grade loop. The
+ * BACKEND (eval.ts) owns construction of both `harnessTask` (the LEAK-CONTROLLED
+ * view the solver sees — for jinn-repo this is `solverView(item)`, never the
+ * pool item) and `gradeTask`/`row` (what the evaluator needs — for jinn-repo the
+ * full pool item incl. gold tests). The swe-rebench-v2 path derives these from
+ * `tasksWithRows` via the default mapping below, unchanged.
+ */
+export interface EvalSlateInstance {
+    instance_id: string;
+    /** Task handed to the solver harness — already leak-controlled by the backend. */
+    harnessTask: NonNullable<HarnessContext['task']>;
+    /** Backend-specific grade payload; passed verbatim as the evaluator's `task`. */
+    gradeTask: {
+        instance_id: string;
+    };
+    /** Solution payload `schemaVersion` the evaluator expects. */
+    solutionSchemaVersion: string;
+    /** swe-rebench-v2 HF row reused at grade time; absent for repo-native. */
+    row?: unknown;
+}
 export declare function runEval(args: {
     checkpointManifest: HarnessCheckpointManifest;
     checkpointCid: string;
     slate: LoadedHeldOutSlate;
-    tasksWithRows: ResolvedSlateTask[];
+    /**
+     * swe-rebench-v2 slate tasks (the historical input). When given, each is
+     * normalised via the default swe-rebench mapping. Mutually informative with
+     * `slateInstances`: a backend that builds its own leak-controlled instances
+     * (jinn-repo) passes `slateInstances` directly instead.
+     */
+    tasksWithRows?: ResolvedSlateTask[];
+    /** Pre-built, backend-normalised instances (jinn-repo path). Overrides `tasksWithRows`. */
+    slateInstances?: EvalSlateInstance[];
     parentCheckpointCid: string;
     deps: EvalOrchestratorDeps;
     /** Working dir for the fetched impl-state-dir (defaults to a checkpoint-scoped tmp path). */

package/dist/eval/orchestrator.js

@@ -92,8 +92,32 @@ export class SlateHashMismatchError extends Error {
         this.name = 'SlateHashMismatchError';
     }
 }
+/**
+ * Default swe-rebench-v2 instance mapping — preserves the exact harness `task`
+ * envelope and grade call the orchestrator used before the backend seam existed.
+ */
+function sweRebenchSlateInstance(rt) {
+    return {
+        instance_id: rt.task.instance_id,
+        // Carry the full SweRebenchV2Task in `spec` + the `solverType` dispatch key
+        // so the harness can restore the repo (clone/checkout from
+        // spec.repo/base_commit). id/description/role alone are insufficient.
+        harnessTask: {
+            id: rt.task.instance_id,
+            description: rt.task.problem_statement,
+            role: 'restoration',
+            solverType: 'swe-rebench-v2.v1',
+            spec: rt.task,
+            window: { startTs: 0, endTs: Date.now() + 3_600_000 },
+        },
+        gradeTask: rt.task,
+        solutionSchemaVersion: 'swe-rebench-v2-solution.v1',
+        row: rt.row,
+    };
+}
 export async function runEval(args) {
     const { deps, slate, checkpointCid, parentCheckpointCid } = args;
+    const slateInstances = args.slateInstances ?? (args.tasksWithRows ?? []).map(sweRebenchSlateInstance);
     // Confounder guard (DR-2026-05-28 §2). The store keys aggregates by slate
     // VERSION; if EITHER checkpoint has results recorded under a different slate
     // CONTENT for that version — a content edit that skipped the version bump, or
@@ -122,7 +146,8 @@ export async function runEval(args) {
     // the first run for provenance. Falls back to the manifest digest, which the
     // C1 guard proves equal.
     let evaluatedDigest = args.checkpointManifest.codeDigest;
-    for (const { task, row } of args.tasksWithRows) {
+    for (const inst of slateInstances) {
+        const task = inst.gradeTask;
         let passed;
         let unscorable = false;
         let logExcerpt = '';
@@ -131,27 +156,20 @@ export async function runEval(args) {
                 harness: deps.harness,
                 implStateDir: localImplStateDir,
                 mode: 'frozen',
-                // Carry the full SweRebenchV2Task in `spec` + the `solverType` dispatch
-                // key so the harness can restore the repo (clone/checkout from
-                // spec.repo/base_commit). The harness emits a swe-rebench-v2 patch from
-                // spec; id/description/role alone are insufficient for a real run.
-                task: {
-                    id: task.instance_id,
-                    description: task.problem_statement,
-                    role: 'restoration',
-                    solverType: 'swe-rebench-v2.v1',
-                    spec: task,
-                    window: { startTs: 0, endTs: Date.now() + 3_600_000 },
-                },
+                // The harness task is built by the BACKEND (eval.ts) and is already
+                // leak-controlled (jinn-repo hands the solver `solverView(item)` only —
+                // no gold tests / reference solution). swe-rebench-v2 carries the full
+                // task in `spec` so the harness can restore the repo.
+                task: inst.harnessTask,
             });
             // AC#3: a freeze violation is loud and terminal — do NOT record this
             // instance (its implStateDir mutation taints the run). Thrown OUTSIDE the
             // unscorable catch below so it propagates and aborts the whole eval.
             if (run.violation) {
-                throw new FreezeFenceViolationError(task.instance_id);
+                throw new FreezeFenceViolationError(inst.instance_id);
             }
             if (!run.solution) {
-                throw new Error(`harness produced no solution for instance ${task.instance_id}`);
+                throw new Error(`harness produced no solution for instance ${inst.instance_id}`);
             }
             // C1: on the FIRST successful run, verify the real digest of the evaluated
             // impl-state (computed by the freeze-fence) matches the named checkpoint's
@@ -168,8 +186,8 @@ export async function runEval(args) {
             }
             const verdict = await deps.evaluator.grade({
                 task,
-                solutionPayload: { schemaVersion: 'swe-rebench-v2-solution.v1', patch: run.solution.patch },
-                row,
+                solutionPayload: { schemaVersion: inst.solutionSchemaVersion, patch: run.solution.patch },
+                ...(inst.row !== undefined ? { row: inst.row } : {}),
             });
             passed = verdict.passed_match;
             logExcerpt = verdict.test_log.slice(0, 1000);
@@ -196,14 +214,14 @@ export async function runEval(args) {
             checkpoint_cid: checkpointCid,
             slate_hash: slate.hash,
             slate_version: slate.version,
-            instance_id: task.instance_id,
+            instance_id: inst.instance_id,
             passed,
             unscorable,
             code_digest: args.checkpointManifest.codeDigest,
             run_at_ms: runAtMs,
             test_log_excerpt: logExcerpt,
         });
-        perTask.push({ instance_id: task.instance_id, passed, unscorable });
+        perTask.push({ instance_id: inst.instance_id, passed, unscorable });
     }
     // AC#2: compare child vs parent at the SAME slate version. A parent with no
     // rows is a hard error — no cross-version compare, eval the parent first.

package/dist/eval/orchestrator.js.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/eval/orchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAQH,OAAO,EAAE,YAAY,EAAuB,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,aAAa,EAA2C,MAAM,aAAa,CAAC;AAErF,4EAA4E;AAC5E,MAAM,OAAO,yBAA0B,SAAQ,KAAK;IACtB;IAA5B,YAA4B,UAAkB;QAC5C,KAAK,CACH,sCAAsC,UAAU,qCAAqC;YACnF,qEAAqE,CACxE,CAAC;QAJwB,eAAU,GAAV,UAAU,CAAQ;QAK5C,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,4BAA6B,SAAQ,KAAK;IAEnC;IACA;IACA;IAHlB,YACkB,aAAqB,EACrB,kBAA0B,EAC1B,mBAA2B;QAE3C,KAAK,CACH,8CAA8C,aAAa,IAAI;YAC7D,uBAAuB,kBAAkB,0CAA0C;YACnF,GAAG,mBAAmB,6DAA6D;YACnF,uFAAuF,CAC1F,CAAC;QATc,kBAAa,GAAb,aAAa,CAAQ;QACrB,uBAAkB,GAAlB,kBAAkB,CAAQ;QAC1B,wBAAmB,GAAnB,mBAAmB,CAAQ;QAQ3C,IAAI,CAAC,IAAI,GAAG,8BAA8B,CAAC;IAC7C,CAAC;CACF;AAED,wFAAwF;AACxF,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IACpB;IAA6C;IAAzE,YAA4B,mBAA2B,EAAkB,YAAoB;QAC3F,KAAK,CACH,qBAAqB,mBAAmB,kCAAkC,YAAY,KAAK;YACzF,sFAAsF,CACzF,CAAC;QAJwB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAAkB,iBAAY,GAAZ,YAAY,CAAQ;QAK3F,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AASD;;;;;;;;;;GAUG;AACH,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAE7B;IACA;IACA;IAHlB,YACkB,YAAoB,EACpB,gBAAwB,EACxB,OAAsC;QAEtD,MAAM,MAAM,GAAG,OAAO;aACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,aAAa,aAAa,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;aACjE,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,KAAK,CACH,4FAA4F;YAC1F,GAAG,YAAY,2BAA2B,gBAAgB,QAAQ,MAAM,sBAAsB;YAC9F,gGAAgG;YAChG,yFAAyF;YACzF,qDAAqD,CACxD,CAAC;QAbc,iBAAY,GAAZ,YAAY,CAAQ;QACpB,qBAAgB,GAAhB,gBAAgB,CAAQ;QACxB,YAAO,GAAP,OAAO,CAA+B;QAYtD,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AA8ED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAS7B;IACC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,GAAG,IAAI,CAAC;IAEjE,0EAA0E;IAC1E,6EAA6E;IAC7E,8EAA8E;IAC9E,yEAAyE;IACzE,8EAA8E;IAC9E,8EAA8E;IAC9E,8EAA8E;IAC9E,kCAAkC;IAClC,MAAM,OAAO,GAAG,CAAC,mBAAmB,EAAE,aAAa,CAAC;SACjD,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACb,aAAa,EAAE,GAAG;QAClB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC;KAChG,CAAC,CAAC;SACF,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,sBAAsB,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvE,CAAC;IAED,6EAA6E;IAC7E,qCAAqC;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,kBAAkB,aAAa,EAAE,CAAC;IAC5E,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAC3D,IAAI,CAAC,kBAAkB,CAAC,eAAe,EACvC,YAAY,CACb,CAAC;IAEF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,+EAA+E;IAC/E,6EAA6E;IAC7E,yBAAyB;IACzB,IAAI,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC;IAEzD,KAAK,MAAM,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;QAC/C,IAAI,MAAsB,CAAC;QAC3B,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC;gBACpC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,YAAY,EAAE,iBAAiB;gBAC/B,IAAI,EAAE,QAAQ;gBACd,wEAAwE;gBACxE,+DAA+D;gBAC/D,wEAAwE;gBACxE,mEAAmE;gBACnE,IAAI,EAAE;oBACJ,EAAE,EAAE,IAAI,CAAC,WAAW;oBACpB,WAAW,EAAE,IAAI,CAAC,iBAAiB;oBACnC,IAAI,EAAE,aAAa;oBACnB,UAAU,EAAE,mBAAmB;oBAC/B,IAAI,EAAE,IAA0C;oBAChD,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE;iBACtD;aACF,CAAC,CAAC;YAEH,qEAAqE;YACrE,0EAA0E;YAC1E,qEAAqE;YACrE,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;gBAClB,MAAM,IAAI,yBAAyB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxD,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,6CAA6C,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YACnF,CAAC;YAED,2EAA2E;YAC3E,2EAA2E;YAC3E,4EAA4E;YAC5E,kEAAkE;YAClE,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,aAAa,GAAG,IAAI,CAAC;gBACrB,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC;gBACpD,IAAI,SAAS,IAAI,SAAS,KAAK,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,CAAC;oBAClE,MAAM,IAAI,4BAA4B,CACpC,aAAa,EACb,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAClC,SAAS,CACV,CAAC;gBACJ,CAAC;gBACD,IAAI,SAAS;oBAAE,eAAe,GAAG,SAAS,CAAC;YAC7C,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;gBACzC,IAAI;gBACJ,eAAe,EAAE,EAAE,aAAa,EAAE,4BAA4B,EAAE,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE;gBAC3F,GAAG;aACJ,CAAC,CAAC;YACH,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;YAC9B,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,wEAAwE;YACxE,uEAAuE;YACvE,sEAAsE;YACtE,IAAI,GAAG,YAAY,yBAAyB,IAAI,GAAG,YAAY,4BAA4B,EAAE,CAAC;gBAC5F,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,sEAAsE;YACtE,iEAAiE;YACjE,mEAAmE;YACnE,sEAAsE;YACtE,uEAAuE;YACvE,wEAAwE;YACxE,kCAAkC;YAClC,MAAM,GAAG,IAAI,CAAC;YACd,UAAU,GAAG,IAAI,CAAC;YAClB,UAAU,GAAG,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;YAC1B,cAAc,EAAE,aAAa;YAC7B,UAAU,EAAE,KAAK,CAAC,IAAI;YACtB,aAAa,EAAE,KAAK,CAAC,OAAO;YAC5B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM;YACN,UAAU;YACV,WAAW,EAAE,IAAI,CAAC,kBAAkB,CAAC,UAAU;YAC/C,SAAS,EAAE,OAAO;YAClB,gBAAgB,EAAE,UAAU;SAC7B,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAClF,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC,IAAI,SAAS,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,uBAAuB,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAE3E,MAAM,UAAU,GAAG,YAAY,CAC7B,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,EACxD,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,CAC3D,CAAC;IAEF,6EAA6E;IAC7E,6EAA6E;IAC7E,gEAAgE;IAChE,IAAI,MAAoC,CAAC;IACzC,IAAI,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC9B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7E,MAAM,GAAG,aAAa,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED,OAAO;QACL,OAAO;QACP,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,SAAS,EAAE,EAAE,YAAY,EAAE,iBAAiB,EAAE,UAAU,EAAE,eAAe,EAAE,iBAAiB,EAAE,IAAI,EAAE;KACrG,CAAC;AACJ,CAAC"}
+{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/eval/orchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAOH,OAAO,EAAE,YAAY,EAAuB,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,aAAa,EAA2C,MAAM,aAAa,CAAC;AAErF,4EAA4E;AAC5E,MAAM,OAAO,yBAA0B,SAAQ,KAAK;IACtB;IAA5B,YAA4B,UAAkB;QAC5C,KAAK,CACH,sCAAsC,UAAU,qCAAqC;YACnF,qEAAqE,CACxE,CAAC;QAJwB,eAAU,GAAV,UAAU,CAAQ;QAK5C,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,4BAA6B,SAAQ,KAAK;IAEnC;IACA;IACA;IAHlB,YACkB,aAAqB,EACrB,kBAA0B,EAC1B,mBAA2B;QAE3C,KAAK,CACH,8CAA8C,aAAa,IAAI;YAC7D,uBAAuB,kBAAkB,0CAA0C;YACnF,GAAG,mBAAmB,6DAA6D;YACnF,uFAAuF,CAC1F,CAAC;QATc,kBAAa,GAAb,aAAa,CAAQ;QACrB,uBAAkB,GAAlB,kBAAkB,CAAQ;QAC1B,wBAAmB,GAAnB,mBAAmB,CAAQ;QAQ3C,IAAI,CAAC,IAAI,GAAG,8BAA8B,CAAC;IAC7C,CAAC;CACF;AAED,wFAAwF;AACxF,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IACpB;IAA6C;IAAzE,YAA4B,mBAA2B,EAAkB,YAAoB;QAC3F,KAAK,CACH,qBAAqB,mBAAmB,kCAAkC,YAAY,KAAK;YACzF,sFAAsF,CACzF,CAAC;QAJwB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAAkB,iBAAY,GAAZ,YAAY,CAAQ;QAK3F,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AASD;;;;;;;;;;GAUG;AACH,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAE7B;IACA;IACA;IAHlB,YACkB,YAAoB,EACpB,gBAAwB,EACxB,OAAsC;QAEtD,MAAM,MAAM,GAAG,OAAO;aACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,aAAa,aAAa,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;aACjE,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,KAAK,CACH,4FAA4F;YAC1F,GAAG,YAAY,2BAA2B,gBAAgB,QAAQ,MAAM,sBAAsB;YAC9F,gGAAgG;YAChG,yFAAyF;YACzF,qDAAqD,CACxD,CAAC;QAbc,iBAAY,GAAZ,YAAY,CAAQ;QACpB,qBAAgB,GAAhB,gBAAgB,CAAQ;QACxB,YAAO,GAAP,OAAO,CAA+B;QAYtD,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAgID;;;GAGG;AACH,SAAS,uBAAuB,CAAC,EAAqB;IACpD,OAAO;QACL,WAAW,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW;QAChC,4EAA4E;QAC5E,2DAA2D;QAC3D,sEAAsE;QACtE,WAAW,EAAE;YACX,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW;YACvB,WAAW,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB;YACtC,IAAI,EAAE,aAAa;YACnB,UAAU,EAAE,mBAAmB;YAC/B,IAAI,EAAE,EAAE,CAAC,IAA0C;YACnD,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE;SACtD;QACD,SAAS,EAAE,EAAE,CAAC,IAAI;QAClB,qBAAqB,EAAE,4BAA4B;QACnD,GAAG,EAAE,EAAE,CAAC,GAAG;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAiB7B;IACC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,GAAG,IAAI,CAAC;IACjE,MAAM,cAAc,GAClB,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IAEjF,0EAA0E;IAC1E,6EAA6E;IAC7E,8EAA8E;IAC9E,yEAAyE;IACzE,8EAA8E;IAC9E,8EAA8E;IAC9E,8EAA8E;IAC9E,kCAAkC;IAClC,MAAM,OAAO,GAAG,CAAC,mBAAmB,EAAE,aAAa,CAAC;SACjD,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACb,aAAa,EAAE,GAAG;QAClB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC;KAChG,CAAC,CAAC;SACF,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,sBAAsB,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvE,CAAC;IAED,6EAA6E;IAC7E,qCAAqC;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,kBAAkB,aAAa,EAAE,CAAC;IAC5E,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAC3D,IAAI,CAAC,kBAAkB,CAAC,eAAe,EACvC,YAAY,CACb,CAAC;IAEF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,+EAA+E;IAC/E,6EAA6E;IAC7E,yBAAyB;IACzB,IAAI,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC;IAEzD,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;QAC5B,IAAI,MAAsB,CAAC;QAC3B,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC;gBACpC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,YAAY,EAAE,iBAAiB;gBAC/B,IAAI,EAAE,QAAQ;gBACd,oEAAoE;gBACpE,wEAAwE;gBACxE,uEAAuE;gBACvE,sDAAsD;gBACtD,IAAI,EAAE,IAAI,CAAC,WAAW;aACvB,CAAC,CAAC;YAEH,qEAAqE;YACrE,0EAA0E;YAC1E,qEAAqE;YACrE,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;gBAClB,MAAM,IAAI,yBAAyB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxD,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,6CAA6C,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YACnF,CAAC;YAED,2EAA2E;YAC3E,2EAA2E;YAC3E,4EAA4E;YAC5E,kEAAkE;YAClE,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,aAAa,GAAG,IAAI,CAAC;gBACrB,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC;gBACpD,IAAI,SAAS,IAAI,SAAS,KAAK,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,CAAC;oBAClE,MAAM,IAAI,4BAA4B,CACpC,aAAa,EACb,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAClC,SAAS,CACV,CAAC;gBACJ,CAAC;gBACD,IAAI,SAAS;oBAAE,eAAe,GAAG,SAAS,CAAC;YAC7C,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;gBACzC,IAAI;gBACJ,eAAe,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,qBAAqB,EAAE,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE;gBACzF,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrD,CAAC,CAAC;YACH,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;YAC9B,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,wEAAwE;YACxE,uEAAuE;YACvE,sEAAsE;YACtE,IAAI,GAAG,YAAY,yBAAyB,IAAI,GAAG,YAAY,4BAA4B,EAAE,CAAC;gBAC5F,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,sEAAsE;YACtE,iEAAiE;YACjE,mEAAmE;YACnE,sEAAsE;YACtE,uEAAuE;YACvE,wEAAwE;YACxE,kCAAkC;YAClC,MAAM,GAAG,IAAI,CAAC;YACd,UAAU,GAAG,IAAI,CAAC;YAClB,UAAU,GAAG,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;YAC1B,cAAc,EAAE,aAAa;YAC7B,UAAU,EAAE,KAAK,CAAC,IAAI;YACtB,aAAa,EAAE,KAAK,CAAC,OAAO;YAC5B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM;YACN,UAAU;YACV,WAAW,EAAE,IAAI,CAAC,kBAAkB,CAAC,UAAU;YAC/C,SAAS,EAAE,OAAO;YAClB,gBAAgB,EAAE,UAAU;SAC7B,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAClF,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC,IAAI,SAAS,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,uBAAuB,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAE3E,MAAM,UAAU,GAAG,YAAY,CAC7B,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,EACxD,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,CAC3D,CAAC;IAEF,6EAA6E;IAC7E,6EAA6E;IAC7E,gEAAgE;IAChE,IAAI,MAAoC,CAAC;IACzC,IAAI,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC9B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7E,MAAM,GAAG,aAAa,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED,OAAO;QACL,OAAO;QACP,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,SAAS,EAAE,EAAE,YAAY,EAAE,iBAAiB,EAAE,UAAU,EAAE,eAAe,EAAE,iBAAiB,EAAE,IAAI,EAAE;KACrG,CAAC;AACJ,CAAC"}

package/dist/eval/paired.d.ts

@@ -1,68 +1,2 @@
-/**
- * Paired (matched-design) comparison for the held-out exam.
- *
- * The exam scores the SAME slate against the before- and after-checkpoints, so
- * the two result sets are MATCHED, not independent. The marginal test in
- * `wilson.ts` (`compareRates`) compares two INDEPENDENT Wilson intervals and
- * calls a delta trustworthy only when they are disjoint. On a matched design
- * that is statistically wasteful: it carries the full between-instance
- * difficulty variance ("this bug is just hard") in BOTH intervals, which swamps
- * a real, consistent within-instance improvement. Example: a learner that flips
- * 3 hard instances fail→pass and regresses none moves 4/10→7/10 — two Wilson
- * intervals ([17,69]% vs [40,89]%) overlap → "within-noise", even though every
- * observed change was an improvement.
- *
- * McNemar's test is the textbook test for matched binary data: it looks ONLY at
- * the discordant pairs (instances that flipped) and asks whether the flips are
- * asymmetric beyond chance. This is NOT a weaker bar — it is the CORRECT test
- * for the design, and it still gates the claim on significance (exact two-sided
- * p < alpha) AND on the improvement direction (more fail→pass than pass→fail).
- * It is reported ALONGSIDE the conservative marginal verdict, never replacing it
- * — so the exam is strengthened, never weakened (DR-2026-06-02-b §2a).
- *
- * R=1 (one run per instance) ships here: McNemar exact on per-instance flips.
- * R>1 (per-instance pass RATES → Wilcoxon signed-rank / paired bootstrap) layers
- * on once `eval_results` supports appended runs (DR-2026-06-02-b §2b).
- */
-/** Minimal per-instance result (subset of PerTaskResult / EvalResultRow). */
-export interface PairedInput {
-    instance_id: string;
-    /** null when unscorable. */
-    passed: boolean | null;
-    unscorable: boolean;
-}
-export type PairedVerdict = 'trustworthy' | 'within-noise';
-export interface PairedComparison {
-    /** Instances scorable (passed !== null) in BOTH arms — the matched pairs. */
-    pairs: number;
-    /** b: before fail → after pass (improvements). */
-    improved: number;
-    /** c: before pass → after fail (regressions). */
-    regressed: number;
-    concordantPass: number;
-    concordantFail: number;
-    /** Instances unscorable/absent in either arm — excluded from the paired test. */
-    excluded: number;
-    /** Exact two-sided McNemar p-value over the discordant pairs. */
-    pValue: number;
-    /** 'trustworthy' iff pValue < alpha AND improved > regressed; else 'within-noise'. */
-    verdict: PairedVerdict;
-}
-/**
- * Exact two-sided McNemar p-value for discordant counts `b` (improvements) and
- * `c` (regressions). Under H0 each discordant pair is an independent fair coin,
- * so the count of one direction is Binomial(n=b+c, 0.5); the two-sided p is
- * `2 * P(X <= min(b,c))`, capped at 1. No `b==c==0` division (returns 1: no
- * evidence). Computed iteratively (term ratio) so it is exact and overflow-free
- * for any slate size.
- */
-export declare function mcnemarExact(b: number, c: number): number;
-/**
- * Paired McNemar comparison of two matched per-instance result sets. An instance
- * contributes a pair ONLY when it is scorable (passed !== null, not unscorable)
- * in BOTH arms — an unscorable/missing side is excluded (honest: a disk-skip or
- * harness failure is never coerced into a flip). `alpha` defaults to 0.05.
- */
-export declare function comparePaired(before: readonly PairedInput[], after: readonly PairedInput[], opts?: {
-    alpha?: number;
-}): PairedComparison;
+/** Compatibility export for the core-owned matched-evaluation helpers. */
+export { comparePaired, mcnemarExact, type PairedComparison, type PairedInput, type PairedVerdict, } from '../vendor/@jinn-network/core/dist/index.js';