@jinn-network/client 0.1.8 → 0.1.9-canary.02befda9

package/dist/eval/orchestrator.d.ts

@@ -0,0 +1,224 @@
+/**
+ * `jinn eval` held-out checkpoint orchestrator (issue #818).
+ *
+ * Runs a held-out slate against a checkpoint in FROZEN mode, persists per-task
+ * pass/fail, and emits a Wilson-CI resolved-rate comparison vs the parent
+ * checkpoint. Every external boundary is constructor-injected — this is the
+ * thin slice #819 drives a deterministic slate against with NO Docker/IPFS.
+ *
+ * Acceptance criteria:
+ *   AC#1 — run the slate frozen, write per-task pass/fail.
+ *   AC#2 — emit a resolved-rate comparison vs the parent with a CI.
+ *   AC#3 — the freeze-fence holds: a `runHarnessOnce` `{ violation }` throws
+ *          LOUD and the instance is NOT recorded (no implStateDir mutation
+ *          slips through; enforcement lives in `runHarnessWithFreezeFence`).
+ *
+ * Per log/decisions/2026-05-28-rl-eval-measurement.md §4: v1-simple. Only large
+ * deltas are trustworthy (disjoint Wilson intervals). No seed control, no
+ * multi-run averaging.
+ */
+import type { Harness, HarnessContext } from '../harnesses/types.js';
+import type { HarnessCheckpointManifest } from '../vendor/@jinn-network/sdk/dist/checkpoint.js';
+import type { LoadedHeldOutSlate } from '../solver-types/_swe-rebench-v2-held-out-slate.js';
+import type { ResolvedSlateTask } from './resolve-slate-tasks.js';
+import type { EvalAggregate, EvalResultRecord } from '../store/store.js';
+import { type RateComparison } from './wilson.js';
+import { type PairedComparison, type PairedInput } from './paired.js';
+/** Thrown when `runHarnessOnce` reports a freeze-fence violation (AC#3). */
+export declare class FreezeFenceViolationError extends Error {
+    readonly instanceId: string;
+    constructor(instanceId: string);
+}
+/**
+ * Thrown when the locally-evaluated impl-state's codeDigest does not match the
+ * named checkpoint's manifest (C1). Guards against persisting + comparing
+ * results under a checkpoint name while the operator's local impl-state has
+ * drifted from that checkpoint.
+ */
+export declare class CheckpointStateMismatchError extends Error {
+    readonly checkpointCid: string;
+    readonly manifestCodeDigest: string;
+    readonly evaluatedCodeDigest: string;
+    constructor(checkpointCid: string, manifestCodeDigest: string, evaluatedCodeDigest: string);
+}
+/** Thrown when the parent checkpoint has no aggregate for this slate version (AC#2). */
+export declare class ParentNotEvaluatedError extends Error {
+    readonly parentCheckpointCid: string;
+    readonly slateVersion: string;
+    constructor(parentCheckpointCid: string, slateVersion: string);
+}
+/** A checkpoint whose recorded results carry slate hashes other than the current one. */
+export interface SlateHashDrift {
+    checkpointCid: string;
+    /** The drifted hashes (each != the current slate hash). */
+    hashes: string[];
+}
+/**
+ * Thrown when EITHER compared checkpoint was scored on a DIFFERENT slate
+ * *content* under the same version label (DR-2026-05-28 §2 — confounder control).
+ * A slate is content-addressed and scores are comparable ONLY within identical
+ * content; the store keys aggregates by version, so a content edit that skipped
+ * the version bump (or re-derived the declared hash — which the slate loader
+ * admits passes), or stale rows surviving `recordEvalResult`'s by-instance
+ * upsert, would let the comparison silently compare two checkpoints scored on
+ * different task sets. That is confounder #1 (task-selection) — exactly what the
+ * held-out exam exists to defeat — so we refuse the comparison loudly instead.
+ */
+export declare class SlateHashMismatchError extends Error {
+    readonly slateVersion: string;
+    readonly currentSlateHash: string;
+    readonly drifted: ReadonlyArray<SlateHashDrift>;
+    constructor(slateVersion: string, currentSlateHash: string, drifted: ReadonlyArray<SlateHashDrift>);
+}
+/**
+ * Injected `runHarnessOnce`-shaped boundary. Mirrors the production
+ * `runHarnessOnce` (freeze-fence + mode propagation) but also surfaces the
+ * harness `solution` so the orchestrator can extract the patch to grade. On a
+ * freeze violation it returns `{ violation }` and no solution.
+ */
+export type RunHarnessOnceForEval = (params: {
+    harness: Harness;
+    implStateDir: string;
+    mode: 'train' | 'frozen';
+    task?: HarnessContext['task'];
+}) => Promise<{
+    envelope?: {
+        executor: {
+            mode: 'train' | 'frozen';
+            codeDigest: string;
+        };
+    };
+    violation?: {
+        taskId: string;
+    };
+    solution?: {
+        patch: string;
+    };
+}>;
+/**
+ * The grade verdict the orchestrator consumes. It reads exactly two fields:
+ * `passed_match` (→ per-task pass/fail) and `test_log` (→ recorded excerpt).
+ * This is the swe-rebench-v2 verdict surface; the repo-native JinnRepoEvaluator
+ * is projected onto it by the adapter in eval.ts. Anything unscorable is
+ * signalled by THROWING (the orchestrator's catch records `unscorable`) — never
+ * by a verdict flag — so both backends agree on the unscorable contract.
+ */
+export interface EvalGradeVerdict {
+    passed_match: boolean;
+    test_log: string;
+}
+/**
+ * Minimal structural evaluator the orchestrator depends on. `SweRebenchV2Evaluator`
+ * satisfies it directly (its verdict is a superset of `EvalGradeVerdict`); the
+ * jinn-repo path supplies an adapter in eval.ts that wraps `JinnRepoEvaluator`,
+ * translating the call and projecting its `{ passed, unscorable, logExcerpt }`
+ * result onto this verdict (throwing on unscorable). `task`/`solutionPayload`/`row`
+ * are the swe-rebench-v2 grade call; the orchestrator passes them verbatim and
+ * the backend adapter interprets them.
+ */
+export interface SolutionEvaluator {
+    grade(args: {
+        task: {
+            instance_id: string;
+        };
+        solutionPayload: {
+            schemaVersion: string;
+            patch: string;
+        };
+        row?: unknown;
+    }): Promise<EvalGradeVerdict>;
+}
+export interface EvalOrchestratorDeps {
+    harness: Harness;
+    fetchImplStateDirToLocal(implStateDirCid: string, targetDir: string): Promise<string>;
+    evaluator: SolutionEvaluator;
+    runHarnessOnce: RunHarnessOnceForEval;
+    store: {
+        recordEvalResult(args: EvalResultRecord): void;
+        getEvalAggregate(checkpointCid: string, slateVersion: string): EvalAggregate;
+        /** Distinct slate_hash values the parent's results were recorded under (drift guard). */
+        getEvalSlateHashes(checkpointCid: string, slateVersion: string): string[];
+        /**
+         * Optional: per-instance results for a (checkpoint, slate version). When
+         * present, the orchestrator also emits the paired (matched-design) McNemar
+         * verdict (DR-2026-06-02-b §2a) — the same slate is scored before & after, so
+         * the matched test is the statistically correct one and is far more powerful
+         * than the marginal disjoint-interval test. Reported ALONGSIDE `comparison`,
+         * never replacing it (the exam is strengthened, not weakened).
+         */
+        getEvalResults?(checkpointCid: string, slateVersion: string): PairedInput[];
+    };
+}
+export interface PerTaskResult {
+    instance_id: string;
+    /** null when unscorable. */
+    passed: boolean | null;
+    unscorable: boolean;
+}
+/**
+ * Provenance of the graded artifact (Legibility). The CLI grades the operator's
+ * LOCAL frozen impl-state — verified == the named checkpoint via the C1
+ * codeDigest guard (CheckpointStateMismatchError) — NOT a state re-fetched from
+ * the checkpoint CID. Surfacing this stops a reader from assuming the tool
+ * fetched and graded the published checkpoint's state.
+ */
+export interface EvalProvenance {
+    /** Local impl-state directory that was actually run. */
+    implStateDir: string;
+    /** Real codeDigest of the evaluated impl-state (verified == manifest.codeDigest). */
+    codeDigest: string;
+    /** Always true on a returned result — a mismatch throws CheckpointStateMismatchError. */
+    matchedCheckpoint: true;
+}
+export interface EvalRunResult {
+    perTask: PerTaskResult[];
+    comparison: RateComparison;
+    /**
+     * Paired (matched-design) McNemar verdict vs the parent — present only when
+     * the store exposes `getEvalResults`. The marginal `comparison` above is
+     * conservative; this is the statistically correct test for the same-slate
+     * before/after design and is reported alongside it (DR-2026-06-02-b §2a).
+     */
+    paired?: PairedComparison;
+    /** What was actually graded (local impl-state), not the checkpoint identity. */
+    evaluated: EvalProvenance;
+}
+/**
+ * One slate instance normalised for the backend-agnostic run/grade loop. The
+ * BACKEND (eval.ts) owns construction of both `harnessTask` (the LEAK-CONTROLLED
+ * view the solver sees — for jinn-repo this is `solverView(item)`, never the
+ * pool item) and `gradeTask`/`row` (what the evaluator needs — for jinn-repo the
+ * full pool item incl. gold tests). The swe-rebench-v2 path derives these from
+ * `tasksWithRows` via the default mapping below, unchanged.
+ */
+export interface EvalSlateInstance {
+    instance_id: string;
+    /** Task handed to the solver harness — already leak-controlled by the backend. */
+    harnessTask: NonNullable<HarnessContext['task']>;
+    /** Backend-specific grade payload; passed verbatim as the evaluator's `task`. */
+    gradeTask: {
+        instance_id: string;
+    };
+    /** Solution payload `schemaVersion` the evaluator expects. */
+    solutionSchemaVersion: string;
+    /** swe-rebench-v2 HF row reused at grade time; absent for repo-native. */
+    row?: unknown;
+}
+export declare function runEval(args: {
+    checkpointManifest: HarnessCheckpointManifest;
+    checkpointCid: string;
+    slate: LoadedHeldOutSlate;
+    /**
+     * swe-rebench-v2 slate tasks (the historical input). When given, each is
+     * normalised via the default swe-rebench mapping. Mutually informative with
+     * `slateInstances`: a backend that builds its own leak-controlled instances
+     * (jinn-repo) passes `slateInstances` directly instead.
+     */
+    tasksWithRows?: ResolvedSlateTask[];
+    /** Pre-built, backend-normalised instances (jinn-repo path). Overrides `tasksWithRows`. */
+    slateInstances?: EvalSlateInstance[];
+    parentCheckpointCid: string;
+    deps: EvalOrchestratorDeps;
+    /** Working dir for the fetched impl-state-dir (defaults to a checkpoint-scoped tmp path). */
+    implStateDir?: string;
+}): Promise<EvalRunResult>;

package/dist/eval/orchestrator.js

@@ -0,0 +1,250 @@
+/**
+ * `jinn eval` held-out checkpoint orchestrator (issue #818).
+ *
+ * Runs a held-out slate against a checkpoint in FROZEN mode, persists per-task
+ * pass/fail, and emits a Wilson-CI resolved-rate comparison vs the parent
+ * checkpoint. Every external boundary is constructor-injected — this is the
+ * thin slice #819 drives a deterministic slate against with NO Docker/IPFS.
+ *
+ * Acceptance criteria:
+ *   AC#1 — run the slate frozen, write per-task pass/fail.
+ *   AC#2 — emit a resolved-rate comparison vs the parent with a CI.
+ *   AC#3 — the freeze-fence holds: a `runHarnessOnce` `{ violation }` throws
+ *          LOUD and the instance is NOT recorded (no implStateDir mutation
+ *          slips through; enforcement lives in `runHarnessWithFreezeFence`).
+ *
+ * Per log/decisions/2026-05-28-rl-eval-measurement.md §4: v1-simple. Only large
+ * deltas are trustworthy (disjoint Wilson intervals). No seed control, no
+ * multi-run averaging.
+ */
+import { compareRates } from './wilson.js';
+import { comparePaired } from './paired.js';
+/** Thrown when `runHarnessOnce` reports a freeze-fence violation (AC#3). */
+export class FreezeFenceViolationError extends Error {
+    instanceId;
+    constructor(instanceId) {
+        super(`freeze-fence violation on instance ${instanceId}: the harness mutated implStateDir ` +
+            `during a frozen-mode eval run — refusing to record a tainted result`);
+        this.instanceId = instanceId;
+        this.name = 'FreezeFenceViolationError';
+    }
+}
+/**
+ * Thrown when the locally-evaluated impl-state's codeDigest does not match the
+ * named checkpoint's manifest (C1). Guards against persisting + comparing
+ * results under a checkpoint name while the operator's local impl-state has
+ * drifted from that checkpoint.
+ */
+export class CheckpointStateMismatchError extends Error {
+    checkpointCid;
+    manifestCodeDigest;
+    evaluatedCodeDigest;
+    constructor(checkpointCid, manifestCodeDigest, evaluatedCodeDigest) {
+        super(`local impl-state does not match checkpoint ${checkpointCid}: ` +
+            `manifest codeDigest=${manifestCodeDigest} but the evaluated impl-state hashes to ` +
+            `${evaluatedCodeDigest}. Check out the impl-state for this checkpoint or pass the ` +
+            `correct --impl-state-dir; refusing to record results under a codeDigest that is a lie`);
+        this.checkpointCid = checkpointCid;
+        this.manifestCodeDigest = manifestCodeDigest;
+        this.evaluatedCodeDigest = evaluatedCodeDigest;
+        this.name = 'CheckpointStateMismatchError';
+    }
+}
+/** Thrown when the parent checkpoint has no aggregate for this slate version (AC#2). */
+export class ParentNotEvaluatedError extends Error {
+    parentCheckpointCid;
+    slateVersion;
+    constructor(parentCheckpointCid, slateVersion) {
+        super(`parent checkpoint ${parentCheckpointCid} has no eval results for slate ${slateVersion} — ` +
+            `eval the parent checkpoint first (scores are only comparable within a slate version)`);
+        this.parentCheckpointCid = parentCheckpointCid;
+        this.slateVersion = slateVersion;
+        this.name = 'ParentNotEvaluatedError';
+    }
+}
+/**
+ * Thrown when EITHER compared checkpoint was scored on a DIFFERENT slate
+ * *content* under the same version label (DR-2026-05-28 §2 — confounder control).
+ * A slate is content-addressed and scores are comparable ONLY within identical
+ * content; the store keys aggregates by version, so a content edit that skipped
+ * the version bump (or re-derived the declared hash — which the slate loader
+ * admits passes), or stale rows surviving `recordEvalResult`'s by-instance
+ * upsert, would let the comparison silently compare two checkpoints scored on
+ * different task sets. That is confounder #1 (task-selection) — exactly what the
+ * held-out exam exists to defeat — so we refuse the comparison loudly instead.
+ */
+export class SlateHashMismatchError extends Error {
+    slateVersion;
+    currentSlateHash;
+    drifted;
+    constructor(slateVersion, currentSlateHash, drifted) {
+        const detail = drifted
+            .map((d) => `${d.checkpointCid} carries [${d.hashes.join(', ')}]`)
+            .join('; ');
+        super(`a checkpoint was evaluated against a different slate content under the same version label ` +
+            `${slateVersion}: current slate hash is ${currentSlateHash} but ${detail}. Same version must ` +
+            `mean the same exam — a slate content change is a measurement discontinuity that must bump the ` +
+            `version. Bump the slate version and re-evaluate; refusing to report a delta across two ` +
+            `different task sets (confounder #1, task-selection)`);
+        this.slateVersion = slateVersion;
+        this.currentSlateHash = currentSlateHash;
+        this.drifted = drifted;
+        this.name = 'SlateHashMismatchError';
+    }
+}
+/**
+ * Default swe-rebench-v2 instance mapping — preserves the exact harness `task`
+ * envelope and grade call the orchestrator used before the backend seam existed.
+ */
+function sweRebenchSlateInstance(rt) {
+    return {
+        instance_id: rt.task.instance_id,
+        // Carry the full SweRebenchV2Task in `spec` + the `solverType` dispatch key
+        // so the harness can restore the repo (clone/checkout from
+        // spec.repo/base_commit). id/description/role alone are insufficient.
+        harnessTask: {
+            id: rt.task.instance_id,
+            description: rt.task.problem_statement,
+            role: 'restoration',
+            solverType: 'swe-rebench-v2.v1',
+            spec: rt.task,
+            window: { startTs: 0, endTs: Date.now() + 3_600_000 },
+        },
+        gradeTask: rt.task,
+        solutionSchemaVersion: 'swe-rebench-v2-solution.v1',
+        row: rt.row,
+    };
+}
+export async function runEval(args) {
+    const { deps, slate, checkpointCid, parentCheckpointCid } = args;
+    const slateInstances = args.slateInstances ?? (args.tasksWithRows ?? []).map(sweRebenchSlateInstance);
+    // Confounder guard (DR-2026-05-28 §2). The store keys aggregates by slate
+    // VERSION; if EITHER checkpoint has results recorded under a different slate
+    // CONTENT for that version — a content edit that skipped the version bump, or
+    // stale rows surviving recordEvalResult's by-instance upsert — comparing
+    // child-vs-parent reintroduces confounder #1 (task-selection: two scores over
+    // different task sets). Both arms' hashes are known up front, so detect drift
+    // in EITHER before the loop and refuse — never burn N× real spend producing a
+    // number we'd have to throw away.
+    const drifted = [parentCheckpointCid, checkpointCid]
+        .map((cid) => ({
+        checkpointCid: cid,
+        hashes: deps.store.getEvalSlateHashes(cid, slate.version).filter((hash) => hash !== slate.hash),
+    }))
+        .filter((arm) => arm.hashes.length > 0);
+    if (drifted.length > 0) {
+        throw new SlateHashMismatchError(slate.version, slate.hash, drifted);
+    }
+    // Hoist the impl-state-dir fetch outside the loop: every slate instance runs
+    // against the SAME checkpoint state.
+    const implStateDir = args.implStateDir ?? `/tmp/jinn-eval-${checkpointCid}`;
+    const localImplStateDir = await deps.fetchImplStateDirToLocal(args.checkpointManifest.implStateDirCid, implStateDir);
+    const perTask = [];
+    const runAtMs = Date.now();
+    let digestChecked = false;
+    // Real digest of the evaluated impl-state (from the freeze-fence), captured on
+    // the first run for provenance. Falls back to the manifest digest, which the
+    // C1 guard proves equal.
+    let evaluatedDigest = args.checkpointManifest.codeDigest;
+    for (const inst of slateInstances) {
+        const task = inst.gradeTask;
+        let passed;
+        let unscorable = false;
+        let logExcerpt = '';
+        try {
+            const run = await deps.runHarnessOnce({
+                harness: deps.harness,
+                implStateDir: localImplStateDir,
+                mode: 'frozen',
+                // The harness task is built by the BACKEND (eval.ts) and is already
+                // leak-controlled (jinn-repo hands the solver `solverView(item)` only —
+                // no gold tests / reference solution). swe-rebench-v2 carries the full
+                // task in `spec` so the harness can restore the repo.
+                task: inst.harnessTask,
+            });
+            // AC#3: a freeze violation is loud and terminal — do NOT record this
+            // instance (its implStateDir mutation taints the run). Thrown OUTSIDE the
+            // unscorable catch below so it propagates and aborts the whole eval.
+            if (run.violation) {
+                throw new FreezeFenceViolationError(inst.instance_id);
+            }
+            if (!run.solution) {
+                throw new Error(`harness produced no solution for instance ${inst.instance_id}`);
+            }
+            // C1: on the FIRST successful run, verify the real digest of the evaluated
+            // impl-state (computed by the freeze-fence) matches the named checkpoint's
+            // manifest. Fail fast — before grading any instance — to avoid Docker spend
+            // and to never persist a result under a codeDigest that is a lie.
+            if (!digestChecked) {
+                digestChecked = true;
+                const runDigest = run.envelope?.executor.codeDigest;
+                if (runDigest && runDigest !== args.checkpointManifest.codeDigest) {
+                    throw new CheckpointStateMismatchError(checkpointCid, args.checkpointManifest.codeDigest, runDigest);
+                }
+                if (runDigest)
+                    evaluatedDigest = runDigest;
+            }
+            const verdict = await deps.evaluator.grade({
+                task,
+                solutionPayload: { schemaVersion: inst.solutionSchemaVersion, patch: run.solution.patch },
+                ...(inst.row !== undefined ? { row: inst.row } : {}),
+            });
+            passed = verdict.passed_match;
+            logExcerpt = verdict.test_log.slice(0, 1000);
+        }
+        catch (err) {
+            // A freeze-fence violation taints the run and must stay a LOUD terminal
+            // abort (AC#3). Likewise the C1 digest mismatch — never record results
+            // under a codeDigest that is a lie. Both re-throw and abort the eval.
+            if (err instanceof FreezeFenceViolationError || err instanceof CheckpointStateMismatchError) {
+                throw err;
+            }
+            // Everything else is unscorable: excluded from the denominator, NEVER
+            // coerced to a fail (#476). This covers both grade-side failures
+            // (EvalCouldNotGradeError / InsufficientDiskError) and harness-run
+            // failures (Defect A — harvest missing-artifact, "produced no patch",
+            // clone/timeout, etc.); a harness/infra failure to produce a gradeable
+            // solution is environment-side, not an agent capability fail. The slate
+            // continues to the next instance.
+            passed = null;
+            unscorable = true;
+            logExcerpt = (err instanceof Error ? err.message : String(err)).slice(0, 1000);
+        }
+        deps.store.recordEvalResult({
+            checkpoint_cid: checkpointCid,
+            slate_hash: slate.hash,
+            slate_version: slate.version,
+            instance_id: inst.instance_id,
+            passed,
+            unscorable,
+            code_digest: args.checkpointManifest.codeDigest,
+            run_at_ms: runAtMs,
+            test_log_excerpt: logExcerpt,
+        });
+        perTask.push({ instance_id: inst.instance_id, passed, unscorable });
+    }
+    // AC#2: compare child vs parent at the SAME slate version. A parent with no
+    // rows is a hard error — no cross-version compare, eval the parent first.
+    const parentAgg = deps.store.getEvalAggregate(parentCheckpointCid, slate.version);
+    if (parentAgg.scorable === 0 && parentAgg.unscorable === 0) {
+        throw new ParentNotEvaluatedError(parentCheckpointCid, slate.version);
+    }
+    const childAgg = deps.store.getEvalAggregate(checkpointCid, slate.version);
+    const comparison = compareRates({ passed: childAgg.passed, scorable: childAgg.scorable }, { passed: parentAgg.passed, scorable: parentAgg.scorable });
+    // Paired (matched-design) verdict — same slate scored before & after, so the
+    // matched McNemar test is the correct, higher-power one. Additive: only when
+    // the store exposes per-instance results (DR-2026-06-02-b §2a).
+    let paired;
+    if (deps.store.getEvalResults) {
+        const parentResults = deps.store.getEvalResults(parentCheckpointCid, slate.version);
+        const childResults = deps.store.getEvalResults(checkpointCid, slate.version);
+        paired = comparePaired(parentResults, childResults);
+    }
+    return {
+        perTask,
+        comparison,
+        ...(paired ? { paired } : {}),
+        evaluated: { implStateDir: localImplStateDir, codeDigest: evaluatedDigest, matchedCheckpoint: true },
+    };
+}
+//# sourceMappingURL=orchestrator.js.map

package/dist/eval/orchestrator.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/eval/orchestrator.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAOH,OAAO,EAAE,YAAY,EAAuB,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,aAAa,EAA2C,MAAM,aAAa,CAAC;AAErF,4EAA4E;AAC5E,MAAM,OAAO,yBAA0B,SAAQ,KAAK;IACtB;IAA5B,YAA4B,UAAkB;QAC5C,KAAK,CACH,sCAAsC,UAAU,qCAAqC;YACnF,qEAAqE,CACxE,CAAC;QAJwB,eAAU,GAAV,UAAU,CAAQ;QAK5C,IAAI,CAAC,IAAI,GAAG,2BAA2B,CAAC;IAC1C,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,OAAO,4BAA6B,SAAQ,KAAK;IAEnC;IACA;IACA;IAHlB,YACkB,aAAqB,EACrB,kBAA0B,EAC1B,mBAA2B;QAE3C,KAAK,CACH,8CAA8C,aAAa,IAAI;YAC7D,uBAAuB,kBAAkB,0CAA0C;YACnF,GAAG,mBAAmB,6DAA6D;YACnF,uFAAuF,CAC1F,CAAC;QATc,kBAAa,GAAb,aAAa,CAAQ;QACrB,uBAAkB,GAAlB,kBAAkB,CAAQ;QAC1B,wBAAmB,GAAnB,mBAAmB,CAAQ;QAQ3C,IAAI,CAAC,IAAI,GAAG,8BAA8B,CAAC;IAC7C,CAAC;CACF;AAED,wFAAwF;AACxF,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IACpB;IAA6C;IAAzE,YAA4B,mBAA2B,EAAkB,YAAoB;QAC3F,KAAK,CACH,qBAAqB,mBAAmB,kCAAkC,YAAY,KAAK;YACzF,sFAAsF,CACzF,CAAC;QAJwB,wBAAmB,GAAnB,mBAAmB,CAAQ;QAAkB,iBAAY,GAAZ,YAAY,CAAQ;QAK3F,IAAI,CAAC,IAAI,GAAG,yBAAyB,CAAC;IACxC,CAAC;CACF;AASD;;;;;;;;;;GAUG;AACH,MAAM,OAAO,sBAAuB,SAAQ,KAAK;IAE7B;IACA;IACA;IAHlB,YACkB,YAAoB,EACpB,gBAAwB,EACxB,OAAsC;QAEtD,MAAM,MAAM,GAAG,OAAO;aACnB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,aAAa,aAAa,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;aACjE,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,KAAK,CACH,4FAA4F;YAC1F,GAAG,YAAY,2BAA2B,gBAAgB,QAAQ,MAAM,sBAAsB;YAC9F,gGAAgG;YAChG,yFAAyF;YACzF,qDAAqD,CACxD,CAAC;QAbc,iBAAY,GAAZ,YAAY,CAAQ;QACpB,qBAAgB,GAAhB,gBAAgB,CAAQ;QACxB,YAAO,GAAP,OAAO,CAA+B;QAYtD,IAAI,CAAC,IAAI,GAAG,wBAAwB,CAAC;IACvC,CAAC;CACF;AAgID;;;GAGG;AACH,SAAS,uBAAuB,CAAC,EAAqB;IACpD,OAAO;QACL,WAAW,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW;QAChC,4EAA4E;QAC5E,2DAA2D;QAC3D,sEAAsE;QACtE,WAAW,EAAE;YACX,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW;YACvB,WAAW,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB;YACtC,IAAI,EAAE,aAAa;YACnB,UAAU,EAAE,mBAAmB;YAC/B,IAAI,EAAE,EAAE,CAAC,IAA0C;YACnD,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,EAAE;SACtD;QACD,SAAS,EAAE,EAAE,CAAC,IAAI;QAClB,qBAAqB,EAAE,4BAA4B;QACnD,GAAG,EAAE,EAAE,CAAC,GAAG;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAiB7B;IACC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,aAAa,EAAE,mBAAmB,EAAE,GAAG,IAAI,CAAC;IACjE,MAAM,cAAc,GAClB,IAAI,CAAC,cAAc,IAAI,CAAC,IAAI,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;IAEjF,0EAA0E;IAC1E,6EAA6E;IAC7E,8EAA8E;IAC9E,yEAAyE;IACzE,8EAA8E;IAC9E,8EAA8E;IAC9E,8EAA8E;IAC9E,kCAAkC;IAClC,MAAM,OAAO,GAAG,CAAC,mBAAmB,EAAE,aAAa,CAAC;SACjD,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QACb,aAAa,EAAE,GAAG;QAClB,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC;KAChG,CAAC,CAAC;SACF,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,sBAAsB,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACvE,CAAC;IAED,6EAA6E;IAC7E,qCAAqC;IACrC,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,kBAAkB,aAAa,EAAE,CAAC;IAC5E,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,wBAAwB,CAC3D,IAAI,CAAC,kBAAkB,CAAC,eAAe,EACvC,YAAY,CACb,CAAC;IAEF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,+EAA+E;IAC/E,6EAA6E;IAC7E,yBAAyB;IACzB,IAAI,eAAe,GAAG,IAAI,CAAC,kBAAkB,CAAC,UAAU,CAAC;IAEzD,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC;QAC5B,IAAI,MAAsB,CAAC;QAC3B,IAAI,UAAU,GAAG,KAAK,CAAC;QACvB,IAAI,UAAU,GAAG,EAAE,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC;gBACpC,OAAO,EAAE,IAAI,CAAC,OAAO;gBACrB,YAAY,EAAE,iBAAiB;gBAC/B,IAAI,EAAE,QAAQ;gBACd,oEAAoE;gBACpE,wEAAwE;gBACxE,uEAAuE;gBACvE,sDAAsD;gBACtD,IAAI,EAAE,IAAI,CAAC,WAAW;aACvB,CAAC,CAAC;YAEH,qEAAqE;YACrE,0EAA0E;YAC1E,qEAAqE;YACrE,IAAI,GAAG,CAAC,SAAS,EAAE,CAAC;gBAClB,MAAM,IAAI,yBAAyB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACxD,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;gBAClB,MAAM,IAAI,KAAK,CAAC,6CAA6C,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC;YACnF,CAAC;YAED,2EAA2E;YAC3E,2EAA2E;YAC3E,4EAA4E;YAC5E,kEAAkE;YAClE,IAAI,CAAC,aAAa,EAAE,CAAC;gBACnB,aAAa,GAAG,IAAI,CAAC;gBACrB,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC;gBACpD,IAAI,SAAS,IAAI,SAAS,KAAK,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAAE,CAAC;oBAClE,MAAM,IAAI,4BAA4B,CACpC,aAAa,EACb,IAAI,CAAC,kBAAkB,CAAC,UAAU,EAClC,SAAS,CACV,CAAC;gBACJ,CAAC;gBACD,IAAI,SAAS;oBAAE,eAAe,GAAG,SAAS,CAAC;YAC7C,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;gBACzC,IAAI;gBACJ,eAAe,EAAE,EAAE,aAAa,EAAE,IAAI,CAAC,qBAAqB,EAAE,KAAK,EAAE,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE;gBACzF,GAAG,CAAC,IAAI,CAAC,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrD,CAAC,CAAC;YACH,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;YAC9B,UAAU,GAAG,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,wEAAwE;YACxE,uEAAuE;YACvE,sEAAsE;YACtE,IAAI,GAAG,YAAY,yBAAyB,IAAI,GAAG,YAAY,4BAA4B,EAAE,CAAC;gBAC5F,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,sEAAsE;YACtE,iEAAiE;YACjE,mEAAmE;YACnE,sEAAsE;YACtE,uEAAuE;YACvE,wEAAwE;YACxE,kCAAkC;YAClC,MAAM,GAAG,IAAI,CAAC;YACd,UAAU,GAAG,IAAI,CAAC;YAClB,UAAU,GAAG,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;YAC1B,cAAc,EAAE,aAAa;YAC7B,UAAU,EAAE,KAAK,CAAC,IAAI;YACtB,aAAa,EAAE,KAAK,CAAC,OAAO;YAC5B,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,MAAM;YACN,UAAU;YACV,WAAW,EAAE,IAAI,CAAC,kBAAkB,CAAC,UAAU;YAC/C,SAAS,EAAE,OAAO;YAClB,gBAAgB,EAAE,UAAU;SAC7B,CAAC,CAAC;QAEH,OAAO,CAAC,IAAI,CAAC,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,4EAA4E;IAC5E,0EAA0E;IAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAClF,IAAI,SAAS,CAAC,QAAQ,KAAK,CAAC,IAAI,SAAS,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;QAC3D,MAAM,IAAI,uBAAuB,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;IAE3E,MAAM,UAAU,GAAG,YAAY,CAC7B,EAAE,MAAM,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,QAAQ,EAAE,EACxD,EAAE,MAAM,EAAE,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,CAAC,QAAQ,EAAE,CAC3D,CAAC;IAEF,6EAA6E;IAC7E,6EAA6E;IAC7E,gEAAgE;IAChE,IAAI,MAAoC,CAAC;IACzC,IAAI,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAC9B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,mBAAmB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,aAAa,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC;QAC7E,MAAM,GAAG,aAAa,CAAC,aAAa,EAAE,YAAY,CAAC,CAAC;IACtD,CAAC;IAED,OAAO;QACL,OAAO;QACP,UAAU;QACV,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7B,SAAS,EAAE,EAAE,YAAY,EAAE,iBAAiB,EAAE,UAAU,EAAE,eAAe,EAAE,iBAAiB,EAAE,IAAI,EAAE;KACrG,CAAC;AACJ,CAAC"}

package/dist/eval/paired.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Paired (matched-design) comparison for the held-out exam.
+ *
+ * The exam scores the SAME slate against the before- and after-checkpoints, so
+ * the two result sets are MATCHED, not independent. The marginal test in
+ * `wilson.ts` (`compareRates`) compares two INDEPENDENT Wilson intervals and
+ * calls a delta trustworthy only when they are disjoint. On a matched design
+ * that is statistically wasteful: it carries the full between-instance
+ * difficulty variance ("this bug is just hard") in BOTH intervals, which swamps
+ * a real, consistent within-instance improvement. Example: a learner that flips
+ * 3 hard instances fail→pass and regresses none moves 4/10→7/10 — two Wilson
+ * intervals ([17,69]% vs [40,89]%) overlap → "within-noise", even though every
+ * observed change was an improvement.
+ *
+ * McNemar's test is the textbook test for matched binary data: it looks ONLY at
+ * the discordant pairs (instances that flipped) and asks whether the flips are
+ * asymmetric beyond chance. This is NOT a weaker bar — it is the CORRECT test
+ * for the design, and it still gates the claim on significance (exact two-sided
+ * p < alpha) AND on the improvement direction (more fail→pass than pass→fail).
+ * It is reported ALONGSIDE the conservative marginal verdict, never replacing it
+ * — so the exam is strengthened, never weakened (DR-2026-06-02-b §2a).
+ *
+ * R=1 (one run per instance) ships here: McNemar exact on per-instance flips.
+ * R>1 (per-instance pass RATES → Wilcoxon signed-rank / paired bootstrap) layers
+ * on once `eval_results` supports appended runs (DR-2026-06-02-b §2b).
+ */
+/** Minimal per-instance result (subset of PerTaskResult / EvalResultRow). */
+export interface PairedInput {
+    instance_id: string;
+    /** null when unscorable. */
+    passed: boolean | null;
+    unscorable: boolean;
+}
+export type PairedVerdict = 'trustworthy' | 'within-noise';
+export interface PairedComparison {
+    /** Instances scorable (passed !== null) in BOTH arms — the matched pairs. */
+    pairs: number;
+    /** b: before fail → after pass (improvements). */
+    improved: number;
+    /** c: before pass → after fail (regressions). */
+    regressed: number;
+    concordantPass: number;
+    concordantFail: number;
+    /** Instances unscorable/absent in either arm — excluded from the paired test. */
+    excluded: number;
+    /** Exact two-sided McNemar p-value over the discordant pairs. */
+    pValue: number;
+    /** 'trustworthy' iff pValue < alpha AND improved > regressed; else 'within-noise'. */
+    verdict: PairedVerdict;
+}
+/**
+ * Exact two-sided McNemar p-value for discordant counts `b` (improvements) and
+ * `c` (regressions). Under H0 each discordant pair is an independent fair coin,
+ * so the count of one direction is Binomial(n=b+c, 0.5); the two-sided p is
+ * `2 * P(X <= min(b,c))`, capped at 1. No `b==c==0` division (returns 1: no
+ * evidence). Computed iteratively (term ratio) so it is exact and overflow-free
+ * for any slate size.
+ */
+export declare function mcnemarExact(b: number, c: number): number;
+/**
+ * Paired McNemar comparison of two matched per-instance result sets. An instance
+ * contributes a pair ONLY when it is scorable (passed !== null, not unscorable)
+ * in BOTH arms — an unscorable/missing side is excluded (honest: a disk-skip or
+ * harness failure is never coerced into a flip). `alpha` defaults to 0.05.
+ */
+export declare function comparePaired(before: readonly PairedInput[], after: readonly PairedInput[], opts?: {
+    alpha?: number;
+}): PairedComparison;

package/dist/eval/paired.js

@@ -0,0 +1,93 @@
+/**
+ * Paired (matched-design) comparison for the held-out exam.
+ *
+ * The exam scores the SAME slate against the before- and after-checkpoints, so
+ * the two result sets are MATCHED, not independent. The marginal test in
+ * `wilson.ts` (`compareRates`) compares two INDEPENDENT Wilson intervals and
+ * calls a delta trustworthy only when they are disjoint. On a matched design
+ * that is statistically wasteful: it carries the full between-instance
+ * difficulty variance ("this bug is just hard") in BOTH intervals, which swamps
+ * a real, consistent within-instance improvement. Example: a learner that flips
+ * 3 hard instances fail→pass and regresses none moves 4/10→7/10 — two Wilson
+ * intervals ([17,69]% vs [40,89]%) overlap → "within-noise", even though every
+ * observed change was an improvement.
+ *
+ * McNemar's test is the textbook test for matched binary data: it looks ONLY at
+ * the discordant pairs (instances that flipped) and asks whether the flips are
+ * asymmetric beyond chance. This is NOT a weaker bar — it is the CORRECT test
+ * for the design, and it still gates the claim on significance (exact two-sided
+ * p < alpha) AND on the improvement direction (more fail→pass than pass→fail).
+ * It is reported ALONGSIDE the conservative marginal verdict, never replacing it
+ * — so the exam is strengthened, never weakened (DR-2026-06-02-b §2a).
+ *
+ * R=1 (one run per instance) ships here: McNemar exact on per-instance flips.
+ * R>1 (per-instance pass RATES → Wilcoxon signed-rank / paired bootstrap) layers
+ * on once `eval_results` supports appended runs (DR-2026-06-02-b §2b).
+ */
+/**
+ * Exact two-sided McNemar p-value for discordant counts `b` (improvements) and
+ * `c` (regressions). Under H0 each discordant pair is an independent fair coin,
+ * so the count of one direction is Binomial(n=b+c, 0.5); the two-sided p is
+ * `2 * P(X <= min(b,c))`, capped at 1. No `b==c==0` division (returns 1: no
+ * evidence). Computed iteratively (term ratio) so it is exact and overflow-free
+ * for any slate size.
+ */
+export function mcnemarExact(b, c) {
+    const n = b + c;
+    if (n === 0)
+        return 1;
+    const k = Math.min(b, c);
+    // sum_{i=0}^{k} C(n,i) * 0.5^n, via prob_0 = 0.5^n, prob_i = prob_{i-1}*(n-i+1)/i.
+    let term = Math.pow(0.5, n); // i = 0
+    let cdf = term;
+    for (let i = 1; i <= k; i++) {
+        term = (term * (n - i + 1)) / i;
+        cdf += term;
+    }
+    return Math.min(1, 2 * cdf);
+}
+/**
+ * Paired McNemar comparison of two matched per-instance result sets. An instance
+ * contributes a pair ONLY when it is scorable (passed !== null, not unscorable)
+ * in BOTH arms — an unscorable/missing side is excluded (honest: a disk-skip or
+ * harness failure is never coerced into a flip). `alpha` defaults to 0.05.
+ */
+export function comparePaired(before, after, opts = {}) {
+    const alpha = opts.alpha ?? 0.05;
+    const scorable = (r) => !r.unscorable && r.passed !== null;
+    const beforeById = new Map(before.map((r) => [r.instance_id, r]));
+    let improved = 0;
+    let regressed = 0;
+    let concordantPass = 0;
+    let concordantFail = 0;
+    let pairs = 0;
+    let excluded = 0;
+    const seen = new Set();
+    for (const a of after) {
+        seen.add(a.instance_id);
+        const b = beforeById.get(a.instance_id);
+        if (!b || !scorable(b) || !scorable(a)) {
+            excluded++;
+            continue;
+        }
+        pairs++;
+        const wasPass = b.passed === true;
+        const nowPass = a.passed === true;
+        if (!wasPass && nowPass)
+            improved++;
+        else if (wasPass && !nowPass)
+            regressed++;
+        else if (wasPass && nowPass)
+            concordantPass++;
+        else
+            concordantFail++;
+    }
+    // before-only instances (absent from after) are excluded too.
+    for (const b of before)
+        if (!seen.has(b.instance_id))
+            excluded++;
+    const pValue = mcnemarExact(improved, regressed);
+    const verdict = pValue < alpha && improved > regressed ? 'trustworthy' : 'within-noise';
+    return { pairs, improved, regressed, concordantPass, concordantFail, excluded, pValue, verdict };
+}
+//# sourceMappingURL=paired.js.map

package/dist/eval/paired.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"paired.js","sourceRoot":"","sources":["../../src/eval/paired.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AA6BH;;;;;;;GAOG;AACH,MAAM,UAAU,YAAY,CAAC,CAAS,EAAE,CAAS;IAC/C,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACtB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzB,mFAAmF;IACnF,IAAI,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ;IACrC,IAAI,GAAG,GAAG,IAAI,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAChC,GAAG,IAAI,IAAI,CAAC;IACd,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC;AAC9B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAC3B,MAA8B,EAC9B,KAA6B,EAC7B,OAA2B,EAAE;IAE7B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;IACjC,MAAM,QAAQ,GAAG,CAAC,CAAc,EAAW,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC;IACjF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAElE,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACxB,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;QACxC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,QAAQ,EAAE,CAAC;YACX,SAAS;QACX,CAAC;QACD,KAAK,EAAE,CAAC;QACR,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC;QAClC,MAAM,OAAO,GAAG,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC;QAClC,IAAI,CAAC,OAAO,IAAI,OAAO;YAAE,QAAQ,EAAE,CAAC;aAC/B,IAAI,OAAO,IAAI,CAAC,OAAO;YAAE,SAAS,EAAE,CAAC;aACrC,IAAI,OAAO,IAAI,OAAO;YAAE,cAAc,EAAE,CAAC;;YACzC,cAAc,EAAE,CAAC;IACxB,CAAC;IACD,8DAA8D;IAC9D,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC;YAAE,QAAQ,EAAE,CAAC;IAEjE,MAAM,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IACjD,MAAM,OAAO,GACX,MAAM,GAAG,KAAK,IAAI,QAAQ,GAAG,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,cAAc,CAAC;IAE1E,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,cAAc,EAAE,cAAc,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AACnG,CAAC"}