cognitive-core 0.2.4 → 0.3.1

package/dist/eval/metrics.js

@@ -0,0 +1,361 @@
+import { seededRng } from './rng.js';
+/** Derive `status` from a TaskResult for backwards-compatibility. */
+function taskStatus(r) {
+    return r.status ?? (r.success ? 'success' : 'failure');
+}
+/**
+ * Statistics for the learning-curve eval.
+ *
+ * These are deliberately simple, transparent estimators — the goal is an
+ * interpretable answer to "did learning help, and is the difference real?",
+ * not a publication-grade test battery. Where the literature has a clear
+ * convention (Hake normalized gain, bootstrap CIs, pass@k vs pass^k) we follow
+ * it.
+ */
+/** Mean of a boolean array as a 0..1 rate (0 for empty). */
+export function rate(flags) {
+    if (flags.length === 0)
+        return 0;
+    return flags.filter(Boolean).length / flags.length;
+}
+/** Mean of a numeric array (0 for empty). */
+export function mean(xs) {
+    if (xs.length === 0)
+        return 0;
+    return xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+/**
+ * Rolling + cumulative success-rate curve over a single ordered run.
+ * The rolling window is the trailing `window` tasks; this is what makes the
+ * learning signal visible (a flat cumulative rate can hide a rising trend).
+ * env_error tasks are excluded from all rate computations.
+ */
+export function learningCurve(results, window) {
+    const curve = [];
+    let cumulativeSuccess = 0;
+    let cumulativeNonError = 0;
+    for (let i = 0; i < results.length; i++) {
+        const isEnvError = taskStatus(results[i]) === 'env_error';
+        if (!isEnvError) {
+            cumulativeNonError++;
+            if (results[i].success)
+                cumulativeSuccess++;
+        }
+        const start = Math.max(0, i - window + 1);
+        const windowSlice = results.slice(start, i + 1).filter((r) => taskStatus(r) !== 'env_error');
+        curve.push({
+            index: i,
+            rollingSuccessRate: rate(windowSlice.map((r) => r.success)),
+            cumulativeSuccessRate: cumulativeNonError > 0 ? cumulativeSuccess / cumulativeNonError : 0,
+        });
+    }
+    return curve;
+}
+/**
+ * Average several per-seed curves into one (point-wise mean of rolling and
+ * cumulative rates). Assumes equal-length curves.
+ */
+export function averageCurves(curves) {
+    if (curves.length === 0)
+        return [];
+    const length = curves[0].length;
+    const out = [];
+    for (let i = 0; i < length; i++) {
+        out.push({
+            index: i,
+            rollingSuccessRate: mean(curves.map((c) => c[i]?.rollingSuccessRate ?? 0)),
+            cumulativeSuccessRate: mean(curves.map((c) => c[i]?.cumulativeSuccessRate ?? 0)),
+        });
+    }
+    return out;
+}
+/**
+ * Hake normalized gain: g = (post − pre) / (1 − pre).
+ * Normalizes for headroom so a lift from 0.9→0.95 (g=0.5) is not dwarfed by
+ * 0.2→0.4 (g=0.25). `pre` is clamped below 1 to avoid division by zero.
+ */
+export function normalizedGain(pre, post) {
+    const denom = 1 - Math.min(pre, 0.999999);
+    return (post - pre) / denom;
+}
+/**
+ * 95% bootstrap confidence interval for the mean of a boolean sample.
+ * Resampling is seeded for reproducibility.
+ */
+export function bootstrapCI(flags, opts = {}) {
+    const n = flags.length;
+    if (n === 0)
+        return [0, 0];
+    const iterations = opts.iterations ?? 1000;
+    const rng = seededRng('bootstrap', opts.seed ?? 0, n);
+    const means = [];
+    for (let it = 0; it < iterations; it++) {
+        let sum = 0;
+        for (let i = 0; i < n; i++) {
+            const idx = Math.floor(rng() * n);
+            if (flags[idx])
+                sum++;
+        }
+        means.push(sum / n);
+    }
+    means.sort((a, b) => a - b);
+    const lo = means[Math.floor(0.025 * iterations)];
+    const hi = means[Math.min(iterations - 1, Math.floor(0.975 * iterations))];
+    return [lo, hi];
+}
+/** True if two 95% CIs do not overlap (a simple significance proxy). */
+export function ciDisjoint(a, b) {
+    return a[1] < b[0] || b[1] < a[0];
+}
+/**
+ * pass@k: probability that at least one of k attempts succeeds.
+ * pass^k: probability that all k attempts succeed (a reliability metric — a
+ * learning system should improve this, not just the average).
+ * `attemptsPerItem` is a list of success flags grouped by item.
+ */
+export function passAtK(attemptsPerItem, k) {
+    const items = attemptsPerItem.map((attempts) => {
+        const slice = attempts.slice(0, k);
+        return slice.some(Boolean);
+    });
+    return rate(items);
+}
+export function passCaretK(attemptsPerItem, k) {
+    const items = attemptsPerItem.map((attempts) => {
+        const slice = attempts.slice(0, k);
+        return slice.length === k && slice.every(Boolean);
+    });
+    return rate(items);
+}
+/**
+ * Success rate over a contiguous fraction [from, to) of an ordered run.
+ * env_error tasks are excluded from both numerator and denominator.
+ */
+export function phaseRate(results, from, to) {
+    const n = results.length;
+    const start = Math.floor(from * n);
+    const end = Math.floor(to * n);
+    const slice = results.slice(start, end).filter((r) => taskStatus(r) !== 'env_error');
+    return rate(slice.map((r) => r.success));
+}
+/**
+ * Per-family success rates: groups results by their `skillTag` (family) and
+ * computes success rate per family. Used by SkillFlow / SkillsBench to show
+ * domain-level performance breakdowns.
+ */
+export function familySuccessRates(results) {
+    const groups = new Map();
+    for (const r of results) {
+        const key = r.skillTag;
+        if (!groups.has(key))
+            groups.set(key, []);
+        groups.get(key).push(r.success);
+    }
+    const out = {};
+    for (const [family, flags] of groups) {
+        out[family] = rate(flags);
+    }
+    return out;
+}
+/**
+ * Transfer matrix: for every ordered pair (sourceFamily, targetFamily), the
+ * success rate on targetFamily tasks that came *after* at least one success in
+ * sourceFamily. Diagonal entries are self-transfer (later tasks in the same
+ * family). Used by SkillFlow to quantify cross-domain transfer.
+ */
+export function transferMatrix(results) {
+    const families = [...new Set(results.map((r) => r.skillTag))];
+    const familyIndex = new Map(families.map((f, i) => [f, i]));
+    const firstSuccessIdx = new Map();
+    for (const r of results) {
+        if (r.success && !firstSuccessIdx.has(r.skillTag)) {
+            firstSuccessIdx.set(r.skillTag, r.index);
+        }
+    }
+    const n = families.length;
+    const sums = Array.from({ length: n }, () => Array(n).fill(0));
+    const counts = Array.from({ length: n }, () => Array(n).fill(0));
+    for (const r of results) {
+        const tgtIdx = familyIndex.get(r.skillTag);
+        for (const [srcFamily, srcFirstIdx] of firstSuccessIdx) {
+            if (srcFirstIdx < r.index) {
+                const srcIdx = familyIndex.get(srcFamily);
+                counts[srcIdx][tgtIdx]++;
+                if (r.success)
+                    sums[srcIdx][tgtIdx]++;
+            }
+        }
+    }
+    const matrix = sums.map((row, i) => row.map((s, j) => (counts[i][j] > 0 ? s / counts[i][j] : 0)));
+    return { families, matrix };
+}
+/**
+ * Convergence iteration: the first task index at which the rolling success rate
+ * reaches or exceeds `threshold` and stays there for the rest of the stream.
+ * Returns -1 if the threshold is never stably reached.
+ */
+export function convergenceIteration(curve, threshold) {
+    for (let i = 0; i < curve.length; i++) {
+        if (curve[i].rollingSuccessRate >= threshold) {
+            let held = true;
+            for (let j = i + 1; j < curve.length; j++) {
+                if (curve[j].rollingSuccessRate < threshold) {
+                    held = false;
+                    break;
+                }
+            }
+            if (held)
+                return curve[i].index;
+        }
+    }
+    return -1;
+}
+/**
+ * Expert gain: how much the learning arm's success rate improves over the
+ * no-learning baseline specifically for tasks where prior knowledge was
+ * surfaced. Isolates the memory-injection effect from general model ability.
+ */
+export function expertGain(learningResults, baselineRate) {
+    const surfaced = learningResults.filter((r) => r.knowledgeSurfaced);
+    if (surfaced.length === 0)
+        return { surfacedRate: 0, gain: 0, count: 0 };
+    const surfacedRate = rate(surfaced.map((r) => r.success));
+    return {
+        surfacedRate,
+        gain: surfacedRate - baselineRate,
+        count: surfaced.length,
+    };
+}
+/**
+ * Binomial CDF: P(X <= k) where X ~ Binomial(n, p).
+ * Implemented via direct summation — no external dependencies.
+ */
+function binomialCdf(k, n, p) {
+    if (k < 0)
+        return 0;
+    if (k >= n)
+        return 1;
+    // Compute sum_{i=0}^{k} C(n,i) * p^i * (1-p)^(n-i) via log-space to avoid
+    // overflow for moderate n (< ~1000).
+    const q = 1 - p;
+    let logQ = n * Math.log(q);
+    // log C(n,i) iterated using log C(n,i) = log C(n,i-1) + log((n-i+1)/i)
+    let logC = 0;
+    let sum = 0;
+    for (let i = 0; i <= k; i++) {
+        if (i > 0) {
+            logC += Math.log(n - i + 1) - Math.log(i);
+            logQ += Math.log(p) - Math.log(q);
+        }
+        sum += Math.exp(logC + logQ);
+    }
+    return Math.min(1, sum);
+}
+/**
+ * Paired comparison between the learning and no-learning arms.
+ *
+ * Pairs tasks by index (same stream position). Pairs where either side has
+ * status env_error are dropped.
+ *
+ * Returns:
+ *   discordantPairs  — b + c total discordant pairs
+ *   mcnemarP         — two-sided exact McNemar p-value (binomial(b+c, 0.5))
+ *   meanDelta        — mean(learning_i − cold_i) over non-dropped pairs
+ *   bootstrapCI      — 95% percentile CI on meanDelta (10k iterations, seeded)
+ *   droppedPairs     — count of pairs dropped due to env_error on either side
+ */
+export function pairedComparison(learning, cold, opts = {}) {
+    const n = Math.min(learning.length, cold.length);
+    const lStatuses = opts.learningStatuses;
+    const cStatuses = opts.coldStatuses;
+    // Build paired deltas, dropping env_error pairs.
+    const deltas = [];
+    let droppedPairs = 0;
+    let b = 0; // learning-only success (learning=true, cold=false)
+    let c = 0; // cold-only success (learning=false, cold=true)
+    for (let i = 0; i < n; i++) {
+        const lErr = lStatuses?.[i] === 'env_error';
+        const cErr = cStatuses?.[i] === 'env_error';
+        if (lErr || cErr) {
+            droppedPairs++;
+            continue;
+        }
+        const lVal = learning[i] ? 1 : 0;
+        const cVal = cold[i] ? 1 : 0;
+        deltas.push(lVal - cVal);
+        if (lVal === 1 && cVal === 0)
+            b++;
+        if (lVal === 0 && cVal === 1)
+            c++;
+    }
+    const discordantPairs = b + c;
+    // Two-sided exact McNemar: p = 2 * min(P(X <= min(b,c)), 0.5)
+    // with X ~ Binomial(b+c, 0.5).
+    let mcnemarP;
+    if (discordantPairs === 0) {
+        mcnemarP = 1;
+    }
+    else {
+        const minBC = Math.min(b, c);
+        const tailP = binomialCdf(minBC, discordantPairs, 0.5);
+        mcnemarP = Math.min(1, 2 * Math.min(tailP, 0.5));
+    }
+    const meanDelta = deltas.length > 0 ? deltas.reduce((a, v) => a + v, 0) / deltas.length : 0;
+    // Paired bootstrap CI: resample task indices, 10k iterations.
+    const iterations = opts.iterations ?? 10_000;
+    const rng = seededRng('paired-bootstrap', opts.seed ?? 0, deltas.length);
+    const bootMeans = [];
+    const m = deltas.length;
+    if (m > 0) {
+        for (let it = 0; it < iterations; it++) {
+            let sum = 0;
+            for (let i = 0; i < m; i++) {
+                sum += deltas[Math.floor(rng() * m)];
+            }
+            bootMeans.push(sum / m);
+        }
+        bootMeans.sort((a, b) => a - b);
+    }
+    const ciLo = m > 0 ? bootMeans[Math.floor(0.025 * iterations)] : 0;
+    const ciHi = m > 0 ? bootMeans[Math.min(iterations - 1, Math.floor(0.975 * iterations))] : 0;
+    return {
+        discordantPairs,
+        mcnemarP,
+        meanDelta,
+        bootstrapCI: [ciLo, ciHi],
+        droppedPairs,
+    };
+}
+/**
+ * Assemble per-seed task results into a ConditionResult — the shared reduction
+ * used by both the mock (SequentialEvalRunner) and live (LiveEvalRunner) arms so
+ * their aggregates are computed identically.
+ * env_error tasks are excluded from success-rate aggregates.
+ */
+export function aggregateCondition(label, perSeed, rollingWindow, seeds, bootstrapSeed = 42) {
+    const curves = perSeed.map((r) => learningCurve(r, rollingWindow));
+    const all = perSeed.flat();
+    const nonError = all.filter((r) => taskStatus(r) !== 'env_error');
+    const allFlags = nonError.map((r) => r.success);
+    const allSteps = all.map((r) => r.stepCount);
+    const envErrorRate = all.length > 0 ? (all.length - nonError.length) / all.length : 0;
+    const allCosts = all.map((r) => r.costUsd).filter((c) => c != null);
+    const totalCostUsd = allCosts.length > 0 ? allCosts.reduce((a, b) => a + b, 0) : undefined;
+    const allPartial = nonError.map((r) => r.partialScore).filter((s) => s != null);
+    const avgPartialScore = allPartial.length > 0 ? mean(allPartial) : undefined;
+    return {
+        label,
+        seeds,
+        perSeed,
+        successRate: rate(allFlags),
+        earlySuccessRate: mean(perSeed.map((r) => phaseRate(r, 0, 1 / 3))),
+        lateSuccessRate: mean(perSeed.map((r) => phaseRate(r, 2 / 3, 1))),
+        avgSteps: mean(allSteps),
+        curve: averageCurves(curves),
+        successRateCI: bootstrapCI(allFlags, { seed: bootstrapSeed }),
+        envErrorRate,
+        totalCostUsd,
+        avgPartialScore,
+    };
+}
+//# sourceMappingURL=metrics.js.map

package/dist/eval/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/eval/metrics.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAErC,qEAAqE;AACrE,SAAS,UAAU,CAAC,CAAa;IAC/B,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;AACzD,CAAC;AAED;;;;;;;;GAQG;AAEH,4DAA4D;AAC5D,MAAM,UAAU,IAAI,CAAC,KAAgB;IACnC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACjC,OAAO,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;AACrD,CAAC;AAED,6CAA6C;AAC7C,MAAM,UAAU,IAAI,CAAC,EAAY;IAC/B,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC9B,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;AACnD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAAC,OAAqB,EAAE,MAAc;IACjE,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,IAAI,iBAAiB,GAAG,CAAC,CAAC;IAC1B,IAAI,kBAAkB,GAAG,CAAC,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,UAAU,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC;QAC1D,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,kBAAkB,EAAE,CAAC;YACrB,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO;gBAAE,iBAAiB,EAAE,CAAC;QAC9C,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,MAAM,GAAG,CAAC,CAAC,CAAC;QAC1C,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,CAAC;QAC7F,KAAK,CAAC,IAAI,CAAC;YACT,KAAK,EAAE,CAAC;YACR,kBAAkB,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3D,qBAAqB,EAAE,kBAAkB,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,GAAG,kBAAkB,CAAC,CAAC,CAAC,CAAC;SAC3F,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,MAAsB;IAClD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAChC,MAAM,GAAG,GAAiB,EAAE,CAAC;IAC7B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,GAAG,CAAC,IAAI,CAAC;YACP,KAAK,EAAE,CAAC;YACR,kBAAkB,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,kBAAkB,IAAI,CAAC,CAAC,CAAC;YAC1E,qBAAqB,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,qBAAqB,IAAI,CAAC,CAAC,CAAC;SACjF,CAAC,CAAC;IACL,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY;IACtD,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAC1C,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,KAAK,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CACzB,KAAgB,EAChB,OAA+C,EAAE;IAEjD,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3B,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC;IAC3C,MAAM,GAAG,GAAG,SAAS,CAAC,WAAW,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IACtD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,UAAU,EAAE,EAAE,EAAE,EAAE,CAAC;QACvC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,GAAG,CAAC;gBAAE,GAAG,EAAE,CAAC;QACxB,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;IACtB,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC;IACjD,MAAM,EAAE,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;IAC3E,OAAO,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;AAClB,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,UAAU,CAAC,CAAmB,EAAE,CAAmB;IACjE,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,OAAO,CAAC,eAA4B,EAAE,CAAS;IAC7D,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC7C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC;AACrB,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,eAA4B,EAAE,CAAS;IAChE,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC7C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IACH,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,OAAqB,EAAE,IAAY,EAAE,EAAU;IACvE,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACnC,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,CAAC;IACrF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;AAC3C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAAqB;IACtD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAqB,CAAC;IAC5C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,CAAC,CAAC,QAAQ,CAAC;QACvB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;QAC1C,MAAM,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC;IACD,MAAM,GAAG,GAA2B,EAAE,CAAC;IACvC,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,EAAE,CAAC;QACrC,GAAG,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAAC,OAAqB;IAClD,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;IAC9D,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5D,MAAM,eAAe,GAAG,IAAI,GAAG,EAAkB,CAAC;IAClD,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,IAAI,CAAC,CAAC,OAAO,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;YAClD,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC;QAC3C,CAAC;IACH,CAAC;IAED,MAAM,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;IAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAa,CAAC,CAAC;IAC3E,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAa,CAAC,CAAC;IAE7E,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,SAAS,EAAE,WAAW,CAAC,IAAI,eAAe,EAAE,CAAC;YACvD,IAAI,WAAW,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;gBAC1B,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;gBAC3C,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,CAAC,OAAO;oBAAE,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YACxC,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAClG,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AAC9B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAmB,EAAE,SAAiB;IACzE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,IAAI,SAAS,EAAE,CAAC;YAC7C,IAAI,IAAI,GAAG,IAAI,CAAC;YAChB,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC1C,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,kBAAkB,GAAG,SAAS,EAAE,CAAC;oBAC5C,IAAI,GAAG,KAAK,CAAC;oBACb,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,IAAI;gBAAE,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAClC,CAAC;IACH,CAAC;IACD,OAAO,CAAC,CAAC,CAAC;AACZ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,UAAU,CACxB,eAA6B,EAC7B,YAAoB;IAEpB,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;IACpE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,YAAY,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;IACzE,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;IAC1D,OAAO;QACL,YAAY;QACZ,IAAI,EAAE,YAAY,GAAG,YAAY;QACjC,KAAK,EAAE,QAAQ,CAAC,MAAM;KACvB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,CAAS,EAAE,CAAS,EAAE,CAAS;IAClD,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IACpB,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IACrB,0EAA0E;IAC1E,qCAAqC;IACrC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAChB,IAAI,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC3B,uEAAuE;IACvE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YACV,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC1C,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC;QACD,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC1B,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAC9B,QAAmB,EACnB,IAAe,EACf,OAKI,EAAE;IAQN,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC;IACxC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC;IAEpC,iDAAiD;IACjD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,oDAAoD;IAC/D,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,gDAAgD;IAE3D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC;QAC5C,MAAM,IAAI,GAAG,SAAS,EAAE,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC;QAC5C,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACjB,YAAY,EAAE,CAAC;YACf,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC;QACzB,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC;YAAE,CAAC,EAAE,CAAC;QAClC,IAAI,IAAI,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC;YAAE,CAAC,EAAE,CAAC;IACpC,CAAC;IAED,MAAM,eAAe,GAAG,CAAC,GAAG,CAAC,CAAC;IAE9B,8DAA8D;IAC9D,+BAA+B;IAC/B,IAAI,QAAgB,CAAC;IACrB,IAAI,eAAe,KAAK,CAAC,EAAE,CAAC;QAC1B,QAAQ,GAAG,CAAC,CAAC;IACf,CAAC;SAAM,CAAC;QACN,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,eAAe,EAAE,GAAG,CAAC,CAAC;QACvD,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5F,8DAA8D;IAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,MAAM,CAAC;IAC7C,MAAM,GAAG,GAAG,SAAS,CAAC,kBAAkB,EAAE,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IACzE,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACxB,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,UAAU,EAAE,EAAE,EAAE,EAAE,CAAC;YACvC,IAAI,GAAG,GAAG,CAAC,CAAC;YACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3B,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;YACD,SAAS,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QAC1B,CAAC;QACD,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAE7F,OAAO;QACL,eAAe;QACf,QAAQ;QACR,SAAS;QACT,WAAW,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC;QACzB,YAAY;KACb,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,kBAAkB,CAChC,KAAa,EACb,OAAuB,EACvB,aAAqB,EACrB,KAAe,EACf,aAAa,GAAG,EAAE;IAElB,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,aAAa,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC,CAAC;IACnE,MAAM,GAAG,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAC3B,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,KAAK,WAAW,CAAC,CAAC;IAClE,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC7C,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IAEtF,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;IACjF,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE3F,MAAM,UAAU,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;IAC7F,MAAM,eAAe,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE7E,OAAO;QACL,KAAK;QACL,KAAK;QACL,OAAO;QACP,WAAW,EAAE,IAAI,CAAC,QAAQ,CAAC;QAC3B,gBAAgB,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAClE,eAAe,EAAE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;QACjE,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC;QACxB,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC;QAC5B,aAAa,EAAE,WAAW,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC;QAC7D,YAAY;QACZ,YAAY;QACZ,eAAe;KAChB,CAAC;AACJ,CAAC"}

package/dist/eval/oracle-smoke.d.ts

@@ -0,0 +1,75 @@
+/**
+ * Oracle smoke harness — gate G0.
+ *
+ * Pure, testable logic for the oracle smoke test. No I/O except the manifest
+ * read (passed in as parsed JSON). Wiring lives in src/bin/commands/eval.ts.
+ *
+ * Purpose: prove the seed → retrieve → inject → succeed plumbing works end-to-end
+ * by seeding the learning arm with hand-written "oracle" playbooks that contain
+ * exact winning action sequences for specific ALFWorld games. The measured
+ * learning-vs-cold delta must be large; if it isn't, the plumbing is broken.
+ */
+import { pairedComparison } from './metrics.js';
+/**
+ * Pre-registered success criteria embedded in the manifest.
+ * Written before the live run so the test can't be adjusted post-hoc.
+ */
+export interface OracleCriteria {
+    /** Minimum number of learning-arm successes out of totalGames. */
+    minLearningSuccesses: number;
+    /** Maximum number of cold-arm successes out of totalGames (should be near 0). */
+    maxColdSuccesses: number;
+    /** Total games in the run (must equal manifest.games.length). */
+    totalGames: number;
+}
+/** Parsed oracle manifest. */
+export interface OracleManifest {
+    /** Game indices to run (passed to --games flag). */
+    games: number[];
+    /** Pre-registered criteria. */
+    preRegistered: OracleCriteria;
+}
+/**
+ * Outcome of the oracle smoke criteria check.
+ */
+export interface OracleSmokeResult {
+    pass: boolean;
+    learningSuccesses: number;
+    coldSuccesses: number;
+    criteria: OracleCriteria;
+    /** Paired comparison stats from pairedComparison(). */
+    paired: ReturnType<typeof pairedComparison>;
+    /** Human-readable summary lines. */
+    lines: string[];
+}
+/**
+ * Parse and validate an oracle manifest from its raw JSON string.
+ * Throws a descriptive error if the manifest is malformed.
+ */
+export declare function parseOracleManifest(raw: unknown): OracleManifest;
+/**
+ * Load and parse the oracle manifest from the fixture directory.
+ */
+export declare function loadOracleManifest(fixtureDir: string): OracleManifest;
+/**
+ * Check whether any playbook JSON file in the fixture directory still contains
+ * the `__FILL_FROM_COLD_RUN__` placeholder token. Returns the list of files
+ * that still contain placeholders.
+ *
+ * The smoke harness refuses to run if any placeholder remains — they indicate
+ * the oracle playbooks have not been filled in from a real cold run yet.
+ */
+export declare function findPlaceholderFiles(fixtureDir: string): string[];
+/**
+ * Evaluate the pre-registered oracle criteria against arm results.
+ *
+ * Pure function: takes boolean success arrays and returns pass/fail + stats.
+ * No I/O — fully unit-testable.
+ *
+ * @param learningResults - boolean success per game for the learning arm
+ * @param coldResults - boolean success per game for the cold arm
+ * @param criteria - pre-registered thresholds from the manifest
+ * @param seed - RNG seed for bootstrap CI reproducibility
+ */
+export declare function evaluateOracleCriteria(learningResults: boolean[], coldResults: boolean[], criteria: OracleCriteria, seed?: number): OracleSmokeResult;
+//# sourceMappingURL=oracle-smoke.d.ts.map

package/dist/eval/oracle-smoke.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"oracle-smoke.d.ts","sourceRoot":"","sources":["../../src/eval/oracle-smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAEhD;;;GAGG;AACH,MAAM,WAAW,cAAc;IAC7B,kEAAkE;IAClE,oBAAoB,EAAE,MAAM,CAAC;IAC7B,iFAAiF;IACjF,gBAAgB,EAAE,MAAM,CAAC;IACzB,iEAAiE;IACjE,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,8BAA8B;AAC9B,MAAM,WAAW,cAAc;IAC7B,oDAAoD;IACpD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,+BAA+B;IAC/B,aAAa,EAAE,cAAc,CAAC;CAC/B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,OAAO,CAAC;IACd,iBAAiB,EAAE,MAAM,CAAC;IAC1B,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,cAAc,CAAC;IACzB,uDAAuD;IACvD,MAAM,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,CAAC;IAC5C,oCAAoC;IACpC,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,OAAO,GAAG,cAAc,CAgDhE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,UAAU,EAAE,MAAM,GAAG,cAAc,CASrE;AAED;;;;;;;GAOG;AACH,wBAAgB,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,EAAE,CAsBjE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,sBAAsB,CACpC,eAAe,EAAE,OAAO,EAAE,EAC1B,WAAW,EAAE,OAAO,EAAE,EACtB,QAAQ,EAAE,cAAc,EACxB,IAAI,SAAI,GACP,iBAAiB,CA6BnB"}

package/dist/eval/oracle-smoke.js

@@ -0,0 +1,143 @@
+/**
+ * Oracle smoke harness — gate G0.
+ *
+ * Pure, testable logic for the oracle smoke test. No I/O except the manifest
+ * read (passed in as parsed JSON). Wiring lives in src/bin/commands/eval.ts.
+ *
+ * Purpose: prove the seed → retrieve → inject → succeed plumbing works end-to-end
+ * by seeding the learning arm with hand-written "oracle" playbooks that contain
+ * exact winning action sequences for specific ALFWorld games. The measured
+ * learning-vs-cold delta must be large; if it isn't, the plumbing is broken.
+ */
+import { readFileSync, readdirSync } from 'node:fs';
+import { join } from 'node:path';
+import { pairedComparison } from './metrics.js';
+/**
+ * Parse and validate an oracle manifest from its raw JSON string.
+ * Throws a descriptive error if the manifest is malformed.
+ */
+export function parseOracleManifest(raw) {
+    if (typeof raw !== 'object' || raw === null) {
+        throw new Error('oracle manifest must be a JSON object');
+    }
+    const obj = raw;
+    if (!Array.isArray(obj['games']) || obj['games'].length === 0) {
+        throw new Error('oracle manifest missing non-empty "games" array');
+    }
+    for (const g of obj['games']) {
+        if (typeof g !== 'number' || !Number.isInteger(g) || g < 0) {
+            throw new Error(`oracle manifest "games" must contain non-negative integers; got ${JSON.stringify(g)}`);
+        }
+    }
+    const games = obj['games'];
+    const pr = obj['preRegistered'];
+    if (typeof pr !== 'object' || pr === null) {
+        throw new Error('oracle manifest missing "preRegistered" object');
+    }
+    const crit = pr;
+    if (typeof crit['minLearningSuccesses'] !== 'number') {
+        throw new Error('oracle manifest preRegistered.minLearningSuccesses must be a number');
+    }
+    if (typeof crit['maxColdSuccesses'] !== 'number') {
+        throw new Error('oracle manifest preRegistered.maxColdSuccesses must be a number');
+    }
+    if (typeof crit['totalGames'] !== 'number') {
+        throw new Error('oracle manifest preRegistered.totalGames must be a number');
+    }
+    if (crit['totalGames'] !== games.length) {
+        throw new Error(`oracle manifest preRegistered.totalGames (${crit['totalGames']}) ` +
+            `does not match games.length (${games.length})`);
+    }
+    return {
+        games,
+        preRegistered: {
+            minLearningSuccesses: crit['minLearningSuccesses'],
+            maxColdSuccesses: crit['maxColdSuccesses'],
+            totalGames: crit['totalGames'],
+        },
+    };
+}
+/**
+ * Load and parse the oracle manifest from the fixture directory.
+ */
+export function loadOracleManifest(fixtureDir) {
+    const manifestPath = join(fixtureDir, 'manifest.json');
+    let raw;
+    try {
+        raw = JSON.parse(readFileSync(manifestPath, 'utf-8'));
+    }
+    catch (err) {
+        throw new Error(`failed to read oracle manifest at ${manifestPath}: ${err.message}`);
+    }
+    return parseOracleManifest(raw);
+}
+/**
+ * Check whether any playbook JSON file in the fixture directory still contains
+ * the `__FILL_FROM_COLD_RUN__` placeholder token. Returns the list of files
+ * that still contain placeholders.
+ *
+ * The smoke harness refuses to run if any placeholder remains — they indicate
+ * the oracle playbooks have not been filled in from a real cold run yet.
+ */
+export function findPlaceholderFiles(fixtureDir) {
+    let files;
+    try {
+        files = readdirSync(fixtureDir).filter((f) => f.endsWith('.json') && f !== 'manifest.json');
+    }
+    catch (err) {
+        throw new Error(`failed to read fixture directory ${fixtureDir}: ${err.message}`);
+    }
+    const withPlaceholders = [];
+    for (const file of files) {
+        const fullPath = join(fixtureDir, file);
+        try {
+            const content = readFileSync(fullPath, 'utf-8');
+            if (content.includes('__FILL_FROM_COLD_RUN__')) {
+                withPlaceholders.push(file);
+            }
+        }
+        catch {
+            // If a file can't be read, treat it as having placeholders.
+            withPlaceholders.push(file);
+        }
+    }
+    return withPlaceholders;
+}
+/**
+ * Evaluate the pre-registered oracle criteria against arm results.
+ *
+ * Pure function: takes boolean success arrays and returns pass/fail + stats.
+ * No I/O — fully unit-testable.
+ *
+ * @param learningResults - boolean success per game for the learning arm
+ * @param coldResults - boolean success per game for the cold arm
+ * @param criteria - pre-registered thresholds from the manifest
+ * @param seed - RNG seed for bootstrap CI reproducibility
+ */
+export function evaluateOracleCriteria(learningResults, coldResults, criteria, seed = 0) {
+    const learningSuccesses = learningResults.filter(Boolean).length;
+    const coldSuccesses = coldResults.filter(Boolean).length;
+    const paired = pairedComparison(learningResults, coldResults, { seed, iterations: 10_000 });
+    const passLearning = learningSuccesses >= criteria.minLearningSuccesses;
+    const passCold = coldSuccesses <= criteria.maxColdSuccesses;
+    const pass = passLearning && passCold;
+    const lines = [
+        '=== Oracle Smoke: G0 Gate ===',
+        '',
+        `Learning arm:  ${learningSuccesses}/${criteria.totalGames} successes` +
+            ` (need ≥${criteria.minLearningSuccesses}) — ${passLearning ? 'PASS' : 'FAIL'}`,
+        `Cold arm:      ${coldSuccesses}/${criteria.totalGames} successes` +
+            ` (need ≤${criteria.maxColdSuccesses}) — ${passCold ? 'PASS' : 'FAIL'}`,
+        '',
+        `Paired delta:  ${(paired.meanDelta * 100).toFixed(1)}pp` +
+            ` (95% CI [${(paired.bootstrapCI[0] * 100).toFixed(1)}, ${(paired.bootstrapCI[1] * 100).toFixed(1)}]pp)`,
+        `McNemar p:     ${paired.mcnemarP.toFixed(4)}` +
+            (paired.mcnemarP < 0.05 ? ' (significant)' : ' (not significant)'),
+        `Discordant:    ${paired.discordantPairs} pairs (b+c)`,
+        paired.droppedPairs > 0 ? `Dropped pairs: ${paired.droppedPairs} (env_error)` : '',
+        '',
+        pass ? 'RESULT: PASS — G0 gate holds' : 'RESULT: FAIL — G0 gate did not hold',
+    ].filter((l) => l !== undefined);
+    return { pass, learningSuccesses, coldSuccesses, criteria, paired, lines };
+}
+//# sourceMappingURL=oracle-smoke.js.map

package/dist/eval/oracle-smoke.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"oracle-smoke.js","sourceRoot":"","sources":["../../src/eval/oracle-smoke.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAqChD;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAY;IAC9C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,GAAG,GAAG,GAA8B,CAAC;IAE3C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,OAAO,CAAc,EAAE,CAAC;QAC1C,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CACb,mEAAmE,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CACvF,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAa,CAAC;IAEvC,MAAM,EAAE,GAAG,GAAG,CAAC,eAAe,CAAC,CAAC;IAChC,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;IACpE,CAAC;IACD,MAAM,IAAI,GAAG,EAA6B,CAAC;IAE3C,IAAI,OAAO,IAAI,CAAC,sBAAsB,CAAC,KAAK,QAAQ,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;IACD,IAAI,OAAO,IAAI,CAAC,kBAAkB,CAAC,KAAK,QAAQ,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,iEAAiE,CAAC,CAAC;IACrF,CAAC;IACD,IAAI,OAAO,IAAI,CAAC,YAAY,CAAC,KAAK,QAAQ,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,2DAA2D,CAAC,CAAC;IAC/E,CAAC;IACD,IAAK,IAAI,CAAC,YAAY,CAAY,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CACb,6CAA6C,IAAI,CAAC,YAAY,CAAC,IAAI;YACjE,gCAAgC,KAAK,CAAC,MAAM,GAAG,CAClD,CAAC;IACJ,CAAC;IAED,OAAO;QACL,KAAK;QACL,aAAa,EAAE;YACb,oBAAoB,EAAE,IAAI,CAAC,sBAAsB,CAAW;YAC5D,gBAAgB,EAAE,IAAI,CAAC,kBAAkB,CAAW;YACpD,UAAU,EAAE,IAAI,CAAC,YAAY,CAAW;SACzC;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,UAAkB;IACnD,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;IACvD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC;IACxD,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,qCAAqC,YAAY,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,mBAAmB,CAAC,GAAG,CAAC,CAAC;AAClC,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,oBAAoB,CAAC,UAAkB;IACrD,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,eAAe,CAAC,CAAC;IAC9F,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,oCAAoC,UAAU,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;IAC/F,CAAC;IAED,MAAM,gBAAgB,GAAa,EAAE,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QACxC,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAChD,IAAI,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAC,EAAE,CAAC;gBAC/C,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4DAA4D;YAC5D,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;IACH,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,sBAAsB,CACpC,eAA0B,EAC1B,WAAsB,EACtB,QAAwB,EACxB,IAAI,GAAG,CAAC;IAER,MAAM,iBAAiB,GAAG,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IACjE,MAAM,aAAa,GAAG,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAEzD,MAAM,MAAM,GAAG,gBAAgB,CAAC,eAAe,EAAE,WAAW,EAAE,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC;IAE5F,MAAM,YAAY,GAAG,iBAAiB,IAAI,QAAQ,CAAC,oBAAoB,CAAC;IACxE,MAAM,QAAQ,GAAG,aAAa,IAAI,QAAQ,CAAC,gBAAgB,CAAC;IAC5D,MAAM,IAAI,GAAG,YAAY,IAAI,QAAQ,CAAC;IAEtC,MAAM,KAAK,GAAa;QACtB,+BAA+B;QAC/B,EAAE;QACF,kBAAkB,iBAAiB,IAAI,QAAQ,CAAC,UAAU,YAAY;YACpE,WAAW,QAAQ,CAAC,oBAAoB,OAAO,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE;QACjF,kBAAkB,aAAa,IAAI,QAAQ,CAAC,UAAU,YAAY;YAChE,WAAW,QAAQ,CAAC,gBAAgB,OAAO,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE;QACzE,EAAE;QACF,kBAAkB,CAAC,MAAM,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;YACvD,aAAa,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;QAC1G,kBAAkB,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;YAC5C,CAAC,MAAM,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,oBAAoB,CAAC;QACpE,kBAAkB,MAAM,CAAC,eAAe,cAAc;QACtD,MAAM,CAAC,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,kBAAkB,MAAM,CAAC,YAAY,cAAc,CAAC,CAAC,CAAC,EAAE;QAClF,EAAE;QACF,IAAI,CAAC,CAAC,CAAC,8BAA8B,CAAC,CAAC,CAAC,qCAAqC;KAC9E,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAa,CAAC;IAE7C,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC7E,CAAC"}

package/dist/eval/playbook-calibration-report.d.ts

@@ -0,0 +1,40 @@
+import type { TaskDiagnostics } from './diagnostics.js';
+export type CalibrationOutcomeLabel = 'discordant-win' | 'discordant-loss' | 'both-pass' | 'both-fail' | 'single-arm-pass' | 'single-arm-fail';
+export interface PlaybookCalibrationRow {
+    taskIndex: number;
+    taskId: string;
+    family?: string;
+    difficulty?: number;
+    playbookId: string;
+    playbookName: string;
+    score: number;
+    retrievalMethod?: string;
+    extractionMethod?: string;
+    skillLevel?: string;
+    outcomeSuccess: boolean;
+    baselineSuccess?: boolean;
+    label: CalibrationOutcomeLabel;
+    arm: string;
+    playbookRetriever?: string;
+    scoreFloor?: number;
+    rawCount?: number;
+    excludedByScoreFloor?: number;
+    domainFallbackUsed?: boolean;
+}
+export interface PlaybookCalibrationReportOptions {
+    learningArm?: string;
+    baselineArm?: string;
+    thresholds?: number[];
+}
+export interface CalibrationTraceSet {
+    learning: TaskDiagnostics[];
+    baseline: TaskDiagnostics[];
+    learningArm: string;
+    baselineArm: string;
+    pairedLayout: boolean;
+}
+export declare function loadPlaybookCalibrationRows(traceRoot: string, opts?: PlaybookCalibrationReportOptions): PlaybookCalibrationRow[];
+export declare function renderPlaybookCalibrationReport(traceRoot: string, opts?: PlaybookCalibrationReportOptions): string;
+export declare function labelOutcome(success: boolean, baselineSuccess: boolean | undefined): CalibrationOutcomeLabel;
+export declare function loadTraceSet(traceRoot: string, opts: PlaybookCalibrationReportOptions): CalibrationTraceSet;
+//# sourceMappingURL=playbook-calibration-report.d.ts.map

package/dist/eval/playbook-calibration-report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"playbook-calibration-report.d.ts","sourceRoot":"","sources":["../../src/eval/playbook-calibration-report.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAwB,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAE9E,MAAM,MAAM,uBAAuB,GAC/B,gBAAgB,GAChB,iBAAiB,GACjB,WAAW,GACX,WAAW,GACX,iBAAiB,GACjB,iBAAiB,CAAC;AAEtB,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,OAAO,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,KAAK,EAAE,uBAAuB,CAAC;IAC/B,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,gCAAgC;IAC/C,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,QAAQ,EAAE,eAAe,EAAE,CAAC;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,OAAO,CAAC;CACvB;AAwBD,wBAAgB,2BAA2B,CACzC,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE,gCAAqC,GAC1C,sBAAsB,EAAE,CAa1B;AAED,wBAAgB,+BAA+B,CAC7C,SAAS,EAAE,MAAM,EACjB,IAAI,GAAE,gCAAqC,GAC1C,MAAM,CAwFR;AA+BD,wBAAgB,YAAY,CAC1B,OAAO,EAAE,OAAO,EAChB,eAAe,EAAE,OAAO,GAAG,SAAS,GACnC,uBAAuB,CAMzB;AAED,wBAAgB,YAAY,CAC1B,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,gCAAgC,GACrC,mBAAmB,CAgCrB"}