@miller-tech/uap 1.50.1 → 1.52.0

package/dist/benchmarks/paired/report.js

@@ -0,0 +1,204 @@
+/**
+ * Analysis + reporting.
+ *
+ * Turns raw paired RunRecords into the credible result shape the research
+ * prescribes: NOT a single headline number, but a vector of paired deltas with
+ * confidence intervals (accuracy AND efficiency), a McNemar 2x2 for gate value,
+ * and a cost-accuracy Pareto view. Treatment arms are always compared against
+ * the baseline cell-for-cell (same task + seed).
+ */
+import { mcnemar, mean, pairedDelta, } from './stats.js';
+import { CONTINUOUS_METRICS } from './types.js';
+function cellKey(r) {
+    return `${r.taskId}#${r.seed}`;
+}
+function byCondition(records) {
+    const m = new Map();
+    for (const r of records) {
+        const arr = m.get(r.condition) ?? [];
+        arr.push(r);
+        m.set(r.condition, arr);
+    }
+    return m;
+}
+function summarize(label, recs) {
+    const correct = recs.map((r) => (r.metrics.correct ? 1 : 0));
+    const numeric = (sel) => {
+        const vals = recs.map((r) => sel(r.metrics)).filter((v) => v != null);
+        return vals.length ? mean(vals) : null;
+    };
+    return {
+        label,
+        n: recs.length,
+        successRate: recs.length ? mean(correct) : NaN,
+        meanTokens: numeric((m) => m.tokens),
+        meanTurns: numeric((m) => m.turns),
+        meanCostUsd: numeric((m) => m.costUsd),
+        meanLatencyMs: numeric((m) => m.latencyMs) ?? NaN,
+        errorRate: recs.length ? mean(recs.map((r) => (r.metrics.error ? 1 : 0))) : NaN,
+    };
+}
+/** Build the paired delta arrays between a treatment and baseline cell map. */
+function pairCells(baseline, treatment) {
+    const keys = [];
+    const base = [];
+    const treat = [];
+    for (const [k, b] of baseline) {
+        const t = treatment.get(k);
+        if (t) {
+            keys.push(k);
+            base.push(b);
+            treat.push(t);
+        }
+    }
+    return { keys, base, treat };
+}
+function toCellMap(recs) {
+    const m = new Map();
+    for (const r of recs)
+        m.set(cellKey(r), r);
+    return m;
+}
+export function analyze(output, opts = {}) {
+    const baselineLabel = opts.baselineLabel ?? 'baseline';
+    const grouped = byCondition(output.records);
+    const conditions = [...grouped.keys()];
+    if (!grouped.has(baselineLabel)) {
+        throw new Error(`analyze: baseline condition '${baselineLabel}' not present (have: ${conditions.join(', ')})`);
+    }
+    const perCondition = conditions.map((label) => summarize(label, grouped.get(label)));
+    const baseMap = toCellMap(grouped.get(baselineLabel));
+    const taskCount = new Set(output.records.map((r) => r.taskId)).size;
+    const comparisons = [];
+    for (const label of conditions) {
+        if (label === baselineLabel)
+            continue;
+        const treatMap = toCellMap(grouped.get(label));
+        const { base, treat } = pairCells(baseMap, treatMap);
+        // Correctness: paired binary deltas (+1 fixed, -1 regressed, 0 same).
+        const correctnessDeltas = base.map((b, i) => num(treat[i].metrics.correct) - num(b.metrics.correct));
+        const delta = pairedDelta(correctnessDeltas, opts);
+        const mc = mcnemar(treat.map((t) => t.metrics.correct), base.map((b) => b.metrics.correct));
+        // Continuous metrics: paired deltas where BOTH arms reported the value.
+        const metrics = {};
+        for (const metric of CONTINUOUS_METRICS) {
+            const deltas = [];
+            for (let i = 0; i < base.length; i++) {
+                const bv = base[i].metrics[metric];
+                const tv = treat[i].metrics[metric];
+                if (bv != null && tv != null)
+                    deltas.push(tv - bv);
+            }
+            if (deltas.length > 0)
+                metrics[metric] = pairedDelta(deltas, opts);
+        }
+        comparisons.push({
+            label,
+            baseline: baselineLabel,
+            correctness: {
+                baselineRate: mean(base.map((b) => num(b.metrics.correct))),
+                treatmentRate: mean(treat.map((t) => num(t.metrics.correct))),
+                delta,
+                mcnemar: mc,
+            },
+            metrics,
+        });
+    }
+    return {
+        meta: {
+            model: output.model,
+            adapter: output.adapter,
+            epochs: output.epochs,
+            taskCount,
+            conditions,
+            baseline: baselineLabel,
+            startedAt: output.startedAt,
+            finishedAt: output.finishedAt,
+        },
+        perCondition,
+        comparisons,
+        pareto: perCondition.map((c) => ({
+            label: c.label,
+            successRate: c.successRate,
+            meanTokens: c.meanTokens,
+        })),
+    };
+}
+function num(b) {
+    return b ? 1 : 0;
+}
+// ---------------------------------------------------------------------------
+// Markdown renderer
+// ---------------------------------------------------------------------------
+function pct(x) {
+    return Number.isFinite(x) ? `${(x * 100).toFixed(1)}%` : 'n/a';
+}
+function fixed(x, d = 0) {
+    return x == null || !Number.isFinite(x) ? 'n/a' : x.toFixed(d);
+}
+function ciStr(ci, d = 3) {
+    return `[${ci.lower.toFixed(d)}, ${ci.upper.toFixed(d)}]`;
+}
+function sig(p) {
+    return p.significant ? '✅' : '–';
+}
+export function renderMarkdown(r) {
+    const L = [];
+    L.push(`# UAP Paired Benchmark Report`);
+    L.push('');
+    L.push(`**Model:** ${r.meta.model}  ·  **Adapter:** ${r.meta.adapter}  ·  ` +
+        `**Tasks:** ${r.meta.taskCount}  ·  **Epochs:** ${r.meta.epochs}  ·  ` +
+        `**Baseline:** \`${r.meta.baseline}\``);
+    L.push(`*${r.meta.startedAt} → ${r.meta.finishedAt}*`);
+    L.push('');
+    L.push(`## Per-condition summary`);
+    L.push('');
+    L.push(`| Condition | n | Success | Tokens | Turns | Cost $ | Latency ms | Err |`);
+    L.push(`|---|--:|--:|--:|--:|--:|--:|--:|`);
+    for (const c of r.perCondition) {
+        L.push(`| \`${c.label}\` | ${c.n} | ${pct(c.successRate)} | ${fixed(c.meanTokens)} | ` +
+            `${fixed(c.meanTurns, 1)} | ${fixed(c.meanCostUsd, 4)} | ${fixed(c.meanLatencyMs)} | ${pct(c.errorRate)} |`);
+    }
+    L.push('');
+    L.push(`## Paired comparisons vs \`${r.meta.baseline}\``);
+    L.push('');
+    for (const cmp of r.comparisons) {
+        L.push(`### \`${cmp.label}\``);
+        L.push('');
+        const cd = cmp.correctness;
+        L.push(`**Correctness:** ${pct(cd.baselineRate)} → ${pct(cd.treatmentRate)}  ` +
+            `(Δ ${(cd.delta.meanDelta * 100).toFixed(1)}pp, 95% CI ${ciStr(cd.delta.ci)}, ` +
+            `p=${cd.delta.pValue.toFixed(3)}) ${sig(cd.delta)}`);
+        const m = cd.mcnemar;
+        L.push('');
+        L.push(`**Gate value (McNemar 2×2):** fixed ${m.onlyTreatment}, regressed ${m.onlyBaseline}, ` +
+            `net ${m.netGain >= 0 ? '+' : ''}${m.netGain} (p=${m.pValue.toFixed(3)}); ` +
+            `both✓ ${m.bothCorrect}, both✗ ${m.bothWrong}`);
+        L.push('');
+        L.push(`| Metric | Δ mean | 95% CI | p | sig |`);
+        L.push(`|---|--:|--:|--:|:-:|`);
+        for (const metric of CONTINUOUS_METRICS) {
+            const pd = cmp.metrics[metric];
+            if (!pd)
+                continue;
+            const d = metric === 'costUsd' ? 4 : 1;
+            L.push(`| ${metric} | ${pd.meanDelta.toFixed(d)} | ${ciStr(pd.ci, d)} | ` +
+                `${pd.pValue.toFixed(3)} | ${sig(pd)} |`);
+        }
+        L.push('');
+    }
+    L.push(`## Cost–accuracy Pareto`);
+    L.push('');
+    L.push(`| Condition | Success | Mean tokens |`);
+    L.push(`|---|--:|--:|`);
+    for (const p of [...r.pareto].sort((a, b) => b.successRate - a.successRate)) {
+        L.push(`| \`${p.label}\` | ${pct(p.successRate)} | ${fixed(p.meanTokens)} |`);
+    }
+    L.push('');
+    L.push(`> Interpretation: a credible UAP win is a **positive net gate value** and ` +
+        `**non-inferior success** at **lower tokens/turns** — or higher success at acceptable cost. ` +
+        `Single-arm point estimates without overlapping-zero CIs are not claims.`);
+    L.push('');
+    return L.join('\n');
+}
+//# sourceMappingURL=report.js.map

package/dist/benchmarks/paired/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/benchmarks/paired/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAEL,OAAO,EAEP,IAAI,EACJ,WAAW,GAGZ,MAAM,YAAY,CAAC;AAEpB,OAAO,EAAE,kBAAkB,EAA6C,MAAM,YAAY,CAAC;AA4C3F,SAAS,OAAO,CAAC,CAAY;IAC3B,OAAO,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;AACjC,CAAC;AAED,SAAS,WAAW,CAAC,OAAoB;IACvC,MAAM,CAAC,GAAG,IAAI,GAAG,EAAuB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;QACrC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACZ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAC1B,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,SAAS,SAAS,CAAC,KAAa,EAAE,IAAiB;IACjD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,OAAO,GAAG,CAAC,GAAuC,EAAiB,EAAE;QACzE,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;QACnF,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACzC,CAAC,CAAC;IACF,OAAO;QACL,KAAK;QACL,CAAC,EAAE,IAAI,CAAC,MAAM;QACd,WAAW,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG;QAC9C,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QACpC,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAClC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;QACtC,aAAa,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,GAAG;QACjD,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;KAChF,CAAC;AACJ,CAAC;AAED,+EAA+E;AAC/E,SAAS,SAAS,CAChB,QAAgC,EAChC,SAAiC;IAEjC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,IAAI,GAAgB,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,QAAQ,EAAE,CAAC;QAC9B,MAAM,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC3B,IAAI,CAAC,EAAE,CAAC;YACN,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACb,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACb,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;AAC/B,CAAC;AAED,SAAS,SAAS,CAAC,IAAiB;IAClC,MAAM,CAAC,GAAG,IAAI,GAAG,EAAqB,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,IAAI;QAAE,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC3C,OAAO,CAAC,CAAC;AACX,CAAC;AAMD,MAAM,UAAU,OAAO,CAAC,MAAoB,EAAE,OAAuB,EAAE;IACrE,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,UAAU,CAAC;IACvD,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAE5C,MAAM,UAAU,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC;IACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,gCAAgC,aAAa,wBAAwB,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC9F,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,CAAC,CAAC;IAEtF,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAE,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;IAEpE,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;QAC/B,IAAI,KAAK,KAAK,aAAa;YAAE,SAAS;QACtC,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC,CAAC;QAChD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,SAAS,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;QAErD,sEAAsE;QACtE,MAAM,iBAAiB,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACrG,MAAM,KAAK,GAAG,WAAW,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;QACnD,MAAM,EAAE,GAAG,OAAO,CAChB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,EACnC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CACnC,CAAC;QAEF,wEAAwE;QACxE,MAAM,OAAO,GAAyD,EAAE,CAAC;QACzE,KAAK,MAAM,MAAM,IAAI,kBAAkB,EAAE,CAAC;YACxC,MAAM,MAAM,GAAa,EAAE,CAAC;YAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACrC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBACnC,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;gBACpC,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,IAAI,IAAI;oBAAE,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;YACrD,CAAC;YACD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,CAAC,MAAM,CAAC,GAAG,WAAW,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACrE,CAAC;QAED,WAAW,CAAC,IAAI,CAAC;YACf,KAAK;YACL,QAAQ,EAAE,aAAa;YACvB,WAAW,EAAE;gBACX,YAAY,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC3D,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC7D,KAAK;gBACL,OAAO,EAAE,EAAE;aACZ;YACD,OAAO;SACR,CAAC,CAAC;IACL,CAAC;IAED,OAAO;QACL,IAAI,EAAE;YACJ,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,SAAS;YACT,UAAU;YACV,QAAQ,EAAE,aAAa;YACvB,SAAS,EAAE,MAAM,CAAC,SAAS;YAC3B,UAAU,EAAE,MAAM,CAAC,UAAU;SAC9B;QACD,YAAY;QACZ,WAAW;QACX,MAAM,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC/B,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,WAAW,EAAE,CAAC,CAAC,WAAW;YAC1B,UAAU,EAAE,CAAC,CAAC,UAAU;SACzB,CAAC,CAAC;KACJ,CAAC;AACJ,CAAC;AAED,SAAS,GAAG,CAAC,CAAU;IACrB,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACnB,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E,SAAS,GAAG,CAAC,CAAS;IACpB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC;AACjE,CAAC;AACD,SAAS,KAAK,CAAC,CAAgB,EAAE,CAAC,GAAG,CAAC;IACpC,OAAO,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AACjE,CAAC;AACD,SAAS,KAAK,CAAC,EAAM,EAAE,CAAC,GAAG,CAAC;IAC1B,OAAO,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAC5D,CAAC;AACD,SAAS,GAAG,CAAC,CAAoB;IAC/B,OAAO,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAAiB;IAC9C,MAAM,CAAC,GAAa,EAAE,CAAC;IACvB,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;IACxC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACX,CAAC,CAAC,IAAI,CACJ,cAAc,CAAC,CAAC,IAAI,CAAC,KAAK,qBAAqB,CAAC,CAAC,IAAI,CAAC,OAAO,OAAO;QAClE,cAAc,CAAC,CAAC,IAAI,CAAC,SAAS,oBAAoB,CAAC,CAAC,IAAI,CAAC,MAAM,OAAO;QACtE,mBAAmB,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,CACzC,CAAC;IACF,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,SAAS,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;IACvD,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEX,CAAC,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACnC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACX,CAAC,CAAC,IAAI,CAAC,0EAA0E,CAAC,CAAC;IACnF,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;IAC5C,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,CAAC;QAC/B,CAAC,CAAC,IAAI,CACJ,OAAO,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK;YAC7E,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAC9G,CAAC;IACJ,CAAC;IACD,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEX,CAAC,CAAC,IAAI,CAAC,8BAA8B,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC;IAC1D,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACX,KAAK,MAAM,GAAG,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QAChC,CAAC,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;QAC/B,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACX,MAAM,EAAE,GAAG,GAAG,CAAC,WAAW,CAAC;QAC3B,CAAC,CAAC,IAAI,CACJ,oBAAoB,GAAG,CAAC,EAAE,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,aAAa,CAAC,IAAI;YACrE,MAAM,CAAC,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI;YAC/E,KAAK,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,EAAE,CACtD,CAAC;QACF,MAAM,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC;QACrB,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACX,CAAC,CAAC,IAAI,CACJ,uCAAuC,CAAC,CAAC,aAAa,eAAe,CAAC,CAAC,YAAY,IAAI;YACrF,OAAO,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;YAC3E,SAAS,CAAC,CAAC,WAAW,WAAW,CAAC,CAAC,SAAS,EAAE,CACjD,CAAC;QACF,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACX,CAAC,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QACjD,CAAC,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QAChC,KAAK,MAAM,MAAM,IAAI,kBAAkB,EAAE,CAAC;YACxC,MAAM,EAAE,GAAG,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAC/B,IAAI,CAAC,EAAE;gBAAE,SAAS;YAClB,MAAM,CAAC,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC,CAAC,IAAI,CACJ,KAAK,MAAM,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK;gBAChE,GAAG,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAC3C,CAAC;QACJ,CAAC;QACD,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACb,CAAC;IAED,CAAC,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;IAClC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACX,CAAC,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;IAChD,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IACxB,KAAK,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC;QAC5E,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,KAAK,QAAQ,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAChF,CAAC;IACD,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACX,CAAC,CAAC,IAAI,CACJ,4EAA4E;QAC1E,6FAA6F;QAC7F,yEAAyE,CAC5E,CAAC;IACF,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACX,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACtB,CAAC"}

package/dist/benchmarks/paired/runner.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Paired runner — the experiment engine.
+ *
+ * For every (task, condition, seed) cell it: materializes a fresh isolated copy
+ * of the task's failing repo, runs optional setup, drives the agent adapter
+ * (which injects the UAP surface for non-baseline arms), then applies the
+ * deterministic verify command to score correctness. The SAME seed is used for
+ * the same task across all conditions, so records pair up cell-for-cell — the
+ * key to the paired statistics in ./stats.ts.
+ */
+import { RunRecord, RunnerConfig } from './types.js';
+export interface RunnerOutput {
+    records: RunRecord[];
+    model: string;
+    adapter: string;
+    epochs: number;
+    startedAt: string;
+    finishedAt: string;
+}
+/**
+ * Execute the full (tasks × conditions × epochs) grid.
+ * `suiteDir` is required so the runner can locate each task's fixture repo.
+ */
+export declare function runPaired(cfg: RunnerConfig, suiteDir: string, startedAtIso: string): Promise<RunnerOutput>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/benchmarks/paired/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/paired/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,OAAO,EAAgB,SAAS,EAAE,YAAY,EAAuB,MAAM,YAAY,CAAC;AAQxF,MAAM,WAAW,YAAY;IAC3B,OAAO,EAAE,SAAS,EAAE,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,GAAG,EAAE,YAAY,EACjB,QAAQ,EAAE,MAAM,EAChB,YAAY,EAAE,MAAM,GACnB,OAAO,CAAC,YAAY,CAAC,CAgCvB"}

package/dist/benchmarks/paired/runner.js

@@ -0,0 +1,125 @@
+/**
+ * Paired runner — the experiment engine.
+ *
+ * For every (task, condition, seed) cell it: materializes a fresh isolated copy
+ * of the task's failing repo, runs optional setup, drives the agent adapter
+ * (which injects the UAP surface for non-baseline arms), then applies the
+ * deterministic verify command to score correctness. The SAME seed is used for
+ * the same task across all conditions, so records pair up cell-for-cell — the
+ * key to the paired statistics in ./stats.ts.
+ */
+import { concurrentMap } from '../../utils/concurrency-pool.js';
+import { materializeWorkdir, runSetup, runVerify } from './suite.js';
+/**
+ * Execute the full (tasks × conditions × epochs) grid.
+ * `suiteDir` is required so the runner can locate each task's fixture repo.
+ */
+export async function runPaired(cfg, suiteDir, startedAtIso) {
+    const cells = [];
+    for (const task of cfg.tasks) {
+        for (let seed = 0; seed < cfg.epochs; seed++) {
+            for (const condition of cfg.conditions) {
+                cells.push({ task, condition, seed });
+            }
+        }
+    }
+    let done = 0;
+    const total = cells.length;
+    const records = await concurrentMap(cells, async (cell) => {
+        const rec = await executeCell(cell, cfg, suiteDir);
+        done++;
+        cfg.onProgress?.(done, total, `${cell.task.id}/${cell.condition.label}#${cell.seed}`);
+        return rec;
+    }, { maxConcurrent: cfg.concurrency, mode: 'cpu' });
+    return {
+        records,
+        model: cfg.model,
+        adapter: cfg.adapter.id,
+        epochs: cfg.epochs,
+        startedAt: startedAtIso,
+        finishedAt: nowIso(),
+    };
+}
+async function executeCell(cell, cfg, suiteDir) {
+    const { task, condition, seed } = cell;
+    const t0 = Date.now();
+    const baseMetrics = () => ({
+        correct: false,
+        tokens: null,
+        costUsd: null,
+        turns: null,
+        toolCalls: null,
+        latencyMs: Date.now() - t0,
+        wellFormed: null,
+        error: null,
+    });
+    let workdir;
+    try {
+        workdir = materializeWorkdir(suiteDir, task, cfg.workRoot);
+    }
+    catch (e) {
+        return record(cell, cfg, { ...baseMetrics(), error: `materialize failed: ${errMsg(e)}` });
+    }
+    // Optional setup; a failed setup means the cell can't be scored fairly.
+    try {
+        const setup = runSetup(task, workdir);
+        if (setup && !setup.passed) {
+            return record(cell, cfg, {
+                ...baseMetrics(),
+                error: `setup failed (exit ${setup.exitCode})`,
+            });
+        }
+    }
+    catch (e) {
+        return record(cell, cfg, { ...baseMetrics(), error: `setup error: ${errMsg(e)}` });
+    }
+    // Drive the agent (adapters capture their own failures in `error`).
+    let agent;
+    try {
+        agent = await cfg.adapter.run({ task, condition, workdir, seed, model: cfg.model });
+    }
+    catch (e) {
+        return record(cell, cfg, { ...baseMetrics(), error: `adapter threw: ${errMsg(e)}` });
+    }
+    // Score with the deterministic ground-truth verify command.
+    let correct = false;
+    let verifyErr = agent.error;
+    try {
+        const v = runVerify(task, workdir);
+        correct = v.passed;
+        if (!correct && !verifyErr) {
+            verifyErr = v.timedOut ? 'verify timed out' : `verify failed (exit ${v.exitCode})`;
+        }
+    }
+    catch (e) {
+        verifyErr = `verify error: ${errMsg(e)}`;
+    }
+    const metrics = {
+        correct,
+        tokens: agent.tokens,
+        costUsd: agent.costUsd,
+        turns: agent.turns,
+        toolCalls: agent.toolCalls,
+        latencyMs: Date.now() - t0,
+        wellFormed: agent.wellFormed,
+        error: verifyErr,
+    };
+    return record(cell, cfg, metrics);
+}
+function record(cell, cfg, metrics) {
+    return {
+        taskId: cell.task.id,
+        condition: cell.condition.label,
+        seed: cell.seed,
+        metrics,
+        adapter: cfg.adapter.id,
+        model: cfg.model,
+    };
+}
+function errMsg(e) {
+    return e instanceof Error ? e.message : String(e);
+}
+function nowIso() {
+    return new Date().toISOString();
+}
+//# sourceMappingURL=runner.js.map

package/dist/benchmarks/paired/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/benchmarks/paired/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,iCAAiC,CAAC;AAChE,OAAO,EAAE,kBAAkB,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAkBrE;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,GAAiB,EACjB,QAAgB,EAChB,YAAoB;IAEpB,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;QAC7B,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;YAC7C,KAAK,MAAM,SAAS,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;gBACvC,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;IAE3B,MAAM,OAAO,GAAG,MAAM,aAAa,CACjC,KAAK,EACL,KAAK,EAAE,IAAI,EAAsB,EAAE;QACjC,MAAM,GAAG,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC;QACnD,IAAI,EAAE,CAAC;QACP,GAAG,CAAC,UAAU,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACtF,OAAO,GAAG,CAAC;IACb,CAAC,EACD,EAAE,aAAa,EAAE,GAAG,CAAC,WAAW,EAAE,IAAI,EAAE,KAAK,EAAE,CAChD,CAAC;IAEF,OAAO;QACL,OAAO;QACP,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE;QACvB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,YAAY;QACvB,UAAU,EAAE,MAAM,EAAE;KACrB,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,WAAW,CAAC,IAAU,EAAE,GAAiB,EAAE,QAAgB;IACxE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,GAAG,IAAI,CAAC;IACvC,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEtB,MAAM,WAAW,GAAG,GAAiB,EAAE,CAAC,CAAC;QACvC,OAAO,EAAE,KAAK;QACd,MAAM,EAAE,IAAI;QACZ,OAAO,EAAE,IAAI;QACb,KAAK,EAAE,IAAI;QACX,SAAS,EAAE,IAAI;QACf,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE;QAC1B,UAAU,EAAE,IAAI;QAChB,KAAK,EAAE,IAAI;KACZ,CAAC,CAAC;IAEH,IAAI,OAAe,CAAC;IACpB,IAAI,CAAC;QACH,OAAO,GAAG,kBAAkB,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC7D,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,GAAG,WAAW,EAAE,EAAE,KAAK,EAAE,uBAAuB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC5F,CAAC;IAED,wEAAwE;IACxE,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACtC,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YAC3B,OAAO,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE;gBACvB,GAAG,WAAW,EAAE;gBAChB,KAAK,EAAE,sBAAsB,KAAK,CAAC,QAAQ,GAAG;aAC/C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,GAAG,WAAW,EAAE,EAAE,KAAK,EAAE,gBAAgB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACrF,CAAC;IAED,oEAAoE;IACpE,IAAI,KAAK,CAAC;IACV,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;IACtF,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,OAAO,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,GAAG,WAAW,EAAE,EAAE,KAAK,EAAE,kBAAkB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACvF,CAAC;IAED,4DAA4D;IAC5D,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,SAAS,GAAkB,KAAK,CAAC,KAAK,CAAC;IAC3C,IAAI,CAAC;QACH,MAAM,CAAC,GAAG,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QACnC,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC;QACnB,IAAI,CAAC,OAAO,IAAI,CAAC,SAAS,EAAE,CAAC;YAC3B,SAAS,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,QAAQ,GAAG,CAAC;QACrF,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,SAAS,GAAG,iBAAiB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;IAC3C,CAAC;IAED,MAAM,OAAO,GAAiB;QAC5B,OAAO;QACP,MAAM,EAAE,KAAK,CAAC,MAAM;QACpB,OAAO,EAAE,KAAK,CAAC,OAAO;QACtB,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE;QAC1B,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,KAAK,EAAE,SAAS;KACjB,CAAC;IACF,OAAO,MAAM,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CAAC;AACpC,CAAC;AAED,SAAS,MAAM,CAAC,IAAU,EAAE,GAAiB,EAAE,OAAqB;IAClE,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE;QACpB,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK;QAC/B,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,OAAO;QACP,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,EAAE;QACvB,KAAK,EAAE,GAAG,CAAC,KAAK;KACjB,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,CAAU;IACxB,OAAO,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,MAAM;IACb,OAAO,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAClC,CAAC"}

package/dist/benchmarks/paired/scaffold.d.ts

@@ -0,0 +1,28 @@
+/**
+ * UAP scaffolding injection — the actual on/off mechanism for the A/B.
+ *
+ * A controlled experiment must change exactly ONE thing between arms: the
+ * presence of the UAP layer. For a real coding agent, "UAP" is the instruction
+ * surface + component config the agent's installed hooks consume. This module
+ * writes that surface into the scratch repo for the enabled components, and
+ * nothing for the baseline. Everything else (model, prompt, repo state, seed)
+ * is held identical by the runner.
+ *
+ * The injected files are deliberately plain-text and inspectable so a reviewer
+ * can audit precisely what the "treatment" was (HAL-style log/scaffold audit),
+ * pre-empting the "asymmetric prompt" critique that sank prior memory-layer
+ * benchmarks.
+ */
+import { Condition, UapComponent } from './types.js';
+export interface ScaffoldManifest {
+    components: UapComponent[];
+    injectedFiles: string[];
+}
+/**
+ * Write the UAP instruction surface for `condition` into `workdir`.
+ * Returns a manifest of what was injected (empty for the baseline).
+ */
+export declare function applyScaffolding(workdir: string, condition: Condition): ScaffoldManifest;
+/** Environment variables exported to the agent describing the active condition. */
+export declare function scaffoldEnv(condition: Condition): NodeJS.ProcessEnv;
+//# sourceMappingURL=scaffold.d.ts.map

package/dist/benchmarks/paired/scaffold.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scaffold.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/paired/scaffold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAKH,OAAO,EAAE,SAAS,EAAkB,YAAY,EAAE,MAAM,YAAY,CAAC;AAyBrE,MAAM,WAAW,gBAAgB;IAC/B,UAAU,EAAE,YAAY,EAAE,CAAC;IAC3B,aAAa,EAAE,MAAM,EAAE,CAAC;CACzB;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,GAAG,gBAAgB,CA6BxF;AAED,mFAAmF;AACnF,wBAAgB,WAAW,CAAC,SAAS,EAAE,SAAS,GAAG,MAAM,CAAC,UAAU,CAQnE"}

package/dist/benchmarks/paired/scaffold.js

@@ -0,0 +1,74 @@
+/**
+ * UAP scaffolding injection — the actual on/off mechanism for the A/B.
+ *
+ * A controlled experiment must change exactly ONE thing between arms: the
+ * presence of the UAP layer. For a real coding agent, "UAP" is the instruction
+ * surface + component config the agent's installed hooks consume. This module
+ * writes that surface into the scratch repo for the enabled components, and
+ * nothing for the baseline. Everything else (model, prompt, repo state, seed)
+ * is held identical by the runner.
+ *
+ * The injected files are deliberately plain-text and inspectable so a reviewer
+ * can audit precisely what the "treatment" was (HAL-style log/scaffold audit),
+ * pre-empting the "asymmetric prompt" critique that sank prior memory-layer
+ * benchmarks.
+ */
+import { writeFileSync } from 'fs';
+import { join } from 'path';
+import { UAP_COMPONENTS } from './types.js';
+/** Human-readable instruction block injected per enabled component. */
+const COMPONENT_INSTRUCTIONS = {
+    gates: '## Completion Gates\nBefore claiming the task is done, you MUST run the build, ' +
+        'the test suite, and the linter, and confirm all pass. Do not stop while any ' +
+        'gate is failing. Re-verify at least once after your final edit.',
+    worktree: '## Isolation\nMake all edits within this repository only. Do not touch files ' +
+        'outside the working directory. Keep changes scoped and atomic.',
+    memory: '## Memory\nRelevant prior context and learned patterns for this repository are ' +
+        'available. Consult them before editing and prefer established patterns over ad-hoc ones.',
+    experts: '## Expert Routing\nFor specialized sub-problems (security, performance, tests), ' +
+        'apply the matching expert discipline and review the change against its checklist.',
+    skills: '## Skills\nWhen a domain-specific workflow matches the task, follow its ' +
+        'structured procedure rather than improvising.',
+    patterns: '## Patterns\nMatch the task to a known execution pattern; in particular always ' +
+        'verify the produced output actually exists and the decoder/contract is satisfied.',
+};
+/**
+ * Write the UAP instruction surface for `condition` into `workdir`.
+ * Returns a manifest of what was injected (empty for the baseline).
+ */
+export function applyScaffolding(workdir, condition) {
+    const enabled = UAP_COMPONENTS.filter((c) => condition.components.has(c));
+    if (enabled.length === 0) {
+        return { components: [], injectedFiles: [] };
+    }
+    const injectedFiles = [];
+    // AGENTS.md is the cross-agent instruction file (Claude, opencode, Codex, etc.
+    // all read it). This is the primary, honest treatment surface.
+    const header = '# UAP Operating Protocol\n\n' +
+        'The following protocol is active for this task. Follow it precisely.\n';
+    const body = enabled.map((c) => COMPONENT_INSTRUCTIONS[c]).join('\n\n');
+    const agentsMd = `${header}\n${body}\n`;
+    writeFileSync(join(workdir, 'AGENTS.md'), agentsMd, 'utf-8');
+    injectedFiles.push('AGENTS.md');
+    // Machine-readable manifest that installed UAP hooks can consume to enable the
+    // corresponding enforcement (gates/memory/reactor) at runtime.
+    const manifest = {
+        enabled,
+        condition: condition.label,
+        generatedBy: 'uap-paired-bench',
+    };
+    writeFileSync(join(workdir, '.uap-bench.json'), JSON.stringify(manifest, null, 2), 'utf-8');
+    injectedFiles.push('.uap-bench.json');
+    return { components: enabled, injectedFiles };
+}
+/** Environment variables exported to the agent describing the active condition. */
+export function scaffoldEnv(condition) {
+    const enabled = UAP_COMPONENTS.filter((c) => condition.components.has(c));
+    return {
+        UAP_BENCH_CONDITION: condition.label,
+        UAP_BENCH_COMPONENTS: enabled.join(','),
+        // Enable/disable the real delivery-enforcement surface to match the arm.
+        UAP_DELIVER_ACTIVE: enabled.includes('gates') ? '1' : '0',
+    };
+}
+//# sourceMappingURL=scaffold.js.map

package/dist/benchmarks/paired/scaffold.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scaffold.js","sourceRoot":"","sources":["../../../src/benchmarks/paired/scaffold.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAa,cAAc,EAAgB,MAAM,YAAY,CAAC;AAErE,uEAAuE;AACvE,MAAM,sBAAsB,GAAiC;IAC3D,KAAK,EACH,iFAAiF;QACjF,8EAA8E;QAC9E,iEAAiE;IACnE,QAAQ,EACN,+EAA+E;QAC/E,gEAAgE;IAClE,MAAM,EACJ,iFAAiF;QACjF,0FAA0F;IAC5F,OAAO,EACL,kFAAkF;QAClF,mFAAmF;IACrF,MAAM,EACJ,0EAA0E;QAC1E,+CAA+C;IACjD,QAAQ,EACN,iFAAiF;QACjF,mFAAmF;CACtF,CAAC;AAOF;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,SAAoB;IACpE,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,aAAa,EAAE,EAAE,EAAE,CAAC;IAC/C,CAAC;IAED,MAAM,aAAa,GAAa,EAAE,CAAC;IAEnC,+EAA+E;IAC/E,+DAA+D;IAC/D,MAAM,MAAM,GACV,8BAA8B;QAC9B,wEAAwE,CAAC;IAC3E,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACxE,MAAM,QAAQ,GAAG,GAAG,MAAM,KAAK,IAAI,IAAI,CAAC;IACxC,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC7D,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEhC,+EAA+E;IAC/E,+DAA+D;IAC/D,MAAM,QAAQ,GAAG;QACf,OAAO;QACP,SAAS,EAAE,SAAS,CAAC,KAAK;QAC1B,WAAW,EAAE,kBAAkB;KAChC,CAAC;IACF,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IAC5F,aAAa,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAEtC,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,aAAa,EAAE,CAAC;AAChD,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,WAAW,CAAC,SAAoB;IAC9C,MAAM,OAAO,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1E,OAAO;QACL,mBAAmB,EAAE,SAAS,CAAC,KAAK;QACpC,oBAAoB,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;QACvC,yEAAyE;QACzE,kBAAkB,EAAE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;KAC1D,CAAC;AACJ,CAAC"}

package/dist/benchmarks/paired/stats.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Paired-comparison statistics for the UAP benchmark.
+ *
+ * The research is explicit: agent evals are stochastic, so a point estimate is
+ * meaningless without a variance measure, and the *paired* design (same task &
+ * seed in both arms) is what gives statistical power — it removes between-task
+ * variance, the dominant noise source. This module implements:
+ *
+ *  - mean / std / standard error
+ *  - paired bootstrap confidence interval on a delta (percentile method)
+ *  - paired permutation test (two-sided) for significance of a delta
+ *  - McNemar contingency for paired binary outcomes (the gate-value 2x2)
+ *  - pass@k reducer
+ *
+ * All randomized procedures take an explicit seed so reports are reproducible
+ * (the Mem0/Zep scandal lesson: single-run, non-reproducible numbers are the #1
+ * credibility red flag).
+ */
+export declare function mulberry32(seed: number): () => number;
+export declare function mean(xs: number[]): number;
+export declare function variance(xs: number[]): number;
+export declare function std(xs: number[]): number;
+export declare function stderr(xs: number[]): number;
+export interface CI {
+    lower: number;
+    upper: number;
+}
+export interface PairedDeltaResult {
+    /** Mean of (treatment - baseline) over paired observations. */
+    meanDelta: number;
+    /** Bootstrap percentile CI on the mean delta. */
+    ci: CI;
+    /** Two-sided permutation p-value for H0: delta == 0. */
+    pValue: number;
+    /** Number of paired observations. */
+    n: number;
+    /** True when the CI excludes 0 at the configured level. */
+    significant: boolean;
+}
+export interface PairedOptions {
+    /** Bootstrap/permutation iterations (default 10000). */
+    iterations?: number;
+    /** Confidence level for the CI, e.g. 0.95 (default). */
+    confidence?: number;
+    /** RNG seed for reproducibility (default 1). */
+    seed?: number;
+}
+/**
+ * Paired analysis of a treatment vs baseline metric. `deltas[i]` is
+ * (treatment_i - baseline_i) for the i-th paired observation. Returns the mean
+ * delta, a bootstrap CI, and a permutation p-value (sign-flip, the correct null
+ * for a paired design).
+ */
+export declare function pairedDelta(deltas: number[], opts?: PairedOptions): PairedDeltaResult;
+export interface McNemarResult {
+    bothCorrect: number;
+    onlyTreatment: number;
+    onlyBaseline: number;
+    bothWrong: number;
+    /** Net correctness gain from treatment: (onlyTreatment - onlyBaseline). */
+    netGain: number;
+    /** Two-sided exact McNemar p-value on the discordant pairs. */
+    pValue: number;
+    n: number;
+}
+export declare function mcnemar(treatment: boolean[], baseline: boolean[]): McNemarResult;
+export declare function passAtK(numSamples: number, numCorrect: number, k: number): number;
+//# sourceMappingURL=stats.d.ts.map

package/dist/benchmarks/paired/stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/paired/stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAKH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,MAAM,CASrD;AAKD,wBAAgB,IAAI,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,MAAM,CAGzC;AAED,wBAAgB,QAAQ,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,MAAM,CAI7C;AAED,wBAAgB,GAAG,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,MAAM,CAExC;AAED,wBAAgB,MAAM,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,MAAM,CAG3C;AAED,MAAM,WAAW,EAAE;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,iBAAiB;IAChC,+DAA+D;IAC/D,SAAS,EAAE,MAAM,CAAC;IAClB,iDAAiD;IACjD,EAAE,EAAE,EAAE,CAAC;IACP,wDAAwD;IACxD,MAAM,EAAE,MAAM,CAAC;IACf,qCAAqC;IACrC,CAAC,EAAE,MAAM,CAAC;IACV,2DAA2D;IAC3D,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gDAAgD;IAChD,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAYD;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,IAAI,GAAE,aAAkB,GAAG,iBAAiB,CAgDzF;AAQD,MAAM,WAAW,aAAa;IAC5B,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,YAAY,EAAE,MAAM,CAAC;IACrB,SAAS,EAAE,MAAM,CAAC;IAClB,2EAA2E;IAC3E,OAAO,EAAE,MAAM,CAAC;IAChB,+DAA+D;IAC/D,MAAM,EAAE,MAAM,CAAC;IACf,CAAC,EAAE,MAAM,CAAC;CACX;AAuBD,wBAAgB,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,aAAa,CAyBhF;AAMD,wBAAgB,OAAO,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAQjF"}