@miller-tech/uap 1.50.1 → 1.52.0

package/dist/benchmarks/paired/stats.js

@@ -0,0 +1,182 @@
+/**
+ * Paired-comparison statistics for the UAP benchmark.
+ *
+ * The research is explicit: agent evals are stochastic, so a point estimate is
+ * meaningless without a variance measure, and the *paired* design (same task &
+ * seed in both arms) is what gives statistical power — it removes between-task
+ * variance, the dominant noise source. This module implements:
+ *
+ *  - mean / std / standard error
+ *  - paired bootstrap confidence interval on a delta (percentile method)
+ *  - paired permutation test (two-sided) for significance of a delta
+ *  - McNemar contingency for paired binary outcomes (the gate-value 2x2)
+ *  - pass@k reducer
+ *
+ * All randomized procedures take an explicit seed so reports are reproducible
+ * (the Mem0/Zep scandal lesson: single-run, non-reproducible numbers are the #1
+ * credibility red flag).
+ */
+// ---------------------------------------------------------------------------
+// Seeded PRNG (mulberry32) — deterministic, dependency-free.
+// ---------------------------------------------------------------------------
+export function mulberry32(seed) {
+    let a = seed >>> 0;
+    return function () {
+        a |= 0;
+        a = (a + 0x6d2b79f5) | 0;
+        let t = Math.imul(a ^ (a >>> 15), 1 | a);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+// ---------------------------------------------------------------------------
+// Descriptive stats
+// ---------------------------------------------------------------------------
+export function mean(xs) {
+    if (xs.length === 0)
+        return NaN;
+    return xs.reduce((a, b) => a + b, 0) / xs.length;
+}
+export function variance(xs) {
+    if (xs.length < 2)
+        return 0;
+    const m = mean(xs);
+    return xs.reduce((a, b) => a + (b - m) * (b - m), 0) / (xs.length - 1);
+}
+export function std(xs) {
+    return Math.sqrt(variance(xs));
+}
+export function stderr(xs) {
+    if (xs.length === 0)
+        return NaN;
+    return std(xs) / Math.sqrt(xs.length);
+}
+function percentile(sorted, q) {
+    if (sorted.length === 0)
+        return NaN;
+    const idx = q * (sorted.length - 1);
+    const lo = Math.floor(idx);
+    const hi = Math.ceil(idx);
+    if (lo === hi)
+        return sorted[lo];
+    const frac = idx - lo;
+    return sorted[lo] * (1 - frac) + sorted[hi] * frac;
+}
+/**
+ * Paired analysis of a treatment vs baseline metric. `deltas[i]` is
+ * (treatment_i - baseline_i) for the i-th paired observation. Returns the mean
+ * delta, a bootstrap CI, and a permutation p-value (sign-flip, the correct null
+ * for a paired design).
+ */
+export function pairedDelta(deltas, opts = {}) {
+    const iterations = opts.iterations ?? 10000;
+    const confidence = opts.confidence ?? 0.95;
+    const rng = mulberry32(opts.seed ?? 1);
+    const n = deltas.length;
+    if (n === 0) {
+        return { meanDelta: NaN, ci: { lower: NaN, upper: NaN }, pValue: NaN, n: 0, significant: false };
+    }
+    const observed = mean(deltas);
+    // --- Bootstrap CI: resample paired deltas with replacement.
+    const bootMeans = new Array(iterations);
+    for (let b = 0; b < iterations; b++) {
+        let acc = 0;
+        for (let i = 0; i < n; i++) {
+            acc += deltas[(rng() * n) | 0];
+        }
+        bootMeans[b] = acc / n;
+    }
+    bootMeans.sort((a, b) => a - b);
+    const alpha = 1 - confidence;
+    const ci = {
+        lower: percentile(bootMeans, alpha / 2),
+        upper: percentile(bootMeans, 1 - alpha / 2),
+    };
+    // --- Permutation test: under H0 the sign of each paired delta is arbitrary,
+    //     so randomly flip signs and compare |mean| to observed.
+    let extreme = 0;
+    const absObs = Math.abs(observed);
+    for (let p = 0; p < iterations; p++) {
+        let acc = 0;
+        for (let i = 0; i < n; i++) {
+            acc += rng() < 0.5 ? deltas[i] : -deltas[i];
+        }
+        if (Math.abs(acc / n) >= absObs - 1e-12)
+            extreme++;
+    }
+    const pValue = (extreme + 1) / (iterations + 1);
+    return {
+        meanDelta: observed,
+        ci,
+        pValue,
+        n,
+        significant: ci.lower > 0 || ci.upper < 0,
+    };
+}
+/** Exact binomial two-sided tail (used for McNemar on discordant pairs). */
+function binomTwoSided(b, c) {
+    const nDisc = b + c;
+    if (nDisc === 0)
+        return 1;
+    // P(X <= min) under Binomial(nDisc, 0.5), doubled, capped at 1.
+    const k = Math.min(b, c);
+    let logFac = 0;
+    const logFacs = new Array(nDisc + 1);
+    logFacs[0] = 0;
+    for (let i = 1; i <= nDisc; i++) {
+        logFac += Math.log(i);
+        logFacs[i] = logFac;
+    }
+    const logChoose = (nn, kk) => logFacs[nn] - logFacs[kk] - logFacs[nn - kk];
+    let tail = 0;
+    for (let i = 0; i <= k; i++) {
+        tail += Math.exp(logChoose(nDisc, i) + nDisc * Math.log(0.5));
+    }
+    return Math.min(1, 2 * tail);
+}
+export function mcnemar(treatment, baseline) {
+    if (treatment.length !== baseline.length) {
+        throw new Error('mcnemar: paired arrays must be equal length');
+    }
+    let bothCorrect = 0;
+    let onlyTreatment = 0;
+    let onlyBaseline = 0;
+    let bothWrong = 0;
+    for (let i = 0; i < treatment.length; i++) {
+        const t = treatment[i];
+        const b = baseline[i];
+        if (t && b)
+            bothCorrect++;
+        else if (t && !b)
+            onlyTreatment++;
+        else if (!t && b)
+            onlyBaseline++;
+        else
+            bothWrong++;
+    }
+    return {
+        bothCorrect,
+        onlyTreatment,
+        onlyBaseline,
+        bothWrong,
+        netGain: onlyTreatment - onlyBaseline,
+        pValue: binomTwoSided(onlyTreatment, onlyBaseline),
+        n: treatment.length,
+    };
+}
+// ---------------------------------------------------------------------------
+// pass@k — probability at least one of k i.i.d. attempts succeeds, using the
+// unbiased estimator from the Codex/HumanEval literature.
+// ---------------------------------------------------------------------------
+export function passAtK(numSamples, numCorrect, k) {
+    if (k > numSamples)
+        throw new Error('passAtK: k cannot exceed numSamples');
+    if (numSamples - numCorrect < k)
+        return 1;
+    let prod = 1;
+    for (let i = 0; i < k; i++) {
+        prod *= (numSamples - numCorrect - i) / (numSamples - i);
+    }
+    return 1 - prod;
+}
+//# sourceMappingURL=stats.js.map

package/dist/benchmarks/paired/stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../../src/benchmarks/paired/stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,8EAA8E;AAC9E,6DAA6D;AAC7D,8EAA8E;AAC9E,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,IAAI,CAAC,GAAG,IAAI,KAAK,CAAC,CAAC;IACnB,OAAO;QACL,CAAC,IAAI,CAAC,CAAC;QACP,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QACzB,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACzC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/C,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,UAAU,CAAC;IAC/C,CAAC,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAC9E,MAAM,UAAU,IAAI,CAAC,EAAY;IAC/B,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAChC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC;AACnD,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,EAAY;IACnC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC5B,MAAM,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC;IACnB,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,GAAG,CAAC,EAAY;IAC9B,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,EAAY;IACjC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAChC,OAAO,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;AACxC,CAAC;AA6BD,SAAS,UAAU,CAAC,MAAgB,EAAE,CAAS;IAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IACpC,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,MAAM,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAC3B,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAC1B,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,MAAM,CAAC,EAAE,CAAC,CAAC;IACjC,MAAM,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;IACtB,OAAO,MAAM,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,MAAM,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC;AACrD,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,WAAW,CAAC,MAAgB,EAAE,OAAsB,EAAE;IACpE,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,KAAK,CAAC;IAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC;IAC3C,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAExB,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,EAAE,WAAW,EAAE,KAAK,EAAE,CAAC;IACnG,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;IAE9B,6DAA6D;IAC7D,MAAM,SAAS,GAAa,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC;IAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,GAAG,IAAI,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACjC,CAAC;QACD,SAAS,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC;IACzB,CAAC;IACD,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAChC,MAAM,KAAK,GAAG,CAAC,GAAG,UAAU,CAAC;IAC7B,MAAM,EAAE,GAAO;QACb,KAAK,EAAE,UAAU,CAAC,SAAS,EAAE,KAAK,GAAG,CAAC,CAAC;QACvC,KAAK,EAAE,UAAU,CAAC,SAAS,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;KAC5C,CAAC;IAEF,6EAA6E;IAC7E,6DAA6D;IAC7D,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,GAAG,GAAG,CAAC,CAAC;QACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,GAAG,IAAI,GAAG,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAC9C,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC,IAAI,MAAM,GAAG,KAAK;YAAE,OAAO,EAAE,CAAC;IACrD,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IAEhD,OAAO;QACL,SAAS,EAAE,QAAQ;QACnB,EAAE;QACF,MAAM;QACN,CAAC;QACD,WAAW,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,GAAG,CAAC;KAC1C,CAAC;AACJ,CAAC;AAoBD,4EAA4E;AAC5E,SAAS,aAAa,CAAC,CAAS,EAAE,CAAS;IACzC,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;IACpB,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC1B,gEAAgE;IAChE,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACzB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAa,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAC/C,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAChC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACtB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC;IACtB,CAAC;IACD,MAAM,SAAS,GAAG,CAAC,EAAU,EAAE,EAAU,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;IAC3F,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,OAAO,CAAC,SAAoB,EAAE,QAAmB;IAC/D,IAAI,SAAS,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC1C,MAAM,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;QACvB,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,IAAI,CAAC;YAAE,WAAW,EAAE,CAAC;aACrB,IAAI,CAAC,IAAI,CAAC,CAAC;YAAE,aAAa,EAAE,CAAC;aAC7B,IAAI,CAAC,CAAC,IAAI,CAAC;YAAE,YAAY,EAAE,CAAC;;YAC5B,SAAS,EAAE,CAAC;IACnB,CAAC;IACD,OAAO;QACL,WAAW;QACX,aAAa;QACb,YAAY;QACZ,SAAS;QACT,OAAO,EAAE,aAAa,GAAG,YAAY;QACrC,MAAM,EAAE,aAAa,CAAC,aAAa,EAAE,YAAY,CAAC;QAClD,CAAC,EAAE,SAAS,CAAC,MAAM;KACpB,CAAC;AACJ,CAAC;AAED,8EAA8E;AAC9E,6EAA6E;AAC7E,0DAA0D;AAC1D,8EAA8E;AAC9E,MAAM,UAAU,OAAO,CAAC,UAAkB,EAAE,UAAkB,EAAE,CAAS;IACvE,IAAI,CAAC,GAAG,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC3E,IAAI,UAAU,GAAG,UAAU,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAC1C,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,IAAI,CAAC,UAAU,GAAG,UAAU,GAAG,CAAC,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,CAAC,GAAG,IAAI,CAAC;AAClB,CAAC"}

package/dist/benchmarks/paired/suite.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Real-gate task-suite loader + verifier.
+ *
+ * A suite is a directory of task folders:
+ *
+ *   benchmarks/suites/real-gate/
+ *     <task-id>/
+ *       task.json        # TaskSpec (without `id`; id derives from folder name)
+ *       repo/            # git fixture in a failing state
+ *
+ * The `verifyCmd` in task.json is the deterministic ground-truth scorer: it runs
+ * inside an isolated copy of `repo/` and must exit 0 iff the task is resolved.
+ * No LLM judge — coding tasks give us real test execution as ground truth.
+ */
+import { TaskSpec } from './types.js';
+/** Load and validate a single task folder. The folder name becomes the id. */
+export declare function loadTask(taskDir: string): TaskSpec;
+/** Load every task folder under a suite directory (sorted by id for stability). */
+export declare function loadSuite(suiteDir: string): TaskSpec[];
+/** Absolute path to a task's fixture repo, given the suite dir. */
+export declare function taskRepoPath(suiteDir: string, task: TaskSpec): string;
+/**
+ * Copy a task's fixture repo into a fresh isolated scratch directory so every
+ * run starts from the identical failing state (common-random-numbers principle:
+ * only the UAP toggle differs between arms).
+ */
+export declare function materializeWorkdir(suiteDir: string, task: TaskSpec, workRoot: string): string;
+/** Make a fresh scratch dir under the OS tmp (used when no workRoot is given). */
+export declare function tmpWorkRoot(): string;
+export interface VerifyResult {
+    passed: boolean;
+    exitCode: number;
+    timedOut: boolean;
+    stdout: string;
+    stderr: string;
+}
+/** Run the task's deterministic verify command inside `workdir`. */
+export declare function runVerify(task: TaskSpec, workdir: string): VerifyResult;
+/** Run an optional one-time setup command inside `workdir`. */
+export declare function runSetup(task: TaskSpec, workdir: string): VerifyResult | null;
+/** Copy of process.env with GIT_* repository pointers removed. */
+export declare function sanitizedEnv(): NodeJS.ProcessEnv;
+//# sourceMappingURL=suite.d.ts.map

package/dist/benchmarks/paired/suite.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"suite.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/paired/suite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAeH,OAAO,EAAE,QAAQ,EAAkB,MAAM,YAAY,CAAC;AAEtD,8EAA8E;AAC9E,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,QAAQ,CAelD;AAED,mFAAmF;AACnF,wBAAgB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,QAAQ,EAAE,CActD;AAED,mEAAmE;AACnE,wBAAgB,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,GAAG,MAAM,CAErE;AAED;;;;GAIG;AACH,wBAAgB,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAM7F;AAED,kFAAkF;AAClF,wBAAgB,WAAW,IAAI,MAAM,CAEpC;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,oEAAoE;AACpE,wBAAgB,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAAG,YAAY,CAiBvE;AAED,+DAA+D;AAC/D,wBAAgB,QAAQ,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI,CAe7E;AAED,kEAAkE;AAClE,wBAAgB,YAAY,IAAI,MAAM,CAAC,UAAU,CAMhD"}

package/dist/benchmarks/paired/suite.js

@@ -0,0 +1,117 @@
+/**
+ * Real-gate task-suite loader + verifier.
+ *
+ * A suite is a directory of task folders:
+ *
+ *   benchmarks/suites/real-gate/
+ *     <task-id>/
+ *       task.json        # TaskSpec (without `id`; id derives from folder name)
+ *       repo/            # git fixture in a failing state
+ *
+ * The `verifyCmd` in task.json is the deterministic ground-truth scorer: it runs
+ * inside an isolated copy of `repo/` and must exit 0 iff the task is resolved.
+ * No LLM judge — coding tasks give us real test execution as ground truth.
+ */
+import { spawnSync } from 'child_process';
+import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, statSync, } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { TaskSpecSchema } from './types.js';
+/** Load and validate a single task folder. The folder name becomes the id. */
+export function loadTask(taskDir) {
+    const specPath = join(taskDir, 'task.json');
+    if (!existsSync(specPath)) {
+        throw new Error(`Task folder ${taskDir} is missing task.json`);
+    }
+    const raw = JSON.parse(readFileSync(specPath, 'utf-8'));
+    // Folder name is the canonical id unless task.json overrides it.
+    const id = raw.id ?? taskDir.split('/').filter(Boolean).pop() ?? 'task';
+    const parsed = TaskSpecSchema.parse({ ...raw, id });
+    const repoPath = join(taskDir, parsed.repoDir);
+    if (!existsSync(repoPath)) {
+        throw new Error(`Task ${id}: repo dir '${parsed.repoDir}' not found at ${repoPath}`);
+    }
+    return parsed;
+}
+/** Load every task folder under a suite directory (sorted by id for stability). */
+export function loadSuite(suiteDir) {
+    if (!existsSync(suiteDir)) {
+        throw new Error(`Suite directory not found: ${suiteDir}`);
+    }
+    const entries = readdirSync(suiteDir)
+        .filter((name) => !name.startsWith('.'))
+        .map((name) => join(suiteDir, name))
+        .filter((p) => statSync(p).isDirectory() && existsSync(join(p, 'task.json')));
+    const tasks = entries.map(loadTask).sort((a, b) => a.id.localeCompare(b.id));
+    if (tasks.length === 0) {
+        throw new Error(`No tasks found in suite ${suiteDir}`);
+    }
+    return tasks;
+}
+/** Absolute path to a task's fixture repo, given the suite dir. */
+export function taskRepoPath(suiteDir, task) {
+    return join(suiteDir, task.id, task.repoDir);
+}
+/**
+ * Copy a task's fixture repo into a fresh isolated scratch directory so every
+ * run starts from the identical failing state (common-random-numbers principle:
+ * only the UAP toggle differs between arms).
+ */
+export function materializeWorkdir(suiteDir, task, workRoot) {
+    mkdirSync(workRoot, { recursive: true });
+    const scratch = mkdtempSync(join(workRoot, `${task.id}-`));
+    const dest = join(scratch, 'repo');
+    cpSync(taskRepoPath(suiteDir, task), dest, { recursive: true });
+    return dest;
+}
+/** Make a fresh scratch dir under the OS tmp (used when no workRoot is given). */
+export function tmpWorkRoot() {
+    return mkdtempSync(join(tmpdir(), 'uap-bench-'));
+}
+/** Run the task's deterministic verify command inside `workdir`. */
+export function runVerify(task, workdir) {
+    const res = spawnSync('bash', ['-lc', task.verifyCmd], {
+        cwd: workdir,
+        encoding: 'utf-8',
+        timeout: task.verifyTimeoutSec * 1000,
+        // Strip inherited GIT_* env so a verify that shells out to git targets the
+        // scratch repo, not an enclosing worktree (known GIT_DIR poisoning hazard).
+        env: sanitizedEnv(),
+    });
+    const timedOut = res.signal === 'SIGTERM' && res.status === null;
+    return {
+        passed: res.status === 0 && !timedOut,
+        exitCode: res.status ?? (timedOut ? 124 : 1),
+        timedOut,
+        stdout: res.stdout ?? '',
+        stderr: res.stderr ?? '',
+    };
+}
+/** Run an optional one-time setup command inside `workdir`. */
+export function runSetup(task, workdir) {
+    if (!task.setupCmd)
+        return null;
+    const res = spawnSync('bash', ['-lc', task.setupCmd], {
+        cwd: workdir,
+        encoding: 'utf-8',
+        timeout: task.verifyTimeoutSec * 1000,
+        env: sanitizedEnv(),
+    });
+    return {
+        passed: res.status === 0,
+        exitCode: res.status ?? 1,
+        timedOut: false,
+        stdout: res.stdout ?? '',
+        stderr: res.stderr ?? '',
+    };
+}
+/** Copy of process.env with GIT_* repository pointers removed. */
+export function sanitizedEnv() {
+    const env = { ...process.env };
+    for (const key of Object.keys(env)) {
+        if (key.startsWith('GIT_'))
+            delete env[key];
+    }
+    return env;
+}
+//# sourceMappingURL=suite.js.map

package/dist/benchmarks/paired/suite.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"suite.js","sourceRoot":"","sources":["../../../src/benchmarks/paired/suite.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAC1C,OAAO,EACL,MAAM,EACN,UAAU,EACV,SAAS,EACT,WAAW,EACX,YAAY,EACZ,WAAW,EACX,QAAQ,GACT,MAAM,IAAI,CAAC;AACZ,OAAO,EAAE,MAAM,EAAE,MAAM,IAAI,CAAC;AAC5B,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAE5B,OAAO,EAAY,cAAc,EAAE,MAAM,YAAY,CAAC;AAEtD,8EAA8E;AAC9E,MAAM,UAAU,QAAQ,CAAC,OAAe;IACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC;IAC5C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,eAAe,OAAO,uBAAuB,CAAC,CAAC;IACjE,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAA4B,CAAC;IACnF,iEAAiE;IACjE,MAAM,EAAE,GAAI,GAAG,CAAC,EAAa,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,IAAI,MAAM,CAAC;IACpF,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IAEpD,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IAC/C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,QAAQ,EAAE,eAAe,MAAM,CAAC,OAAO,kBAAkB,QAAQ,EAAE,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,SAAS,CAAC,QAAgB;IACxC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,8BAA8B,QAAQ,EAAE,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,CAAC;SAClC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;SACvC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;SACnC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;IAEhF,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC7E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CAAC,2BAA2B,QAAQ,EAAE,CAAC,CAAC;IACzD,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,YAAY,CAAC,QAAgB,EAAE,IAAc;IAC3D,OAAO,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;AAC/C,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,kBAAkB,CAAC,QAAgB,EAAE,IAAc,EAAE,QAAgB;IACnF,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACzC,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;IAC3D,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACnC,MAAM,CAAC,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,WAAW;IACzB,OAAO,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC;AACnD,CAAC;AAUD,oEAAoE;AACpE,MAAM,UAAU,SAAS,CAAC,IAAc,EAAE,OAAe;IACvD,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE;QACrD,GAAG,EAAE,OAAO;QACZ,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,IAAI,CAAC,gBAAgB,GAAG,IAAI;QACrC,2EAA2E;QAC3E,4EAA4E;QAC5E,GAAG,EAAE,YAAY,EAAE;KACpB,CAAC,CAAC;IACH,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,KAAK,SAAS,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC;IACjE,OAAO;QACL,MAAM,EAAE,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,QAAQ;QACrC,QAAQ,EAAE,GAAG,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5C,QAAQ;QACR,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QACxB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;KACzB,CAAC;AACJ,CAAC;AAED,+DAA+D;AAC/D,MAAM,UAAU,QAAQ,CAAC,IAAc,EAAE,OAAe;IACtD,IAAI,CAAC,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAChC,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,EAAE;QACpD,GAAG,EAAE,OAAO;QACZ,QAAQ,EAAE,OAAO;QACjB,OAAO,EAAE,IAAI,CAAC,gBAAgB,GAAG,IAAI;QACrC,GAAG,EAAE,YAAY,EAAE;KACpB,CAAC,CAAC;IACH,OAAO;QACL,MAAM,EAAE,GAAG,CAAC,MAAM,KAAK,CAAC;QACxB,QAAQ,EAAE,GAAG,CAAC,MAAM,IAAI,CAAC;QACzB,QAAQ,EAAE,KAAK;QACf,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;QACxB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE;KACzB,CAAC;AACJ,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,YAAY;IAC1B,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC/B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,IAAI,GAAG,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}

package/dist/benchmarks/paired/types.d.ts

@@ -0,0 +1,195 @@
+/**
+ * Paired UAP Benchmark Harness — Shared Types
+ *
+ * Design goal (from the eval-methodology research): measure a *scaffold* layer
+ * (UAP) by holding the base model + base agent constant and toggling UAP on/off
+ * over the SAME task suite and seeds. The unit of measurement is not a single
+ * accuracy number but a *vector* of paired metrics (correctness, tokens, cost,
+ * turns, tool-calls, latency), reported with confidence intervals on the paired
+ * delta. See ./stats.ts for the statistics and ./report.ts for the framing.
+ *
+ * This file is the dependency-free core: pure type/Zod definitions reused by the
+ * adapter, runner, ablation, and report modules.
+ */
+import { z } from 'zod';
+/**
+ * A single real-gate benchmark task. Each task is a self-contained git fixture
+ * with a failing state and a deterministic `verify` command that is the ground
+ * truth: exit 0 => the agent resolved the task. No LLM judge.
+ */
+export declare const TaskSpecSchema: z.ZodObject<{
+    id: z.ZodString;
+    name: z.ZodString;
+    /** Natural-language instruction handed verbatim to the agent under test. */
+    instruction: z.ZodString;
+    difficulty: z.ZodDefault<z.ZodEnum<["easy", "medium", "hard"]>>;
+    /** Free-form tags for slicing results (e.g. 'bugfix', 'feature', 'refactor'). */
+    tags: z.ZodDefault<z.ZodArray<z.ZodString, "many">>;
+    /**
+     * Path (relative to the task directory) to the repo fixture that gets copied
+     * into an isolated scratch dir for each run. Defaults to 'repo'.
+     */
+    repoDir: z.ZodDefault<z.ZodString>;
+    /**
+     * Shell command (run inside the scratch repo) that returns exit 0 iff the
+     * task is resolved. This is the deterministic ground-truth scorer (HIDDEN —
+     * the agent never sees it).
+     */
+    verifyCmd: z.ZodString;
+    /**
+     * Optional VISIBLE in-repo gate command (e.g. `node test.js`) that a
+     * gate-enforcing agent runs to self-verify and iterate. Distinct from
+     * verifyCmd: this is what the UAP gate loop optimizes against; verifyCmd
+     * (a superset) remains the authoritative ground truth. Used by the raw
+     * single-shot-vs-gate-loop adapter to isolate gate value.
+     */
+    gateCmd: z.ZodOptional<z.ZodString>;
+    /** Optional setup command run once after the repo is copied, before the agent. */
+    setupCmd: z.ZodOptional<z.ZodString>;
+    /** Seconds before the verify command is killed and treated as failure. */
+    verifyTimeoutSec: z.ZodDefault<z.ZodNumber>;
+    /** Seconds before the agent run is killed and treated as failure. */
+    agentTimeoutSec: z.ZodDefault<z.ZodNumber>;
+}, "strip", z.ZodTypeAny, {
+    id: string;
+    name: string;
+    instruction: string;
+    tags: string[];
+    difficulty: "medium" | "easy" | "hard";
+    repoDir: string;
+    verifyCmd: string;
+    verifyTimeoutSec: number;
+    agentTimeoutSec: number;
+    gateCmd?: string | undefined;
+    setupCmd?: string | undefined;
+}, {
+    id: string;
+    name: string;
+    instruction: string;
+    verifyCmd: string;
+    tags?: string[] | undefined;
+    difficulty?: "medium" | "easy" | "hard" | undefined;
+    repoDir?: string | undefined;
+    gateCmd?: string | undefined;
+    setupCmd?: string | undefined;
+    verifyTimeoutSec?: number | undefined;
+    agentTimeoutSec?: number | undefined;
+}>;
+export type TaskSpec = z.infer<typeof TaskSpecSchema>;
+/**
+ * The toggleable UAP components, used by the ablation harness. Turning all of
+ * them off is equivalent to the bare-agent baseline; turning all on is full UAP.
+ */
+export declare const UAP_COMPONENTS: readonly ["gates", "worktree", "memory", "experts", "skills", "patterns"];
+export type UapComponent = (typeof UAP_COMPONENTS)[number];
+/**
+ * A condition is one arm of the experiment: which UAP components are active.
+ * `baseline` = all off; `full` = all on; ablations turn exactly one off.
+ */
+export interface Condition {
+    /** Stable label used in reports, e.g. 'baseline', 'uap-full', 'no-gates'. */
+    label: string;
+    /** Components enabled for this arm. */
+    components: ReadonlySet<UapComponent>;
+}
+export declare function makeBaselineCondition(): Condition;
+export declare function makeFullCondition(): Condition;
+/** True when this condition is the bare-agent baseline (no UAP at all). */
+export declare function isBaseline(c: Condition): boolean;
+export declare const MetricVectorSchema: z.ZodObject<{
+    /** Ground truth: did the verify command pass. */
+    correct: z.ZodBoolean;
+    /** Total tokens (prompt + completion) consumed by the agent, if known. */
+    tokens: z.ZodNullable<z.ZodNumber>;
+    /** Estimated USD cost, if a price could be applied. */
+    costUsd: z.ZodNullable<z.ZodNumber>;
+    /** Agent loop iterations / turns, if reported. */
+    turns: z.ZodNullable<z.ZodNumber>;
+    /** Tool calls made by the agent, if reported. */
+    toolCalls: z.ZodNullable<z.ZodNumber>;
+    /** Wall-clock latency of the agent run in milliseconds. */
+    latencyMs: z.ZodNumber;
+    /** Whether the agent produced a well-formed edit (format compliance), if known. */
+    wellFormed: z.ZodNullable<z.ZodBoolean>;
+    /** Non-fatal note or error message captured during the run. */
+    error: z.ZodNullable<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    error: string | null;
+    tokens: number | null;
+    latencyMs: number;
+    toolCalls: number | null;
+    turns: number | null;
+    correct: boolean;
+    costUsd: number | null;
+    wellFormed: boolean | null;
+}, {
+    error: string | null;
+    tokens: number | null;
+    latencyMs: number;
+    toolCalls: number | null;
+    turns: number | null;
+    correct: boolean;
+    costUsd: number | null;
+    wellFormed: boolean | null;
+}>;
+export type MetricVector = z.infer<typeof MetricVectorSchema>;
+/** The continuous (numeric) metric keys eligible for paired-delta analysis. */
+export declare const CONTINUOUS_METRICS: readonly ["tokens", "costUsd", "turns", "toolCalls", "latencyMs"];
+export type ContinuousMetric = (typeof CONTINUOUS_METRICS)[number];
+/** One execution of one task under one condition at one seed/epoch. */
+export interface RunRecord {
+    taskId: string;
+    condition: string;
+    /** Seed/epoch index — same value pairs runs across conditions for analysis. */
+    seed: number;
+    metrics: MetricVector;
+    /** Adapter that produced this run (e.g. 'mock', 'opencode', 'claude'). */
+    adapter: string;
+    /** Model identifier the adapter ran against. */
+    model: string;
+}
+/**
+ * Result returned by an AgentAdapter for a single run, before the verify
+ * command is applied. `correct` is filled in by the runner from the verify
+ * command, so adapters return everything *except* correctness.
+ */
+export interface AgentRunResult {
+    tokens: number | null;
+    costUsd: number | null;
+    turns: number | null;
+    toolCalls: number | null;
+    wellFormed: boolean | null;
+    error: string | null;
+    /** Raw stdout/log for debugging and post-hoc audit (HAL-style log inspection). */
+    rawLog?: string;
+}
+export interface AgentRunContext {
+    task: TaskSpec;
+    condition: Condition;
+    /** Absolute path to the isolated scratch repo the agent should operate on. */
+    workdir: string;
+    seed: number;
+    model: string;
+}
+export interface AgentAdapter {
+    /** Stable adapter id used in reports. */
+    readonly id: string;
+    /** Drive the agent over `ctx.workdir`. Must not throw on agent failure — */
+    /** capture it in `error` and return. May throw only on harness misconfig.  */
+    run(ctx: AgentRunContext): Promise<AgentRunResult>;
+}
+export interface RunnerConfig {
+    tasks: TaskSpec[];
+    conditions: Condition[];
+    adapter: AgentAdapter;
+    model: string;
+    /** Number of paired seeds/epochs per (task, condition). Research: >=5. */
+    epochs: number;
+    /** Max concurrent runs. */
+    concurrency: number;
+    /** Directory for scratch repos + artifacts. */
+    workRoot: string;
+    /** Optional progress callback. */
+    onProgress?: (done: number, total: number, label: string) => void;
+}
+//# sourceMappingURL=types.d.ts.map

package/dist/benchmarks/paired/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/benchmarks/paired/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAMxB;;;;GAIG;AACH,eAAO,MAAM,cAAc;;;IAGzB,4EAA4E;;;IAG5E,iFAAiF;;IAEjF;;;OAGG;;IAEH;;;;OAIG;;IAEH;;;;;;OAMG;;IAEH,kFAAkF;;IAElF,0EAA0E;;IAE1E,qEAAqE;;;;;;;;;;;;;;;;;;;;;;;;;;EAErE,CAAC;AAEH,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAMtD;;;GAGG;AACH,eAAO,MAAM,cAAc,2EAOjB,CAAC;AAEX,MAAM,MAAM,YAAY,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAE3D;;;GAGG;AACH,MAAM,WAAW,SAAS;IACxB,6EAA6E;IAC7E,KAAK,EAAE,MAAM,CAAC;IACd,uCAAuC;IACvC,UAAU,EAAE,WAAW,CAAC,YAAY,CAAC,CAAC;CACvC;AAED,wBAAgB,qBAAqB,IAAI,SAAS,CAEjD;AAED,wBAAgB,iBAAiB,IAAI,SAAS,CAE7C;AAED,2EAA2E;AAC3E,wBAAgB,UAAU,CAAC,CAAC,EAAE,SAAS,GAAG,OAAO,CAEhD;AAMD,eAAO,MAAM,kBAAkB;IAC7B,iDAAiD;;IAEjD,0EAA0E;;IAE1E,uDAAuD;;IAEvD,kDAAkD;;IAElD,iDAAiD;;IAEjD,2DAA2D;;IAE3D,mFAAmF;;IAEnF,+DAA+D;;;;;;;;;;;;;;;;;;;;EAE/D,CAAC;AAEH,MAAM,MAAM,YAAY,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,kBAAkB,CAAC,CAAC;AAE9D,+EAA+E;AAC/E,eAAO,MAAM,kBAAkB,mEAMrB,CAAC;AACX,MAAM,MAAM,gBAAgB,GAAG,CAAC,OAAO,kBAAkB,CAAC,CAAC,MAAM,CAAC,CAAC;AAMnE,uEAAuE;AACvE,MAAM,WAAW,SAAS;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,YAAY,CAAC;IACtB,0EAA0E;IAC1E,OAAO,EAAE,MAAM,CAAC;IAChB,gDAAgD;IAChD,KAAK,EAAE,MAAM,CAAC;CACf;AAMD;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,UAAU,EAAE,OAAO,GAAG,IAAI,CAAC;IAC3B,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,kFAAkF;IAClF,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,QAAQ,CAAC;IACf,SAAS,EAAE,SAAS,CAAC;IACrB,8EAA8E;IAC9E,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,YAAY;IAC3B,yCAAyC;IACzC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,4EAA4E;IAC5E,8EAA8E;IAC9E,GAAG,CAAC,GAAG,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CACpD;AAMD,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,QAAQ,EAAE,CAAC;IAClB,UAAU,EAAE,SAAS,EAAE,CAAC;IACxB,OAAO,EAAE,YAAY,CAAC;IACtB,KAAK,EAAE,MAAM,CAAC;IACd,0EAA0E;IAC1E,MAAM,EAAE,MAAM,CAAC;IACf,2BAA2B;IAC3B,WAAW,EAAE,MAAM,CAAC;IACpB,+CAA+C;IAC/C,QAAQ,EAAE,MAAM,CAAC;IACjB,kCAAkC;IAClC,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;CACnE"}