cognitive-core 0.2.5 → 0.3.1

package/dist/eval/live/stopping.d.ts

@@ -0,0 +1,69 @@
+import type { TaskResult } from '../types.js';
+/** Pooled outcome counts for one arm (env_error episodes excluded). */
+export interface ArmCounts {
+    successes: number;
+    failures: number;
+}
+/** Snapshot of a run's evidence at a decision point. */
+export interface StoppingState {
+    learning: ArmCounts;
+    cold: ArmCounts;
+    /** Episodes per arm still in the budget (e.g. remaining seeds × taskCount). */
+    remainingEpisodesPerArm: number;
+}
+export interface StoppingConfig {
+    /**
+     * Posterior probability of a one-sided conclusion required to stop early.
+     * P(delta>0) ≥ threshold → learning superior; ≤ 1-threshold → harmful.
+     * Conservative default (0.98) compensates for seed-boundary peeking.
+     */
+    decisionThreshold?: number;
+    /**
+     * Futility (stochastic curtailment): stop when the probability that the
+     * run reaches ANY decisive conclusion within the remaining budget —
+     * estimated by simulating remaining episodes from the posterior
+     * predictive — falls below this. Default 0.05.
+     */
+    futilityEpsilon?: number;
+    /**
+     * Stop when the 95% credible interval on delta is narrower than this AND
+     * the sign question is settled enough to act on. 0 disables (default).
+     */
+    targetCiWidth?: number;
+    /** Monte Carlo samples per evaluation (default 8000). */
+    mcSamples?: number;
+    /** RNG seed for deterministic decisions (default 1). */
+    seed?: number;
+}
+export type StoppingRule = 'threshold-superior' | 'threshold-harmful' | 'futility' | 'precision';
+export interface StoppingDecision {
+    decision: 'continue' | 'stop';
+    rule?: StoppingRule;
+    /** P(learning success rate > cold success rate | data). */
+    probDeltaPositive: number;
+    /** 95% central credible interval on delta = pLearning - pCold. */
+    deltaCi: [number, number];
+    reason: string;
+}
+/**
+ * Fast normal approximation of P(pA > pB) for Beta(aA,bA) vs Beta(aB,bB)
+ * posteriors — used inside the futility simulation where an exact nested
+ * Monte Carlo would be quadratic. Exported for accuracy validation against
+ * the exact MC estimate (tests/eval/stopping-validation.test.ts).
+ */
+export declare function probGreaterNormalApprox(aA: number, bA: number, aB: number, bB: number): number;
+/** Posterior summary of delta = pLearning - pCold under Beta(1,1) priors. */
+export declare function posteriorDelta(learning: ArmCounts, cold: ArmCounts, mcSamples: number, seed: number): {
+    probPositive: number;
+    ci: [number, number];
+};
+/**
+ * Evaluate the stopping rules against the current evidence.
+ *
+ * Rule order matters: a settled answer (threshold) beats futility, and both
+ * beat precision. `continue` is returned only when none fire.
+ */
+export declare function evaluateStopping(state: StoppingState, config?: StoppingConfig): StoppingDecision;
+/** Pool per-seed task rows into arm counts, excluding env_error episodes. */
+export declare function poolArmCounts(perSeed: TaskResult[][]): ArmCounts;
+//# sourceMappingURL=stopping.d.ts.map

package/dist/eval/live/stopping.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stopping.d.ts","sourceRoot":"","sources":["../../../src/eval/live/stopping.ts"],"names":[],"mappings":"AAwBA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAE9C,uEAAuE;AACvE,MAAM,WAAW,SAAS;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,wDAAwD;AACxD,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,SAAS,CAAC;IACpB,IAAI,EAAE,SAAS,CAAC;IAChB,+EAA+E;IAC/E,uBAAuB,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B;;;;;OAKG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,yDAAyD;IACzD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,wDAAwD;IACxD,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,MAAM,YAAY,GAAG,oBAAoB,GAAG,mBAAmB,GAAG,UAAU,GAAG,WAAW,CAAC;AAEjG,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,UAAU,GAAG,MAAM,CAAC;IAC9B,IAAI,CAAC,EAAE,YAAY,CAAC;IACpB,2DAA2D;IAC3D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kEAAkE;IAClE,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC1B,MAAM,EAAE,MAAM,CAAC;CAChB;AAmBD;;;;;GAKG;AACH,wBAAgB,uBAAuB,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,GAAG,MAAM,CAM9F;AAsCD,6EAA6E;AAC7E,wBAAgB,cAAc,CAC5B,QAAQ,EAAE,SAAS,EACnB,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,GACX;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAAE,CAehD;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,aAAa,EACpB,MAAM,GAAE,cAAmB,GAC1B,gBAAgB,CA+FlB;AAED,6EAA6E;AAC7E,wBAAgB,aAAa,CAAC,OAAO,EAAE,UAAU,EAAE,EAAE,GAAG,SAAS,CAWhE"}

package/dist/eval/live/stopping.js

@@ -0,0 +1,213 @@
+/**
+ * Sequential stopping controller for live evals (Phase B of the adaptive
+ * eval-economics roadmap — see DESIGN-meta-learning-validation.md §10).
+ *
+ * A multi-seed two-arm run commits to seeds × tasks × 2 episodes up front,
+ * but the answer often arrives earlier: after one seed the learning arm may
+ * already be decisively ahead (stop, declare), decisively behind (stop,
+ * declare), or the remaining budget may be provably unable to resolve the
+ * question (stop, redesign). Each avoided seed saves ~16 real-agent episodes.
+ *
+ * Statistical machinery (deliberately simple, biased conservative):
+ *   - Per-arm Beta(1,1)-binomial posteriors over success rates, pooled across
+ *     completed episodes (env_error rows excluded, matching the aggregates).
+ *   - Monte Carlo posterior sampling (seeded, deterministic) for
+ *     P(delta > 0) and the central credible interval of delta.
+ *   - Rules follow the taxonomy in "Stop Guessing When to Stop Testing"
+ *     (ACL 2026 Findings): threshold-crossing, futility, precision.
+ *
+ * Anytime-validity caveat: repeated peeking inflates error rates. We only
+ * check at seed boundaries (≤ seeds-1 looks, typically 2) and use a
+ * conservative default threshold (0.98) rather than full e-process machinery.
+ * Decisions are logged with the posterior so a human can always overrule.
+ */
+import { seededRng } from '../rng.js';
+const DEFAULTS = {
+    decisionThreshold: 0.98,
+    futilityEpsilon: 0.05,
+    targetCiWidth: 0,
+    mcSamples: 8000,
+    seed: 1,
+};
+/** Standard normal CDF via the Abramowitz-Stegun erf approximation. */
+function normalCdf(z) {
+    const t = 1 / (1 + 0.2316419 * Math.abs(z));
+    const d = 0.3989423 * Math.exp((-z * z) / 2);
+    const p = d * t * (0.3193815 + t * (-0.3565638 + t * (1.781478 + t * (-1.821256 + t * 1.330274))));
+    return z > 0 ? 1 - p : p;
+}
+/**
+ * Fast normal approximation of P(pA > pB) for Beta(aA,bA) vs Beta(aB,bB)
+ * posteriors — used inside the futility simulation where an exact nested
+ * Monte Carlo would be quadratic. Exported for accuracy validation against
+ * the exact MC estimate (tests/eval/stopping-validation.test.ts).
+ */
+export function probGreaterNormalApprox(aA, bA, aB, bB) {
+    const mA = aA / (aA + bA);
+    const mB = aB / (aB + bB);
+    const vA = (mA * (1 - mA)) / (aA + bA + 1);
+    const vB = (mB * (1 - mB)) / (aB + bB + 1);
+    return normalCdf((mA - mB) / Math.sqrt(vA + vB));
+}
+function sampleBinomial(n, p, rng) {
+    let k = 0;
+    for (let i = 0; i < n; i++)
+        if (rng() < p)
+            k++;
+    return k;
+}
+/**
+ * Marsaglia-Tsang gamma sampler (shape k ≥ 1), with the standard U^(1/k)
+ * boost for k < 1. Normal deviates via Box-Muller from the seeded uniform.
+ */
+function sampleGamma(k, rng) {
+    if (k < 1) {
+        const u = Math.max(rng(), Number.MIN_VALUE);
+        return sampleGamma(k + 1, rng) * Math.pow(u, 1 / k);
+    }
+    const d = k - 1 / 3;
+    const c = 1 / Math.sqrt(9 * d);
+    for (;;) {
+        const u1 = Math.max(rng(), Number.MIN_VALUE);
+        const u2 = rng();
+        const normal = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
+        const v = Math.pow(1 + c * normal, 3);
+        if (v <= 0)
+            continue;
+        const u = Math.max(rng(), Number.MIN_VALUE);
+        if (Math.log(u) < 0.5 * normal * normal + d - d * v + d * Math.log(v)) {
+            return d * v;
+        }
+    }
+}
+function sampleBeta(alpha, beta, rng) {
+    const x = sampleGamma(alpha, rng);
+    const y = sampleGamma(beta, rng);
+    return x / (x + y);
+}
+/** Posterior summary of delta = pLearning - pCold under Beta(1,1) priors. */
+export function posteriorDelta(learning, cold, mcSamples, seed) {
+    const rng = seededRng('stopping-controller', seed);
+    const deltas = new Array(mcSamples);
+    let positive = 0;
+    for (let i = 0; i < mcSamples; i++) {
+        const pl = sampleBeta(learning.successes + 1, learning.failures + 1, rng);
+        const pc = sampleBeta(cold.successes + 1, cold.failures + 1, rng);
+        const d = pl - pc;
+        deltas[i] = d;
+        if (d > 0)
+            positive++;
+    }
+    deltas.sort((a, b) => a - b);
+    const lo = deltas[Math.floor(0.025 * mcSamples)];
+    const hi = deltas[Math.min(mcSamples - 1, Math.floor(0.975 * mcSamples))];
+    return { probPositive: positive / mcSamples, ci: [lo, hi] };
+}
+/**
+ * Evaluate the stopping rules against the current evidence.
+ *
+ * Rule order matters: a settled answer (threshold) beats futility, and both
+ * beat precision. `continue` is returned only when none fire.
+ */
+export function evaluateStopping(state, config = {}) {
+    const cfg = { ...DEFAULTS, ...config };
+    const completed = state.learning.successes + state.learning.failures + state.cold.successes + state.cold.failures;
+    if (completed === 0) {
+        return {
+            decision: 'continue',
+            probDeltaPositive: 0.5,
+            deltaCi: [-1, 1],
+            reason: 'no completed episodes yet',
+        };
+    }
+    const { probPositive, ci } = posteriorDelta(state.learning, state.cold, cfg.mcSamples, cfg.seed);
+    const fmt = (p) => p.toFixed(3);
+    const base = `P(delta>0)=${fmt(probPositive)}, 95% CI [${fmt(ci[0])}, ${fmt(ci[1])}]`;
+    // Threshold-crossing: the sign question is settled — more episodes are spend
+    // without information.
+    if (probPositive >= cfg.decisionThreshold) {
+        return {
+            decision: 'stop',
+            rule: 'threshold-superior',
+            probDeltaPositive: probPositive,
+            deltaCi: ci,
+            reason: `${base} ≥ ${cfg.decisionThreshold} — learning arm superior; further episodes uninformative`,
+        };
+    }
+    if (probPositive <= 1 - cfg.decisionThreshold) {
+        return {
+            decision: 'stop',
+            rule: 'threshold-harmful',
+            probDeltaPositive: probPositive,
+            deltaCi: ci,
+            reason: `${base} ≤ ${fmt(1 - cfg.decisionThreshold)} — learning arm not helping; further episodes uninformative`,
+        };
+    }
+    // Futility (stochastic curtailment): simulate the remaining episodes from
+    // the posterior predictive and measure how often the completed run would
+    // reach a decisive conclusion in either direction. When that conditional
+    // power is below futilityEpsilon, the remaining budget almost surely buys
+    // no answer — stop and redesign instead of spending it. This is the rule
+    // that fires when both arms are floored/saturated (e.g. EDR-8 2026-07-02:
+    // learning 0/8 vs cold 1/8 — no plausible continuation separates them).
+    if (state.remainingEpisodesPerArm > 0) {
+        const n = state.remainingEpisodesPerArm;
+        const rng = seededRng('stopping-futility', cfg.seed);
+        const sims = Math.max(500, Math.floor(cfg.mcSamples / 4));
+        let decisive = 0;
+        for (let i = 0; i < sims; i++) {
+            const pl = sampleBeta(state.learning.successes + 1, state.learning.failures + 1, rng);
+            const pc = sampleBeta(state.cold.successes + 1, state.cold.failures + 1, rng);
+            const kl = sampleBinomial(n, pl, rng);
+            const kc = sampleBinomial(n, pc, rng);
+            const pFinal = probGreaterNormalApprox(state.learning.successes + kl + 1, state.learning.failures + (n - kl) + 1, state.cold.successes + kc + 1, state.cold.failures + (n - kc) + 1);
+            if (pFinal >= cfg.decisionThreshold || pFinal <= 1 - cfg.decisionThreshold)
+                decisive++;
+        }
+        const conditionalPower = decisive / sims;
+        if (conditionalPower < cfg.futilityEpsilon) {
+            return {
+                decision: 'stop',
+                rule: 'futility',
+                probDeltaPositive: probPositive,
+                deltaCi: ci,
+                reason: `${base}; P(decisive conclusion within remaining ${n} episodes/arm)=` +
+                    `${fmt(conditionalPower)} < ${cfg.futilityEpsilon} — ` +
+                    `budget cannot resolve the question; stop and redesign`,
+            };
+        }
+    }
+    // Precision: the estimate is already as tight as we asked for.
+    if (cfg.targetCiWidth > 0 && ci[1] - ci[0] <= cfg.targetCiWidth) {
+        return {
+            decision: 'stop',
+            rule: 'precision',
+            probDeltaPositive: probPositive,
+            deltaCi: ci,
+            reason: `${base}; CI width ${fmt(ci[1] - ci[0])} ≤ target ${cfg.targetCiWidth}`,
+        };
+    }
+    return {
+        decision: 'continue',
+        probDeltaPositive: probPositive,
+        deltaCi: ci,
+        reason: `${base} — undecided and remaining budget can still resolve it`,
+    };
+}
+/** Pool per-seed task rows into arm counts, excluding env_error episodes. */
+export function poolArmCounts(perSeed) {
+    let successes = 0;
+    let failures = 0;
+    for (const rows of perSeed) {
+        for (const r of rows) {
+            if (r.status === 'env_error')
+                continue;
+            if (r.success)
+                successes++;
+            else
+                failures++;
+        }
+    }
+    return { successes, failures };
+}
+//# sourceMappingURL=stopping.js.map

package/dist/eval/live/stopping.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stopping.js","sourceRoot":"","sources":["../../../src/eval/live/stopping.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAsDtC,MAAM,QAAQ,GAA6B;IACzC,iBAAiB,EAAE,IAAI;IACvB,eAAe,EAAE,IAAI;IACrB,aAAa,EAAE,CAAC;IAChB,SAAS,EAAE,IAAI;IACf,IAAI,EAAE,CAAC;CACR,CAAC;AAEF,uEAAuE;AACvE,SAAS,SAAS,CAAC,CAAS;IAC1B,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5C,MAAM,CAAC,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7C,MAAM,CAAC,GACL,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,GAAG,CAAC,GAAG,CAAC,CAAC,SAAS,GAAG,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3F,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3B,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,uBAAuB,CAAC,EAAU,EAAE,EAAU,EAAE,EAAU,EAAE,EAAU;IACpF,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;IAC1B,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;IAC1B,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAC3C,MAAM,EAAE,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC;IAC3C,OAAO,SAAS,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,cAAc,CAAC,CAAS,EAAE,CAAS,EAAE,GAAiB;IAC7D,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;QAAE,IAAI,GAAG,EAAE,GAAG,CAAC;YAAE,CAAC,EAAE,CAAC;IAC/C,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,CAAS,EAAE,GAAiB;IAC/C,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAC5C,OAAO,WAAW,CAAC,CAAC,GAAG,CAAC,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACtD,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACpB,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/B,SAAS,CAAC;QACR,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAC7C,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;QACjB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;QACzE,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,EAAE,CAAC,CAAC,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QACrB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;QAC5C,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,MAAM,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,OAAO,CAAC,GAAG,CAAC,CAAC;QACf,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,KAAa,EAAE,IAAY,EAAE,GAAiB;IAChE,MAAM,CAAC,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;IAClC,MAAM,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACjC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AACrB,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,cAAc,CAC5B,QAAmB,EACnB,IAAe,EACf,SAAiB,EACjB,IAAY;IAEZ,MAAM,GAAG,GAAG,SAAS,CAAC,qBAAqB,EAAE,IAAI,CAAC,CAAC;IACnD,MAAM,MAAM,GAAG,IAAI,KAAK,CAAS,SAAS,CAAC,CAAC;IAC5C,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,MAAM,EAAE,GAAG,UAAU,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,EAAE,QAAQ,CAAC,QAAQ,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;QAC1E,MAAM,EAAE,GAAG,UAAU,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,IAAI,CAAC,QAAQ,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;QAClE,MAAM,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;QAClB,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QACd,IAAI,CAAC,GAAG,CAAC;YAAE,QAAQ,EAAE,CAAC;IACxB,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC7B,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC;IACjD,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,GAAG,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;IAC1E,OAAO,EAAE,YAAY,EAAE,QAAQ,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC;AAC9D,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAC9B,KAAoB,EACpB,SAAyB,EAAE;IAE3B,MAAM,GAAG,GAAG,EAAE,GAAG,QAAQ,EAAE,GAAG,MAAM,EAAE,CAAC;IACvC,MAAM,SAAS,GACb,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,KAAK,CAAC,QAAQ,CAAC,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC;IAClG,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO;YACL,QAAQ,EAAE,UAAU;YACpB,iBAAiB,EAAE,GAAG;YACtB,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChB,MAAM,EAAE,2BAA2B;SACpC,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,YAAY,EAAE,EAAE,EAAE,GAAG,cAAc,CAAC,KAAK,CAAC,QAAQ,EAAE,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;IACjG,MAAM,GAAG,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,cAAc,GAAG,CAAC,YAAY,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;IAEtF,6EAA6E;IAC7E,uBAAuB;IACvB,IAAI,YAAY,IAAI,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAC1C,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,oBAAoB;YAC1B,iBAAiB,EAAE,YAAY;YAC/B,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,GAAG,IAAI,MAAM,GAAG,CAAC,iBAAiB,0DAA0D;SACrG,CAAC;IACJ,CAAC;IACD,IAAI,YAAY,IAAI,CAAC,GAAG,GAAG,CAAC,iBAAiB,EAAE,CAAC;QAC9C,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,mBAAmB;YACzB,iBAAiB,EAAE,YAAY;YAC/B,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,GAAG,IAAI,MAAM,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,iBAAiB,CAAC,6DAA6D;SACjH,CAAC;IACJ,CAAC;IAED,0EAA0E;IAC1E,yEAAyE;IACzE,yEAAyE;IACzE,0EAA0E;IAC1E,yEAAyE;IACzE,0EAA0E;IAC1E,wEAAwE;IACxE,IAAI,KAAK,CAAC,uBAAuB,GAAG,CAAC,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,KAAK,CAAC,uBAAuB,CAAC;QACxC,MAAM,GAAG,GAAG,SAAS,CAAC,mBAAmB,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1D,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YAC9B,MAAM,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;YACtF,MAAM,EAAE,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC;YAC9E,MAAM,EAAE,GAAG,cAAc,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACtC,MAAM,EAAE,GAAG,cAAc,CAAC,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC;YACtC,MAAM,MAAM,GAAG,uBAAuB,CACpC,KAAK,CAAC,QAAQ,CAAC,SAAS,GAAG,EAAE,GAAG,CAAC,EACjC,KAAK,CAAC,QAAQ,CAAC,QAAQ,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,EACtC,KAAK,CAAC,IAAI,CAAC,SAAS,GAAG,EAAE,GAAG,CAAC,EAC7B,KAAK,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CACnC,CAAC;YACF,IAAI,MAAM,IAAI,GAAG,CAAC,iBAAiB,IAAI,MAAM,IAAI,CAAC,GAAG,GAAG,CAAC,iBAAiB;gBAAE,QAAQ,EAAE,CAAC;QACzF,CAAC;QACD,MAAM,gBAAgB,GAAG,QAAQ,GAAG,IAAI,CAAC;QACzC,IAAI,gBAAgB,GAAG,GAAG,CAAC,eAAe,EAAE,CAAC;YAC3C,OAAO;gBACL,QAAQ,EAAE,MAAM;gBAChB,IAAI,EAAE,UAAU;gBAChB,iBAAiB,EAAE,YAAY;gBAC/B,OAAO,EAAE,EAAE;gBACX,MAAM,EACJ,GAAG,IAAI,4CAA4C,CAAC,iBAAiB;oBACrE,GAAG,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,eAAe,KAAK;oBACtD,uDAAuD;aAC1D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,+DAA+D;IAC/D,IAAI,GAAG,CAAC,aAAa,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,aAAa,EAAE,CAAC;QAChE,OAAO;YACL,QAAQ,EAAE,MAAM;YAChB,IAAI,EAAE,WAAW;YACjB,iBAAiB,EAAE,YAAY;YAC/B,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,GAAG,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,aAAa,EAAE;SAChF,CAAC;IACJ,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,UAAU;QACpB,iBAAiB,EAAE,YAAY;QAC/B,OAAO,EAAE,EAAE;QACX,MAAM,EAAE,GAAG,IAAI,wDAAwD;KACxE,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,MAAM,UAAU,aAAa,CAAC,OAAuB;IACnD,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,IAAI,IAAI,OAAO,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,IAAI,CAAC,CAAC,MAAM,KAAK,WAAW;gBAAE,SAAS;YACvC,IAAI,CAAC,CAAC,OAAO;gBAAE,SAAS,EAAE,CAAC;;gBACtB,QAAQ,EAAE,CAAC;QAClB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AACjC,CAAC"}

package/dist/eval/live/trace.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Shared trace sink for live eval benchmarks.
+ *
+ * Benchmarks emit structured JSONL entries to a per-task trace file so
+ * post-run analysis (eval report --diagnostics) can read them back.
+ */
+/**
+ * A function that appends one JSONL entry to a task's trace file.
+ * Best-effort: errors are swallowed so tracing never breaks the eval.
+ */
+export type TraceWriter = (entry: Record<string, unknown>) => void;
+/**
+ * Create a TraceWriter bound to a specific (dir, taskId) pair.
+ *
+ * @param dir  Directory where trace files live. If undefined, returns a no-op.
+ * @param taskId  Logical task identifier; sanitized for filesystem safety.
+ * @returns A function that appends `{ ...entry, timestamp }` as a JSONL line.
+ */
+export declare function createTraceWriter(dir: string | undefined, taskId: string): TraceWriter;
+//# sourceMappingURL=trace.d.ts.map

package/dist/eval/live/trace.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trace.d.ts","sourceRoot":"","sources":["../../../src/eval/live/trace.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAKH;;;GAGG;AACH,MAAM,MAAM,WAAW,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,IAAI,CAAC;AAEnE;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,MAAM,GAAG,WAAW,CA2BtF"}

package/dist/eval/live/trace.js

@@ -0,0 +1,41 @@
+/**
+ * Shared trace sink for live eval benchmarks.
+ *
+ * Benchmarks emit structured JSONL entries to a per-task trace file so
+ * post-run analysis (eval report --diagnostics) can read them back.
+ */
+import { appendFileSync, mkdirSync } from 'node:fs';
+import { join } from 'node:path';
+/**
+ * Create a TraceWriter bound to a specific (dir, taskId) pair.
+ *
+ * @param dir  Directory where trace files live. If undefined, returns a no-op.
+ * @param taskId  Logical task identifier; sanitized for filesystem safety.
+ * @returns A function that appends `{ ...entry, timestamp }` as a JSONL line.
+ */
+export function createTraceWriter(dir, taskId) {
+    if (!dir) {
+        return (_entry) => {
+            /* no-op when dir undefined */
+        };
+    }
+    // Sanitize: keep alphanumeric and hyphens only.
+    const sanitized = taskId.replace(/[^a-z0-9-]/g, '-');
+    const filePath = join(dir, `${sanitized}.jsonl`);
+    // Ensure the directory exists before the first write.
+    try {
+        mkdirSync(dir, { recursive: true });
+    }
+    catch {
+        // best-effort
+    }
+    return (entry) => {
+        try {
+            appendFileSync(filePath, JSON.stringify({ ...entry, timestamp: new Date().toISOString() }) + '\n');
+        }
+        catch {
+            // best-effort trace writing — never throws into the caller
+        }
+    };
+}
+//# sourceMappingURL=trace.js.map

package/dist/eval/live/trace.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"trace.js","sourceRoot":"","sources":["../../../src/eval/live/trace.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAQjC;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAuB,EAAE,MAAc;IACvE,IAAI,CAAC,GAAG,EAAE,CAAC;QACT,OAAO,CAAC,MAA+B,EAAE,EAAE;YACzC,8BAA8B;QAChC,CAAC,CAAC;IACJ,CAAC;IACD,gDAAgD;IAChD,MAAM,SAAS,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC,CAAC;IACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,QAAQ,CAAC,CAAC;IAEjD,sDAAsD;IACtD,IAAI,CAAC;QACH,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtC,CAAC;IAAC,MAAM,CAAC;QACP,cAAc;IAChB,CAAC;IAED,OAAO,CAAC,KAA8B,EAAQ,EAAE;QAC9C,IAAI,CAAC;YACH,cAAc,CACZ,QAAQ,EACR,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,KAAK,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,GAAG,IAAI,CACzE,CAAC;QACJ,CAAC;QAAC,MAAM,CAAC;YACP,2DAA2D;QAC7D,CAAC;IACH,CAAC,CAAC;AACJ,CAAC"}

package/dist/eval/mechanism-trace-adapter.d.ts

@@ -0,0 +1,8 @@
+import type { TaskDiagnostics } from './diagnostics.js';
+import { type MechanismTraceEvent } from '../learning/mechanism-trace.js';
+export interface DiagnosticsMechanismTraceContext {
+    benchmark?: string;
+    source?: string;
+}
+export declare function diagnosticsToMechanismTraceEvents(diag: TaskDiagnostics, context?: DiagnosticsMechanismTraceContext): MechanismTraceEvent[];
+//# sourceMappingURL=mechanism-trace-adapter.d.ts.map

package/dist/eval/mechanism-trace-adapter.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"mechanism-trace-adapter.d.ts","sourceRoot":"","sources":["../../src/eval/mechanism-trace-adapter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACxD,OAAO,EAIL,KAAK,mBAAmB,EAIzB,MAAM,gCAAgC,CAAC;AAExC,MAAM,WAAW,gCAAgC;IAC/C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,wBAAgB,iCAAiC,CAC/C,IAAI,EAAE,eAAe,EACrB,OAAO,GAAE,gCAAqC,GAC7C,mBAAmB,EAAE,CAgEvB"}

package/dist/eval/mechanism-trace-adapter.js

@@ -0,0 +1,118 @@
+import { createMechanismTraceEvent, } from '../learning/mechanism-trace.js';
+export function diagnosticsToMechanismTraceEvents(diag, context = {}) {
+    const base = {
+        benchmark: context.benchmark,
+        arm: diag.arm,
+        taskId: diag.taskId,
+        source: context.source ?? 'eval.diagnostics',
+    };
+    const retrievalCandidates = buildRetrievalCandidates(diag);
+    const retrieval = createMechanismTraceEvent({
+        ...base,
+        family: 'retrieval',
+        query: diag.retrieval.query,
+        candidates: retrievalCandidates,
+        filters: {
+            threshold: diag.retrieval.threshold,
+            playbookExcludedByScoreFloor: diag.retrieval.playbookExcludedByScoreFloor,
+            playbookExcludedByApplicability: diag.retrieval.playbookExcludedByApplicability,
+            playbookApplicabilityGateEnabled: diag.retrieval.playbookApplicabilityGateEnabled,
+        },
+        fallback: {
+            used: diag.retrieval.playbookDomainFallbackUsed ?? false,
+        },
+    });
+    const injectedPlaybooks = (diag.injection.playbooks ?? []).map((playbook) => ({
+        id: playbook.id,
+        type: 'playbook',
+        score: playbook.score,
+        incorporationMode: playbook.applicabilityDecision,
+    }));
+    const injectedExperiences = diag.retrieval.candidates
+        .filter((candidate) => candidate.type === 'experience')
+        .slice(0, diag.injection.experienceCount)
+        .map((candidate) => ({
+        id: candidate.id,
+        type: 'experience',
+        score: candidate.score,
+    }));
+    const serve = createMechanismTraceEvent({
+        ...base,
+        family: 'serve',
+        injected: [...injectedExperiences, ...injectedPlaybooks],
+        tokenFootprint: diag.injection.blob.length,
+        applicability: {
+            gateEnabled: diag.retrieval.playbookApplicabilityGateEnabled,
+            conditionsSurfaced: (diag.injection.playbooks ?? []).some((playbook) => playbook.applicabilityDecision !== undefined),
+        },
+    });
+    const outcome = createMechanismTraceEvent({
+        ...base,
+        family: 'outcome',
+        success: diag.outcome.success,
+        status: diag.outcome.status,
+        costUsd: diag.outcome.costUsd,
+        latencyMs: diag.outcome.wallTimeMs,
+    });
+    return [retrieval, serve, outcome];
+}
+function buildRetrievalCandidates(diag) {
+    const candidates = [];
+    const decisionsById = new Map((diag.retrieval.candidateDecisions ?? []).map((decision) => [decision.playbookId, decision]));
+    const usedDecisionIds = new Set();
+    for (const [index, candidate] of diag.retrieval.candidates.entries()) {
+        const decision = candidate.type === 'playbook' ? decisionsById.get(candidate.id) : undefined;
+        if (decision) {
+            usedDecisionIds.add(decision.playbookId);
+            candidates.push(decisionToMechanismCandidate(decision));
+            continue;
+        }
+        candidates.push({
+            id: candidate.id,
+            type: toArtifactType(candidate.type),
+            score: candidate.score,
+            rank: index,
+            retrievalMethod: candidate.retrievalMethod,
+            decision: candidate.applicabilityDecision === 'abstain' ? 'abstain' : undefined,
+            reason: candidate.applicabilityReason,
+            domainFallbackUsed: diag.retrieval.playbookDomainFallbackUsed,
+        });
+    }
+    const remainingDecisions = (diag.retrieval.candidateDecisions ?? [])
+        .filter((decision) => !usedDecisionIds.has(decision.playbookId))
+        .sort((a, b) => a.candidateRank - b.candidateRank);
+    for (const decision of remainingDecisions) {
+        candidates.push(decisionToMechanismCandidate(decision));
+    }
+    return candidates;
+}
+function decisionToMechanismCandidate(decision) {
+    return {
+        id: decision.playbookId,
+        type: 'playbook',
+        score: decision.score,
+        rank: decision.candidateRank,
+        retrievalMethod: decision.retrievalMethod,
+        decision: decision.scoreFloorPassed ? decision.decision : 'filter',
+        reason: decision.scoreFloorPassed ? decision.reason : `score below floor: ${decision.reason}`,
+        domainFallbackUsed: decision.domainFallbackUsed,
+        scoreFloorPassed: decision.scoreFloorPassed,
+        gateEnforced: decision.gateEnforced,
+        injected: decision.injected,
+        rawCandidateDecision: { ...decision },
+    };
+}
+function toArtifactType(type) {
+    switch (type) {
+        case 'experience':
+        case 'knowledge':
+        case 'playbook':
+        case 'metaPlaybook':
+        case 'shortTerm':
+        case 'frontier':
+            return type;
+        default:
+            return 'diagnostic';
+    }
+}
+//# sourceMappingURL=mechanism-trace-adapter.js.map

package/dist/eval/mechanism-trace-adapter.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mechanism-trace-adapter.js","sourceRoot":"","sources":["../../src/eval/mechanism-trace-adapter.ts"],"names":[],"mappings":"AACA,OAAO,EACL,yBAAyB,GAO1B,MAAM,gCAAgC,CAAC;AAOxC,MAAM,UAAU,iCAAiC,CAC/C,IAAqB,EACrB,UAA4C,EAAE;IAE9C,MAAM,IAAI,GAAG;QACX,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,kBAAkB;KAC7C,CAAC;IAEF,MAAM,mBAAmB,GAAG,wBAAwB,CAAC,IAAI,CAAC,CAAC;IAE3D,MAAM,SAAS,GAAG,yBAAyB,CAA+B;QACxE,GAAG,IAAI;QACP,MAAM,EAAE,WAAW;QACnB,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK;QAC3B,UAAU,EAAE,mBAAmB;QAC/B,OAAO,EAAE;YACP,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS;YACnC,4BAA4B,EAAE,IAAI,CAAC,SAAS,CAAC,4BAA4B;YACzE,+BAA+B,EAAE,IAAI,CAAC,SAAS,CAAC,+BAA+B;YAC/E,gCAAgC,EAAE,IAAI,CAAC,SAAS,CAAC,gCAAgC;SAClF;QACD,QAAQ,EAAE;YACR,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,0BAA0B,IAAI,KAAK;SACzD;KACF,CAAC,CAAC;IAEH,MAAM,iBAAiB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QAC5E,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,IAAI,EAAE,UAAmB;QACzB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,iBAAiB,EAAE,QAAQ,CAAC,qBAAqB;KAClD,CAAC,CAAC,CAAC;IACJ,MAAM,mBAAmB,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU;SAClD,MAAM,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,IAAI,KAAK,YAAY,CAAC;SACtD,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,eAAe,CAAC;SACxC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;QACnB,EAAE,EAAE,SAAS,CAAC,EAAE;QAChB,IAAI,EAAE,YAAqB;QAC3B,KAAK,EAAE,SAAS,CAAC,KAAK;KACvB,CAAC,CAAC,CAAC;IAEN,MAAM,KAAK,GAAG,yBAAyB,CAA2B;QAChE,GAAG,IAAI;QACP,MAAM,EAAE,OAAO;QACf,QAAQ,EAAE,CAAC,GAAG,mBAAmB,EAAE,GAAG,iBAAiB,CAAC;QACxD,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM;QAC1C,aAAa,EAAE;YACb,WAAW,EAAE,IAAI,CAAC,SAAS,CAAC,gCAAgC;YAC5D,kBAAkB,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,IAAI,CACvD,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,qBAAqB,KAAK,SAAS,CAC3D;SACF;KACF,CAAC,CAAC;IAEH,MAAM,OAAO,GAAG,yBAAyB,CAA6B;QACpE,GAAG,IAAI;QACP,MAAM,EAAE,SAAS;QACjB,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;QAC7B,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;QAC3B,OAAO,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO;QAC7B,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,UAAU;KACnC,CAAC,CAAC;IAEH,OAAO,CAAC,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,wBAAwB,CAAC,IAAqB;IACrD,MAAM,UAAU,GAAkC,EAAE,CAAC;IACrD,MAAM,aAAa,GAAG,IAAI,GAAG,CAC3B,CAAC,IAAI,CAAC,SAAS,CAAC,kBAAkB,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC,CAC7F,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;IAE1C,KAAK,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,IAAI,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC;QACrE,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7F,IAAI,QAAQ,EAAE,CAAC;YACb,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACzC,UAAU,CAAC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,CAAC,CAAC,CAAC;YACxD,SAAS;QACX,CAAC;QACD,UAAU,CAAC,IAAI,CAAC;YACd,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,IAAI,EAAE,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC;YACpC,KAAK,EAAE,SAAS,CAAC,KAAK;YACtB,IAAI,EAAE,KAAK;YACX,eAAe,EAAE,SAAS,CAAC,eAAe;YAC1C,QAAQ,EAAE,SAAS,CAAC,qBAAqB,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YAC/E,MAAM,EAAE,SAAS,CAAC,mBAAmB;YACrC,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAAC,0BAA0B;SAC9D,CAAC,CAAC;IACL,CAAC;IAED,MAAM,kBAAkB,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,kBAAkB,IAAI,EAAE,CAAC;SACjE,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;SAC/D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC,aAAa,CAAC,CAAC;IACrD,KAAK,MAAM,QAAQ,IAAI,kBAAkB,EAAE,CAAC;QAC1C,UAAU,CAAC,IAAI,CAAC,4BAA4B,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC1D,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,4BAA4B,CACnC,QAAiF;IAEjF,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,UAAU;QACvB,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,IAAI,EAAE,QAAQ,CAAC,aAAa;QAC5B,eAAe,EAAE,QAAQ,CAAC,eAAe;QACzC,QAAQ,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QAClE,MAAM,EAAE,QAAQ,CAAC,gBAAgB,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,sBAAsB,QAAQ,CAAC,MAAM,EAAE;QAC7F,kBAAkB,EAAE,QAAQ,CAAC,kBAAkB;QAC/C,gBAAgB,EAAE,QAAQ,CAAC,gBAAgB;QAC3C,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,oBAAoB,EAAE,EAAE,GAAG,QAAQ,EAAE;KACtC,CAAC;AACJ,CAAC;AAED,SAAS,cAAc,CAAC,IAAY;IAClC,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,YAAY,CAAC;QAClB,KAAK,WAAW,CAAC;QACjB,KAAK,UAAU,CAAC;QAChB,KAAK,cAAc,CAAC;QACpB,KAAK,WAAW,CAAC;QACjB,KAAK,UAAU;YACb,OAAO,IAAI,CAAC;QACd;YACE,OAAO,YAAY,CAAC;IACxB,CAAC;AACH,CAAC"}

package/dist/eval/memory-ab.d.ts

@@ -0,0 +1,61 @@
+import type { ConditionResult, EvalRunConfig } from './types.js';
+import type { EffectivenessMetrics } from '../learning/effectiveness.js';
+import type { ABSignal } from './retrieval/knowledge-quality.js';
+import { type LadderReport, type LadderThresholds } from './retrieval/ladder.js';
+export interface ArmSummary {
+    label: string;
+    successRate: number;
+    successRateCI: [number, number];
+    tasks: number;
+    knowledgeSurfacedRate: number;
+}
+/**
+ * Frame two arms as the ladder's outcome rungs: `treatment` is "guided",
+ * `baseline` is "unguided", so `successRateDelta` is the **arm-vs-arm lift**
+ * (the A/B signal we care about), and `knowledgeApplicationRate` is the
+ * treatment arm's utilization. Pass the treatment's real
+ * `memory.getEffectivenessMetrics()` as `liveTreatment` to use true
+ * application inference instead of the surfaced-rate proxy.
+ */
+export declare function armsToEffectiveness(treatment: ConditionResult, baseline: ConditionResult, liveTreatment?: EffectivenessMetrics): EffectivenessMetrics;
+export interface MemoryABReport {
+    backend: 'mock' | 'live';
+    arms: ArmSummary[];
+    comparison: {
+        treatment: string;
+        baseline: string;
+        successDelta: number;
+        /** 95% bootstrap CIs disjoint — advisory significance (see metrics.ts). */
+        ciDisjoint: boolean;
+    };
+    /** Composed only when an offline retrieval `ABSignal` is supplied. */
+    ladder?: LadderReport;
+    note?: string;
+}
+/**
+ * Compare a treatment arm against a baseline arm and (when an offline retrieval
+ * `ABSignal` is supplied) compose the full proxy → outcome ladder.
+ */
+export declare function compareMemoryArms(input: {
+    arms: ConditionResult[];
+    treatment: string;
+    baseline: string;
+    retrieval?: ABSignal;
+    liveTreatmentEffectiveness?: EffectivenessMetrics;
+    ladderThresholds?: Partial<LadderThresholds>;
+    backend?: 'mock' | 'live';
+    note?: string;
+}): MemoryABReport;
+/**
+ * Axis 1 — memory on/off, on the deterministic mock. Runs the existing
+ * `SequentialEvalRunner` arm machinery (carryMemory true vs false) and returns
+ * the comparison. Validates the whole pipeline without a live agent; the
+ * embedding-vs-lexical ladder needs {@link compareMemoryArms} fed by live arms.
+ */
+export declare function runMemoryOnOffMock(opts?: {
+    config?: Partial<EvalRunConfig>;
+    retrieval?: ABSignal;
+}): Promise<MemoryABReport>;
+/** Human-readable report for logs / `docs/runs`. */
+export declare function formatMemoryAB(r: MemoryABReport): string;
+//# sourceMappingURL=memory-ab.d.ts.map

package/dist/eval/memory-ab.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"memory-ab.d.ts","sourceRoot":"","sources":["../../src/eval/memory-ab.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAc,MAAM,YAAY,CAAC;AAI7E,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,8BAA8B,CAAC;AACzE,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,kCAAkC,CAAC;AACjE,OAAO,EAGL,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACtB,MAAM,uBAAuB,CAAC;AA6C/B,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,qBAAqB,EAAE,MAAM,CAAC;CAC/B;AAYD;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,eAAe,EAC1B,QAAQ,EAAE,eAAe,EACzB,aAAa,CAAC,EAAE,oBAAoB,GACnC,oBAAoB,CA8BtB;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,GAAG,MAAM,CAAC;IACzB,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,UAAU,EAAE;QACV,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,2EAA2E;QAC3E,UAAU,EAAE,OAAO,CAAC;KACrB,CAAC;IACF,sEAAsE;IACtE,MAAM,CAAC,EAAE,YAAY,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE;IACvC,IAAI,EAAE,eAAe,EAAE,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,CAAC,EAAE,QAAQ,CAAC;IACrB,0BAA0B,CAAC,EAAE,oBAAoB,CAAC;IAClD,gBAAgB,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC7C,OAAO,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,GAAG,cAAc,CA+BjB;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,CAAC,EAAE;IAC9C,MAAM,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAChC,SAAS,CAAC,EAAE,QAAQ,CAAC;CACtB,GAAG,OAAO,CAAC,cAAc,CAAC,CAc1B;AAED,oDAAoD;AACpD,wBAAgB,cAAc,CAAC,CAAC,EAAE,cAAc,GAAG,MAAM,CAkBxD"}