@trazum/core 1.50.6 → 1.50.8

package/dist/experiment.d.ts

@@ -0,0 +1,134 @@
+/**
+ * Two arms on real traffic, and the three things that stop it being theatre.
+ *
+ * `eval` compares two prompts on cases somebody wrote; `route` compares two
+ * models on the same. Both measure agreement in a laboratory. The traffic is
+ * the only place the real question gets answered — and the moment a comparison
+ * runs on real traffic, three failures become available that a laboratory does
+ * not have.
+ *
+ * ## 1. A winner where there is none
+ *
+ * Two arms always produce two numbers, and one of them is always larger. An
+ * A/B report that names a winner from that is a coin flip with a dashboard.
+ * The verdict here is **three-valued**, the way `verify`'s has been since
+ * 1.39: A wins, B wins, or **not separable on this traffic** — and the third
+ * one comes with the number of outcomes per arm that *would* separate them, so
+ * "run it longer" is a quantified instruction rather than a shrug.
+ *
+ * ## 2. Peeking
+ *
+ * A test stopped on the first afternoon it looked good is not a test. The
+ * stopping rule is declared **before** the experiment starts, and the report
+ * says whether it was honoured. It cannot enforce that — nobody can stop
+ * somebody reading a number early — but it can make an early stop *visible*
+ * to whoever reads the result later, which is the part that matters.
+ *
+ * ## 3. Quality and cost judged apart, then together
+ *
+ * The interesting arm is almost never better *and* cheaper. It is better and
+ * dearer, and the decision needs one figure nobody computes: **what an extra
+ * success costs**. That is the marginal figure — the difference in spend over
+ * the difference in successes — and it is the number a product decision
+ * actually turns on, printed rather than left as an exercise.
+ *
+ * ## The statistics are shown, not asserted
+ *
+ * Wilson score intervals per arm and Newcombe's interval on the difference,
+ * because both behave at the small samples this will actually see. The
+ * intervals are returned, not just the verdict: a reader who disagrees with
+ * the threshold can see the numbers it was applied to, which is the same
+ * discipline `eval` established by running the original twice before judging
+ * anything against it.
+ */
+import type { OutcomeTally, OutcomeVocabulary } from './outcome.js';
+/**
+ * The declaration, made before the experiment runs.
+ *
+ * Its whole purpose is to exist *earlier* than the result. A stopping rule
+ * invented after looking at the numbers is not a stopping rule.
+ */
+export interface ExperimentDeclaration {
+    /** Two arms. More would need a multiple-comparison correction nobody asked for. */
+    arms: [string, string];
+    /**
+     * Outcomes each arm must record before the result may be read.
+     *
+     * Declared as a count rather than a duration, because a duration is a proxy
+     * for a count and the proxy breaks the week traffic doubles.
+     */
+    minOutcomesPerArm: number;
+}
+/** What one arm actually did. */
+export interface ExperimentArm {
+    name: string;
+    tally: OutcomeTally;
+    /** Everything this arm spent, recorded outcome or not. */
+    totalUsd: number;
+}
+export interface ArmResult {
+    name: string;
+    successes: number;
+    /** Calls carrying a *declared* outcome — the denominator. */
+    recorded: number;
+    /** Successes over recorded, or null when nothing was recorded. */
+    rate: number | null;
+    /** Wilson score interval on that rate, or null. */
+    interval: {
+        low: number;
+        high: number;
+    } | null;
+    /** Spend on calls carrying a declared outcome. */
+    recordedUsd: number;
+}
+export type Separation = 'a-wins' | 'b-wins' | 'not-separable';
+export type NotSeparableReason = 'interval-includes-zero' | 'no-difference-observed' | 'nothing-recorded';
+export interface Marginal {
+    /**
+     * What one extra success costs, going from the worse arm to the better one.
+     *
+     * `(dearer spend − cheaper spend) / (more successes − fewer successes)`,
+     * both per call so arms of different sizes compare. Null when the better arm
+     * is also the cheaper one, because then nothing is being bought and a
+     * "cost per extra success" would be a negative number people would quote.
+     */
+    usdPerExtraSuccess: number | null;
+    /** The arm that resolved more, by rate. */
+    better: string;
+    /** Whether that arm also costs more per call. */
+    dearer: boolean;
+}
+export interface ExperimentResult {
+    a: ArmResult;
+    b: ArmResult;
+    separation: Separation;
+    /** Why, when not separable. A refusal never arrives bare. */
+    notSeparable: NotSeparableReason | null;
+    /** Newcombe's interval on (rate A − rate B), or null. */
+    difference: {
+        point: number;
+        low: number;
+        high: number;
+    } | null;
+    /**
+     * Outcomes **per arm** that would separate the observed difference, or null.
+     *
+     * Null when the arms recorded the same rate: no sample size separates a
+     * difference of zero, and returning a very large number would read as "keep
+     * going" when the honest answer is "there is nothing here to find".
+     */
+    outcomesNeededPerArm: number | null;
+    stopping: {
+        declared: number;
+        /** Whether both arms cleared the declared minimum. */
+        honoured: boolean;
+        /** The arm that has not, when one has not. */
+        short: string | null;
+    };
+    marginal: Marginal | null;
+}
+export declare function runExperiment(declaration: ExperimentDeclaration, arms: {
+    a: ExperimentArm;
+    b: ExperimentArm;
+}, vocabulary: OutcomeVocabulary | null): ExperimentResult;
+//# sourceMappingURL=experiment.d.ts.map

package/dist/experiment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../src/experiment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAGH,OAAO,KAAK,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAMpE;;;;;GAKG;AACH,MAAM,WAAW,qBAAqB;IACpC,mFAAmF;IACnF,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACvB;;;;;OAKG;IACH,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,iCAAiC;AACjC,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,6DAA6D;IAC7D,QAAQ,EAAE,MAAM,CAAC;IACjB,kEAAkE;IAClE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,mDAAmD;IACnD,QAAQ,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAC/C,kDAAkD;IAClD,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,UAAU,GAAG,QAAQ,GAAG,QAAQ,GAAG,eAAe,CAAC;AAE/D,MAAM,MAAM,kBAAkB,GAC1B,wBAAwB,GACxB,wBAAwB,GACxB,kBAAkB,CAAC;AAEvB,MAAM,WAAW,QAAQ;IACvB;;;;;;;OAOG;IACH,kBAAkB,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,2CAA2C;IAC3C,MAAM,EAAE,MAAM,CAAC;IACf,iDAAiD;IACjD,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,CAAC,EAAE,SAAS,CAAC;IACb,CAAC,EAAE,SAAS,CAAC;IACb,UAAU,EAAE,UAAU,CAAC;IACvB,6DAA6D;IAC7D,YAAY,EAAE,kBAAkB,GAAG,IAAI,CAAC;IACxC,yDAAyD;IACzD,UAAU,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAChE;;;;;;OAMG;IACH,oBAAoB,EAAE,MAAM,GAAG,IAAI,CAAC;IACpC,QAAQ,EAAE;QACR,QAAQ,EAAE,MAAM,CAAC;QACjB,sDAAsD;QACtD,QAAQ,EAAE,OAAO,CAAC;QAClB,8CAA8C;QAC9C,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;KACtB,CAAC;IACF,QAAQ,EAAE,QAAQ,GAAG,IAAI,CAAC;CAC3B;AAmCD,wBAAgB,aAAa,CAC3B,WAAW,EAAE,qBAAqB,EAClC,IAAI,EAAE;IAAE,CAAC,EAAE,aAAa,CAAC;IAAC,CAAC,EAAE,aAAa,CAAA;CAAE,EAC5C,UAAU,EAAE,iBAAiB,GAAG,IAAI,GACnC,gBAAgB,CAkGlB"}

package/dist/experiment.js

@@ -0,0 +1,168 @@
+/**
+ * Two arms on real traffic, and the three things that stop it being theatre.
+ *
+ * `eval` compares two prompts on cases somebody wrote; `route` compares two
+ * models on the same. Both measure agreement in a laboratory. The traffic is
+ * the only place the real question gets answered — and the moment a comparison
+ * runs on real traffic, three failures become available that a laboratory does
+ * not have.
+ *
+ * ## 1. A winner where there is none
+ *
+ * Two arms always produce two numbers, and one of them is always larger. An
+ * A/B report that names a winner from that is a coin flip with a dashboard.
+ * The verdict here is **three-valued**, the way `verify`'s has been since
+ * 1.39: A wins, B wins, or **not separable on this traffic** — and the third
+ * one comes with the number of outcomes per arm that *would* separate them, so
+ * "run it longer" is a quantified instruction rather than a shrug.
+ *
+ * ## 2. Peeking
+ *
+ * A test stopped on the first afternoon it looked good is not a test. The
+ * stopping rule is declared **before** the experiment starts, and the report
+ * says whether it was honoured. It cannot enforce that — nobody can stop
+ * somebody reading a number early — but it can make an early stop *visible*
+ * to whoever reads the result later, which is the part that matters.
+ *
+ * ## 3. Quality and cost judged apart, then together
+ *
+ * The interesting arm is almost never better *and* cheaper. It is better and
+ * dearer, and the decision needs one figure nobody computes: **what an extra
+ * success costs**. That is the marginal figure — the difference in spend over
+ * the difference in successes — and it is the number a product decision
+ * actually turns on, printed rather than left as an exercise.
+ *
+ * ## The statistics are shown, not asserted
+ *
+ * Wilson score intervals per arm and Newcombe's interval on the difference,
+ * because both behave at the small samples this will actually see. The
+ * intervals are returned, not just the verdict: a reader who disagrees with
+ * the threshold can see the numbers it was applied to, which is the same
+ * discipline `eval` established by running the original twice before judging
+ * anything against it.
+ */
+import { judgeOutcome } from './outcome.js';
+/** z for a two-sided 95% interval, and for 80% power. */
+const Z_95 = 1.959963984540054;
+const Z_POWER_80 = 0.8416212335729143;
+/** Wilson score interval — behaves at the sample sizes this will actually see. */
+function wilson(successes, n, z = Z_95) {
+    if (n <= 0)
+        return null;
+    const p = successes / n;
+    const denominator = 1 + (z * z) / n;
+    const centre = (p + (z * z) / (2 * n)) / denominator;
+    const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / denominator;
+    return { low: Math.max(0, centre - half), high: Math.min(1, centre + half) };
+}
+function resultOf(arm, vocabulary) {
+    const declared = vocabulary ?? { values: [], success: [] };
+    let successes = 0;
+    let recorded = 0;
+    let recordedUsd = 0;
+    for (const entry of arm.tally.byValue) {
+        // Undeclared values are out of both halves, as everywhere since 1.50.4: a
+        // typo in an exporter must not decide an experiment.
+        if (judgeOutcome(entry.value, declared) === 'undeclared')
+            continue;
+        recorded += entry.calls;
+        recordedUsd += entry.usd;
+        if (declared.success.includes(entry.value))
+            successes += entry.calls;
+    }
+    return {
+        name: arm.name,
+        successes,
+        recorded,
+        rate: recorded > 0 ? successes / recorded : null,
+        interval: wilson(successes, recorded),
+        recordedUsd,
+    };
+}
+export function runExperiment(declaration, arms, vocabulary) {
+    const a = resultOf(arms.a, vocabulary);
+    const b = resultOf(arms.b, vocabulary);
+    const stopping = {
+        declared: declaration.minOutcomesPerArm,
+        honoured: a.recorded >= declaration.minOutcomesPerArm && b.recorded >= declaration.minOutcomesPerArm,
+        short: a.recorded < declaration.minOutcomesPerArm
+            ? a.name
+            : b.recorded < declaration.minOutcomesPerArm
+                ? b.name
+                : null,
+    };
+    const bare = (reason) => ({
+        a,
+        b,
+        separation: 'not-separable',
+        notSeparable: reason,
+        difference: null,
+        outcomesNeededPerArm: null,
+        stopping,
+        marginal: null,
+    });
+    if (a.rate === null || b.rate === null || a.interval === null || b.interval === null) {
+        return bare('nothing-recorded');
+    }
+    /**
+     * Newcombe's interval on the difference, built from the two Wilson
+     * intervals rather than from a normal approximation to the difference.
+     *
+     * The naive interval is symmetric and can run past 0 or 1, which at the
+     * sample sizes a real experiment starts with is not an edge case — it is
+     * most of the first week.
+     */
+    const point = a.rate - b.rate;
+    const low = point - Math.sqrt((a.rate - a.interval.low) ** 2 + (b.interval.high - b.rate) ** 2);
+    const high = point + Math.sqrt((a.interval.high - a.rate) ** 2 + (b.rate - b.interval.low) ** 2);
+    const difference = { point, low, high };
+    /**
+     * How many outcomes per arm would separate the difference *observed so far*.
+     *
+     * A standard two-proportion power calculation at 95% confidence and 80%
+     * power. It is an estimate about a difference that may itself be noise, and
+     * it is offered as "how much longer" rather than as a promise — but a number
+     * somebody can act on beats "not significant", which tells a reader nothing
+     * about whether to wait a day or abandon the idea.
+     */
+    const spread = Math.abs(point);
+    const outcomesNeededPerArm = spread === 0
+        ? null
+        : Math.ceil((((Z_95 + Z_POWER_80) ** 2) * (a.rate * (1 - a.rate) + b.rate * (1 - b.rate))) /
+            (spread * spread));
+    const perCallA = arms.a.tally.parsed > 0 ? arms.a.totalUsd / arms.a.tally.parsed : 0;
+    const perCallB = arms.b.tally.parsed > 0 ? arms.b.totalUsd / arms.b.tally.parsed : 0;
+    const better = a.rate >= b.rate ? a : b;
+    const worse = better === a ? b : a;
+    const betterPerCall = better === a ? perCallA : perCallB;
+    const worsePerCall = better === a ? perCallB : perCallA;
+    const rateGap = better.rate - worse.rate;
+    const marginal = {
+        /**
+         * Per call on both sides, so arms that took different shares of the
+         * traffic compare. Dividing raw totals would report a marginal cost that
+         * moves when the split changes and the behaviour does not.
+         */
+        usdPerExtraSuccess: betterPerCall > worsePerCall && rateGap > 0
+            ? (betterPerCall - worsePerCall) / rateGap
+            : null,
+        better: better.name,
+        dearer: betterPerCall > worsePerCall,
+    };
+    if (spread === 0)
+        return { ...bare('no-difference-observed'), difference, marginal };
+    if (low <= 0 && high >= 0) {
+        return { ...bare('interval-includes-zero'), difference, outcomesNeededPerArm, marginal };
+    }
+    return {
+        a,
+        b,
+        separation: point > 0 ? 'a-wins' : 'b-wins',
+        notSeparable: null,
+        difference,
+        outcomesNeededPerArm,
+        stopping,
+        marginal,
+    };
+}
+//# sourceMappingURL=experiment.js.map

package/dist/experiment.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../src/experiment.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA0CG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,yDAAyD;AACzD,MAAM,IAAI,GAAG,iBAAiB,CAAC;AAC/B,MAAM,UAAU,GAAG,kBAAkB,CAAC;AA0FtC,kFAAkF;AAClF,SAAS,MAAM,CAAC,SAAiB,EAAE,CAAS,EAAE,CAAC,GAAG,IAAI;IACpD,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,CAAC,GAAG,SAAS,GAAG,CAAC,CAAC;IACxB,MAAM,WAAW,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;IACpC,MAAM,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;IACrD,MAAM,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC;IACtF,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;AAC/E,CAAC;AAED,SAAS,QAAQ,CAAC,GAAkB,EAAE,UAAoC;IACxE,MAAM,QAAQ,GAAG,UAAU,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC3D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;QACtC,0EAA0E;QAC1E,qDAAqD;QACrD,IAAI,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,QAAQ,CAAC,KAAK,YAAY;YAAE,SAAS;QACnE,QAAQ,IAAI,KAAK,CAAC,KAAK,CAAC;QACxB,WAAW,IAAI,KAAK,CAAC,GAAG,CAAC;QACzB,IAAI,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC;YAAE,SAAS,IAAI,KAAK,CAAC,KAAK,CAAC;IACvE,CAAC;IACD,OAAO;QACL,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,SAAS;QACT,QAAQ;QACR,IAAI,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC,CAAC,IAAI;QAChD,QAAQ,EAAE,MAAM,CAAC,SAAS,EAAE,QAAQ,CAAC;QACrC,WAAW;KACZ,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,aAAa,CAC3B,WAAkC,EAClC,IAA4C,EAC5C,UAAoC;IAEpC,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAEvC,MAAM,QAAQ,GAAG;QACf,QAAQ,EAAE,WAAW,CAAC,iBAAiB;QACvC,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,WAAW,CAAC,iBAAiB,IAAI,CAAC,CAAC,QAAQ,IAAI,WAAW,CAAC,iBAAiB;QACpG,KAAK,EACH,CAAC,CAAC,QAAQ,GAAG,WAAW,CAAC,iBAAiB;YACxC,CAAC,CAAC,CAAC,CAAC,IAAI;YACR,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,WAAW,CAAC,iBAAiB;gBAC1C,CAAC,CAAC,CAAC,CAAC,IAAI;gBACR,CAAC,CAAC,IAAI;KACb,CAAC;IAEF,MAAM,IAAI,GAAG,CAAC,MAA0B,EAAoB,EAAE,CAAC,CAAC;QAC9D,CAAC;QACD,CAAC;QACD,UAAU,EAAE,eAAe;QAC3B,YAAY,EAAE,MAAM;QACpB,UAAU,EAAE,IAAI;QAChB,oBAAoB,EAAE,IAAI;QAC1B,QAAQ;QACR,QAAQ,EAAE,IAAI;KACf,CAAC,CAAC;IAEH,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,IAAI,CAAC,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;QACrF,OAAO,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAClC,CAAC;IAED;;;;;;;OAOG;IACH,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC;IAC9B,MAAM,GAAG,GAAG,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IAChG,MAAM,IAAI,GAAG,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;IACjG,MAAM,UAAU,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;IAExC;;;;;;;;OAQG;IACH,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,oBAAoB,GACxB,MAAM,KAAK,CAAC;QACV,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,IAAI,CAAC,IAAI,CACP,CAAC,CAAC,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAC5E,CAAC,MAAM,GAAG,MAAM,CAAC,CACpB,CAAC;IAER,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACrF,MAAM,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACxC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IACzD,MAAM,YAAY,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;IACxD,MAAM,OAAO,GAAI,MAAM,CAAC,IAAe,GAAI,KAAK,CAAC,IAAe,CAAC;IAEjE,MAAM,QAAQ,GAAa;QACzB;;;;WAIG;QACH,kBAAkB,EAChB,aAAa,GAAG,YAAY,IAAI,OAAO,GAAG,CAAC;YACzC,CAAC,CAAC,CAAC,aAAa,GAAG,YAAY,CAAC,GAAG,OAAO;YAC1C,CAAC,CAAC,IAAI;QACV,MAAM,EAAE,MAAM,CAAC,IAAI;QACnB,MAAM,EAAE,aAAa,GAAG,YAAY;KACrC,CAAC;IAEF,IAAI,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,GAAG,IAAI,CAAC,wBAAwB,CAAC,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC;IACrF,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,GAAG,IAAI,CAAC,wBAAwB,CAAC,EAAE,UAAU,EAAE,oBAAoB,EAAE,QAAQ,EAAE,CAAC;IAC3F,CAAC;IAED,OAAO;QACL,CAAC;QACD,CAAC;QACD,UAAU,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QAC3C,YAAY,EAAE,IAAI;QAClB,UAAU;QACV,oBAAoB;QACpB,QAAQ;QACR,QAAQ;KACT,CAAC;AACJ,CAAC"}

package/dist/index.d.ts

@@ -19,6 +19,10 @@ export type { PlanAction, PlanActionKind, PlanAssumption, PlanDocument } from '.
 export { verifyPlan } from './verify.js';
 export { buildHistory, storedReportFrom, MIN_RUN } from './history.js';
 export { outcomeReport, judgeOutcome, OUTCOME_UNLOCKS } from './outcome.js';
+export { runExperiment } from './experiment.js';
+export { qualityGate, MIN_OUTCOMES_EACH_SIDE, MAX_MODEL_MIX_DRIFT, MAX_VOLUME_RATIO, MAX_COVERAGE_DRIFT, } from './quality-gate.js';
+export type { Confounder, GateSide, GateUnknown, GateVerdict, QualityGate } from './quality-gate.js';
+export type { ArmResult, ExperimentArm, ExperimentDeclaration, ExperimentResult, Marginal, NotSeparableReason, Separation, } from './experiment.js';
 export { ladderArithmetic, ladderPosition, validateLadder, MIN_CALLS_FOR_LADDER, BREAK_EVEN_BAND, } from './ladder.js';
 export type { LadderArithmetic, LadderPolicy, LadderPosition, LadderProblem, LadderUnknown, LadderVerdict, } from './ladder.js';
 export { perOutcome, rankPerOutcome, MIN_OUTCOMES_FOR_RATE, MIN_COVERAGE_FOR_RATE, } from './per-outcome.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,QAAQ,GACT,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,cAAc,EACd,YAAY,EACZ,mBAAmB,EACnB,cAAc,EACd,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,WAAW,GACZ,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAGlF,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACtF,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAQ7F,OAAO,EACL,mBAAmB,EACnB,uBAAuB,EACvB,wBAAwB,EACxB,6BAA6B,EAC7B,YAAY,GACb,MAAM,UAAU,CAAC;AAClB,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACpG,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC3F,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACnE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAC1F,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,oBAAoB,EACpB,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,YAAY,EACV,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,aAAa,EACb,aAAa,EACb,aAAa,GACd,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,UAAU,EACV,cAAc,EACd,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,WAAW,EACX,UAAU,EACV,cAAc,GACf,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,eAAe,EACf,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,cAAc,EACd,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACpF,YAAY,EACV,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,eAAe,EACf,cAAc,EACd,aAAa,EACb,eAAe,EACf,YAAY,GACb,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,YAAY,GACb,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAC7E,YAAY,EACV,cAAc,EACd,aAAa,EACb,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,SAAS,GACV,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACrE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC5F,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7E,YAAY,EACV,WAAW,EACX,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,aAAa,GACd,MAAM,WAAW,CAAC;AACnB,YAAY,EACV,aAAa,EACb,aAAa,EACb,cAAc,EACd,YAAY,EACZ,sBAAsB,EACtB,UAAU,GACX,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAC7E,YAAY,EACV,YAAY,EACZ,eAAe,EACf,aAAa,EACb,SAAS,EACT,YAAY,EACZ,WAAW,EACX,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,oBAAoB,EACpB,UAAU,EACV,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,cAAc,EACd,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC1F,OAAO,EACL,UAAU,EACV,YAAY,EACZ,uBAAuB,EACvB,oBAAoB,EACpB,eAAe,EACf,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,cAAc,EACd,aAAa,EACb,mBAAmB,EACnB,oBAAoB,EACpB,aAAa,EACb,OAAO,EACP,kBAAkB,EAClB,WAAW,GACZ,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,eAAe,EAAE,UAAU,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAClG,YAAY,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACrG,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC3D,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC1E,YAAY,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAKxE,OAAO,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACxE,YAAY,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAGzF,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAGnE,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AACxF,YAAY,EAAE,eAAe,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC7G,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAIlF,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,YAAY,EAAE,gBAAgB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAExF,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAChF,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAClG,YAAY,EACV,oBAAoB,EACpB,oBAAoB,EACpB,qBAAqB,GACtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,wBAAwB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC3E,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC7F,YAAY,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACtG,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtF,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAClG,YAAY,EAAE,eAAe,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAC1F,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpG,YAAY,EACV,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,aAAa,GACd,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AACjF,OAAO,EACL,MAAM,EACN,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAG3E,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAMpD,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACrF,YAAY,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AACrE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACjE,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,QAAQ,EACR,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,GACd,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjE,YAAY,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAKhD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,YAAY,EACV,aAAa,EACb,cAAc,EACd,aAAa,EACb,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAKtD,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,YAAY,EACV,eAAe,EACf,YAAY,EACZ,mBAAmB,GACpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACtF,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAItF,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACpF,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,cAAc,EACd,cAAc,EACd,eAAe,EACf,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAClB,YAAY,EACV,aAAa,EACb,uBAAuB,EACvB,wBAAwB,EACxB,sBAAsB,EACtB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAGlB,OAAO,EACL,cAAc,EACd,OAAO,EACP,WAAW,EACX,QAAQ,EACR,WAAW,EACX,aAAa,EACb,EAAE,EACF,EAAE,GACH,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAGhG,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAGpE,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,aAAa,EACb,eAAe,EACf,mBAAmB,GACpB,MAAM,UAAU,CAAC;AAClB,YAAY,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,UAAU,CAAC;AAG3E,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpE,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG/C,OAAO,EAAE,cAAc,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAC3E,YAAY,EAAE,aAAa,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAK3F,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACzF,YAAY,EAAE,mBAAmB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAGjF,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGxF,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,aAAa,GACd,MAAM,cAAc,CAAC;AAKtB,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAKzD,OAAO,EACL,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,kBAAkB,EAClB,gBAAgB,EAChB,uBAAuB,EACvB,SAAS,EACT,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,QAAQ,GACT,MAAM,YAAY,CAAC;AACpB,YAAY,EACV,cAAc,EACd,YAAY,EACZ,mBAAmB,EACnB,cAAc,EACd,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,WAAW,GACZ,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAGlF,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AACtF,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAQ7F,OAAO,EACL,mBAAmB,EACnB,uBAAuB,EACvB,wBAAwB,EACxB,6BAA6B,EAC7B,YAAY,GACb,MAAM,UAAU,CAAC;AAClB,YAAY,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,UAAU,CAAC;AACnE,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACpG,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAC3F,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACnE,YAAY,EAAE,UAAU,EAAE,cAAc,EAAE,cAAc,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAC1F,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EACL,WAAW,EACX,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AACrG,YAAY,EACV,SAAS,EACT,aAAa,EACb,qBAAqB,EACrB,gBAAgB,EAChB,QAAQ,EACR,kBAAkB,EAClB,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,oBAAoB,EACpB,eAAe,GAChB,MAAM,aAAa,CAAC;AACrB,YAAY,EACV,gBAAgB,EAChB,YAAY,EACZ,cAAc,EACd,aAAa,EACb,aAAa,EACb,aAAa,GACd,MAAM,aAAa,CAAC;AACrB,OAAO,EACL,UAAU,EACV,cAAc,EACd,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,UAAU,EACV,iBAAiB,EACjB,WAAW,EACX,UAAU,EACV,cAAc,GACf,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,eAAe,EACf,aAAa,EACb,YAAY,EACZ,YAAY,EACZ,aAAa,EACb,cAAc,EACd,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACpF,YAAY,EACV,eAAe,EACf,aAAa,EACb,kBAAkB,EAClB,WAAW,EACX,eAAe,EACf,cAAc,EACd,aAAa,EACb,eAAe,EACf,YAAY,GACb,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,iBAAiB,EACjB,YAAY,GACb,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAC7E,YAAY,EACV,cAAc,EACd,aAAa,EACb,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,WAAW,EACX,QAAQ,EACR,SAAS,GACV,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AACrE,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACzF,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AACxC,YAAY,EAAE,gBAAgB,EAAE,WAAW,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC5F,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC7E,YAAY,EACV,WAAW,EACX,YAAY,EACZ,iBAAiB,EACjB,OAAO,EACP,gBAAgB,EAChB,WAAW,EACX,YAAY,EACZ,UAAU,EACV,gBAAgB,EAChB,aAAa,GACd,MAAM,WAAW,CAAC;AACnB,YAAY,EACV,aAAa,EACb,aAAa,EACb,cAAc,EACd,YAAY,EACZ,sBAAsB,EACtB,UAAU,GACX,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAC7E,YAAY,EACV,YAAY,EACZ,eAAe,EACf,aAAa,EACb,SAAS,EACT,YAAY,EACZ,WAAW,EACX,eAAe,GAChB,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,oBAAoB,EACpB,UAAU,EACV,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,cAAc,EACd,YAAY,GACb,MAAM,YAAY,CAAC;AACpB,YAAY,EAAE,WAAW,EAAE,aAAa,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC1F,OAAO,EACL,UAAU,EACV,YAAY,EACZ,uBAAuB,EACvB,oBAAoB,EACpB,eAAe,EACf,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,cAAc,EACd,aAAa,EACb,mBAAmB,EACnB,oBAAoB,EACpB,aAAa,EACb,OAAO,EACP,kBAAkB,EAClB,WAAW,GACZ,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,eAAe,EAAE,UAAU,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAClG,YAAY,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACrG,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAC3D,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC1E,YAAY,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACzD,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAKxE,OAAO,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AACxE,YAAY,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAGzF,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAGnE,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AACxF,YAAY,EAAE,eAAe,EAAE,sBAAsB,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AAC7G,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAIlF,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,eAAe,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAC;AACrF,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,YAAY,EAAE,gBAAgB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAExF,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAChF,YAAY,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAClG,YAAY,EACV,oBAAoB,EACpB,oBAAoB,EACpB,qBAAqB,GACtB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAE,wBAAwB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC3E,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,mBAAmB,CAAC;AAC7F,YAAY,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,MAAM,mBAAmB,CAAC;AACtG,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,YAAY,EAAE,UAAU,EAAE,gBAAgB,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AACtF,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC1C,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AAClG,YAAY,EAAE,eAAe,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAC1F,YAAY,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAClE,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACpG,YAAY,EACV,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,aAAa,GACd,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAC;AACjF,OAAO,EACL,MAAM,EACN,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAG3E,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAMpD,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AACrF,YAAY,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,gBAAgB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AACrE,YAAY,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACjE,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,QAAQ,EACR,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,GACd,MAAM,eAAe,CAAC;AACvB,YAAY,EACV,cAAc,EACd,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AACjE,YAAY,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAKhD,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC/C,YAAY,EACV,aAAa,EACb,cAAc,EACd,aAAa,EACb,cAAc,GACf,MAAM,cAAc,CAAC;AACtB,YAAY,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAKtD,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACtE,YAAY,EACV,eAAe,EACf,YAAY,EACZ,mBAAmB,GACpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACtF,YAAY,EAAE,eAAe,EAAE,cAAc,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAItF,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC/C,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACpF,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,cAAc,EACd,cAAc,EACd,eAAe,EACf,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAClB,YAAY,EACV,aAAa,EACb,uBAAuB,EACvB,wBAAwB,EACxB,sBAAsB,EACtB,qBAAqB,EACrB,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAGlB,OAAO,EACL,cAAc,EACd,OAAO,EACP,WAAW,EACX,QAAQ,EACR,WAAW,EACX,aAAa,EACb,EAAE,EACF,EAAE,GACH,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAGhG,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AACxB,YAAY,EACV,aAAa,EACb,iBAAiB,EACjB,iBAAiB,EACjB,eAAe,EACf,YAAY,EACZ,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAGpE,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,aAAa,EACb,eAAe,EACf,mBAAmB,GACpB,MAAM,UAAU,CAAC;AAClB,YAAY,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,UAAU,CAAC;AAG3E,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpE,YAAY,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAG/C,OAAO,EAAE,cAAc,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAC3E,YAAY,EAAE,aAAa,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAK3F,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AACzF,YAAY,EAAE,mBAAmB,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAGjF,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AACjE,YAAY,EAAE,QAAQ,EAAE,UAAU,EAAE,WAAW,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGxF,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC9C,YAAY,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,aAAa,GACd,MAAM,cAAc,CAAC;AAKtB,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAKzD,OAAO,EACL,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,kBAAkB,EAClB,gBAAgB,EAChB,uBAAuB,EACvB,SAAS,EACT,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAChG,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}

package/dist/index.js

@@ -22,6 +22,8 @@ export { buildPlan, planLabelName, parsePlanDocument, PLAN_ACTION_KINDS } from '
 export { verifyPlan } from './verify.js';
 export { buildHistory, storedReportFrom, MIN_RUN } from './history.js';
 export { outcomeReport, judgeOutcome, OUTCOME_UNLOCKS } from './outcome.js';
+export { runExperiment } from './experiment.js';
+export { qualityGate, MIN_OUTCOMES_EACH_SIDE, MAX_MODEL_MIX_DRIFT, MAX_VOLUME_RATIO, MAX_COVERAGE_DRIFT, } from './quality-gate.js';
 export { ladderArithmetic, ladderPosition, validateLadder, MIN_CALLS_FOR_LADDER, BREAK_EVEN_BAND, } from './ladder.js';
 export { perOutcome, rankPerOutcome, MIN_OUTCOMES_FOR_RATE, MIN_COVERAGE_FOR_RATE, } from './per-outcome.js';
 export { gatewayDecision, usageFromResponse, FAILURE_POLICIES } from './gateway.js';

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,QAAQ,GACT,MAAM,YAAY,CAAC;AAWpB,OAAO,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAClF,gFAAgF;AAChF,4EAA4E;AAC5E,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEtF,2EAA2E;AAC3E,+EAA+E;AAC/E,8EAA8E;AAC9E,0EAA0E;AAC1E,yCAAyC;AACzC,4EAA4E;AAC5E,qDAAqD;AACrD,OAAO,EACL,mBAAmB,EACnB,uBAAuB,EACvB,wBAAwB,EACxB,6BAA6B,EAC7B,YAAY,GACb,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACpG,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAG3F,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,oBAAoB,EACpB,eAAe,GAChB,MAAM,aAAa,CAAC;AASrB,OAAO,EACL,UAAU,EACV,cAAc,EACd,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAiB1B,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAYpF,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAOvC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAW7E,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAqB7E,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAU7E,OAAO,EACL,oBAAoB,EACpB,UAAU,EACV,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,cAAc,EACd,YAAY,GACb,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,UAAU,EACV,YAAY,EACZ,uBAAuB,EACvB,oBAAoB,EACpB,eAAe,EACf,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AAiBxB,4EAA4E;AAC5E,8EAA8E;AAC9E,8EAA8E;AAC9E,sBAAsB;AACtB,OAAO,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAExE,2EAA2E;AAC3E,8DAA8D;AAC9D,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACnE,2EAA2E;AAC3E,+DAA+D;AAC/D,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAGxF,yEAAyE;AACzE,mEAAmE;AACnE,2BAA2B;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE7D,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACxF,4EAA4E;AAC5E,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAOhF,OAAO,EAAE,wBAAwB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAG3E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAI1C,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAQpG,OAAO,EACL,MAAM,EACN,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,cAAc,CAAC;AAItB,uEAAuE;AACvE,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,8EAA8E;AAC9E,+EAA+E;AAC/E,6EAA6E;AAC7E,UAAU;AACV,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAErF,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,QAAQ,EACR,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,GACd,MAAM,eAAe,CAAC;AAQvB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAEhD,+EAA+E;AAC/E,+EAA+E;AAC/E,6DAA6D;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAS/C,gFAAgF;AAChF,gFAAgF;AAChF,iEAAiE;AACjE,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAOtE,4EAA4E;AAC5E,6EAA6E;AAC7E,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAGtF,8EAA8E;AAC9E,2EAA2E;AAC3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,cAAc,EACd,cAAc,EACd,eAAe,EACf,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAWlB,uBAAuB;AACvB,OAAO,EACL,cAAc,EACd,OAAO,EACP,WAAW,EACX,QAAQ,EACR,WAAW,EACX,aAAa,EACb,EAAE,EACF,EAAE,GACH,MAAM,iBAAiB,CAAC;AAGzB,sBAAsB;AACtB,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAEpE,kDAAkD;AAClD,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,aAAa,EACb,eAAe,EACf,mBAAmB,GACpB,MAAM,UAAU,CAAC;AAGlB,6BAA6B;AAC7B,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAGpE,qEAAqE;AACrE,OAAO,EAAE,cAAc,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAG3E,kFAAkF;AAClF,kFAAkF;AAClF,wCAAwC;AACxC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAGzF,mEAAmE;AACnE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAGjE,gCAAgC;AAChC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAQ9C,0EAA0E;AAC1E,+EAA+E;AAC/E,8EAA8E;AAC9E,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEzD,8EAA8E;AAC9E,6EAA6E;AAC7E,2DAA2D;AAC3D,OAAO,EACL,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,kBAAkB,EAClB,gBAAgB,EAChB,uBAAuB,EACvB,SAAS,EACT,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,uBAAuB,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAC/F,OAAO,EACL,UAAU,EACV,cAAc,EACd,YAAY,EACZ,cAAc,EACd,YAAY,EACZ,QAAQ,GACT,MAAM,YAAY,CAAC;AAWpB,OAAO,EAAE,kBAAkB,EAAE,yBAAyB,EAAE,MAAM,mBAAmB,CAAC;AAClF,gFAAgF;AAChF,4EAA4E;AAC5E,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAEtF,2EAA2E;AAC3E,+EAA+E;AAC/E,8EAA8E;AAC9E,0EAA0E;AAC1E,yCAAyC;AACzC,4EAA4E;AAC5E,qDAAqD;AACrD,OAAO,EACL,mBAAmB,EACnB,uBAAuB,EACvB,wBAAwB,EACxB,6BAA6B,EAC7B,YAAY,GACb,MAAM,UAAU,CAAC;AAElB,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,UAAU,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtF,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AACpG,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AAG3F,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvE,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAC5E,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,OAAO,EACL,WAAW,EACX,sBAAsB,EACtB,mBAAmB,EACnB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,mBAAmB,CAAC;AAW3B,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,oBAAoB,EACpB,eAAe,GAChB,MAAM,aAAa,CAAC;AASrB,OAAO,EACL,UAAU,EACV,cAAc,EACd,qBAAqB,EACrB,qBAAqB,GACtB,MAAM,kBAAkB,CAAC;AAiB1B,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAYpF,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAOvC,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,oBAAoB,EAAE,MAAM,aAAa,CAAC;AAW7E,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAErE,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExC,OAAO,EAAE,WAAW,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAqB7E,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAU7E,OAAO,EACL,oBAAoB,EACpB,UAAU,EACV,YAAY,EACZ,kBAAkB,EAClB,kBAAkB,EAClB,cAAc,EACd,YAAY,GACb,MAAM,YAAY,CAAC;AAEpB,OAAO,EACL,UAAU,EACV,YAAY,EACZ,uBAAuB,EACvB,oBAAoB,EACpB,eAAe,EACf,sBAAsB,GACvB,MAAM,gBAAgB,CAAC;AAiBxB,4EAA4E;AAC5E,8EAA8E;AAC9E,8EAA8E;AAC9E,sBAAsB;AACtB,OAAO,EAAE,uBAAuB,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAC;AAExE,2EAA2E;AAC3E,8DAA8D;AAC9D,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AACnE,2EAA2E;AAC3E,+DAA+D;AAC/D,OAAO,EAAE,4BAA4B,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAGxF,yEAAyE;AACzE,mEAAmE;AACnE,2BAA2B;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE7D,OAAO,EAAE,0BAA0B,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AACxF,4EAA4E;AAC5E,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAOhF,OAAO,EAAE,wBAAwB,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAG3E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AAChD,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAI1C,OAAO,EAAE,qBAAqB,EAAE,aAAa,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAQpG,OAAO,EACL,MAAM,EACN,aAAa,EACb,gBAAgB,EAChB,qBAAqB,EACrB,aAAa,EACb,iBAAiB,EACjB,QAAQ,EACR,UAAU,EACV,gBAAgB,EAChB,cAAc,EACd,SAAS,EACT,gBAAgB,EAChB,cAAc,GACf,MAAM,cAAc,CAAC;AAItB,uEAAuE;AACvE,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,8EAA8E;AAC9E,+EAA+E;AAC/E,6EAA6E;AAC7E,UAAU;AACV,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAErF,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAGrE,OAAO,EACL,iBAAiB,EACjB,gBAAgB,EAChB,aAAa,EACb,kBAAkB,EAClB,QAAQ,EACR,iBAAiB,EACjB,cAAc,EACd,iBAAiB,EACjB,aAAa,GACd,MAAM,eAAe,CAAC;AAQvB,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAC7D,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AACtF,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAEhD,+EAA+E;AAC/E,+EAA+E;AAC/E,6DAA6D;AAC7D,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAS/C,gFAAgF;AAChF,gFAAgF;AAChF,iEAAiE;AACjE,OAAO,EAAE,gBAAgB,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAOtE,4EAA4E;AAC5E,6EAA6E;AAC7E,OAAO,EAAE,cAAc,EAAE,QAAQ,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAGtF,8EAA8E;AAC9E,2EAA2E;AAC3E,OAAO,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE/C,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9E,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,cAAc,EACd,cAAc,EACd,cAAc,EACd,eAAe,EACf,qBAAqB,GACtB,MAAM,UAAU,CAAC;AAWlB,uBAAuB;AACvB,OAAO,EACL,cAAc,EACd,OAAO,EACP,WAAW,EACX,QAAQ,EACR,WAAW,EACX,aAAa,EACb,EAAE,EACF,EAAE,GACH,MAAM,iBAAiB,CAAC;AAGzB,sBAAsB;AACtB,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,YAAY,EACZ,iBAAiB,EACjB,kBAAkB,GACnB,MAAM,gBAAgB,CAAC;AAWxB,OAAO,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC/D,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC;AAEpE,kDAAkD;AAClD,OAAO,EACL,eAAe,EACf,gBAAgB,EAChB,eAAe,EACf,aAAa,EACb,eAAe,EACf,mBAAmB,GACpB,MAAM,UAAU,CAAC;AAGlB,6BAA6B;AAC7B,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AAGpE,qEAAqE;AACrE,OAAO,EAAE,cAAc,EAAE,4BAA4B,EAAE,MAAM,aAAa,CAAC;AAG3E,kFAAkF;AAClF,kFAAkF;AAClF,wCAAwC;AACxC,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAGzF,mEAAmE;AACnE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAGjE,gCAAgC;AAChC,OAAO,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAQ9C,0EAA0E;AAC1E,+EAA+E;AAC/E,8EAA8E;AAC9E,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAEzD,8EAA8E;AAC9E,6EAA6E;AAC7E,2DAA2D;AAC3D,OAAO,EACL,eAAe,EACf,WAAW,EACX,iBAAiB,EACjB,WAAW,EACX,kBAAkB,EAClB,gBAAgB,EAChB,uBAAuB,EACvB,SAAS,EACT,WAAW,GACZ,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC"}

package/dist/quality-gate.d.ts

@@ -0,0 +1,134 @@
+/**
+ * The failure that actually matters, and the four ways of getting it wrong.
+ *
+ * CI has been able to fail a build for tokens since 1.4 and for dollars since
+ * 1.21. A prompt edit that quietly made the product worse has never been
+ * gateable — which means every cost saving this tool has ever recommended went
+ * into a repository with its most important consequence unmeasured.
+ *
+ * ## This is a before-and-after, not an experiment
+ *
+ * Worth saying in the first paragraph, because the arithmetic is the same as
+ * `experiment`'s and the epistemics are not. An experiment splits traffic
+ * randomly, so the two arms differ only in the thing under test. A
+ * before-and-after splits it by **time**, and everything else that changed at
+ * the same time is in the difference too.
+ *
+ * So this module spends most of its code looking for reasons *not* to blame the
+ * prompt, and it reports `cannot-tell` far more readily than a randomised
+ * comparison would. That is not timidity. A gate that blames the prompt because
+ * the prompt is the thing it can see will be switched off within a month, and a
+ * switched-off gate catches nothing at all.
+ *
+ * ## The three confounders it can actually see
+ *
+ * **The model moved.** If the mix of models shifted across the boundary, the
+ * prompt is not the only variable, and the drop may be entirely somebody else's
+ * migration.
+ *
+ * **The volume moved.** A workload whose traffic tripled is usually a workload
+ * whose *population* changed — a new surface, a new customer, a marketing
+ * campaign — and the questions being asked are not the questions from before.
+ *
+ * **The coverage moved.** The subtle one, and the one nobody thinks of: if the
+ * share of calls recording an outcome changed, the two rates describe different
+ * populations. A team that started instrumenting its hard cases will see its
+ * measured rate fall without anything getting worse.
+ *
+ * Any of them present, and the verdict is `cannot-tell` with the confounder
+ * **named**. Not a hedge attached to a blame — a refusal to blame.
+ *
+ * ## What it cannot see, and says so
+ *
+ * Everything else deployed that day. This module has one label's numbers and no
+ * knowledge of the world, and it never pretends otherwise: a `dropped` verdict
+ * is a statement that the rate fell and the three things it can check did not
+ * move, which is a smaller claim than "the prompt did it" and is the largest
+ * claim the evidence supports.
+ */
+import type { ArmResult, ExperimentArm } from './experiment.js';
+import type { OutcomeVocabulary } from './outcome.js';
+/**
+ * Outcomes each side of the change needs before a verdict is offered.
+ *
+ * A hundred rather than the ten used for a rate elsewhere, because this one
+ * fails builds. The cost of a wrong `dropped` is somebody reverting a good
+ * change and losing the saving; the cost of a wrong `cannot-tell` is waiting a
+ * day. Those are not symmetric and the threshold is not either.
+ */
+export declare const MIN_OUTCOMES_EACH_SIDE = 100;
+/** How far the model mix may move before the comparison stops being about the prompt. */
+export declare const MAX_MODEL_MIX_DRIFT = 0.1;
+/**
+ * How far the call volume may move, as a ratio.
+ *
+ * Half again, either way. Ordinary week-to-week traffic moves less than that;
+ * a workload that doubled is usually a workload whose population changed.
+ */
+export declare const MAX_VOLUME_RATIO = 1.5;
+/** How far outcome coverage may move before the two rates describe different populations. */
+export declare const MAX_COVERAGE_DRIFT = 0.1;
+export type Confounder = {
+    kind: 'model-mix-moved';
+    drift: number;
+    model: string;
+} | {
+    kind: 'volume-moved';
+    beforeCalls: number;
+    afterCalls: number;
+    ratio: number;
+} | {
+    kind: 'coverage-moved';
+    before: number;
+    after: number;
+};
+export type GateVerdict = 'held' | 'dropped' | 'cannot-tell';
+export type GateUnknown = 'too-few-before' | 'too-few-after' | 'confounded' | 'no-vocabulary' | 'not-separable';
+/** One side of the boundary, as the caller measured it. */
+export interface GateSide {
+    arm: ExperimentArm;
+    /** Every call in the window, outcome or not — the coverage denominator. */
+    calls: number;
+    /** Spend per model, for the mix comparison. */
+    usdByModel: Array<{
+        model: string;
+        usd: number;
+    }>;
+}
+export interface QualityGate {
+    verdict: GateVerdict;
+    /** Why, when the verdict is `cannot-tell`. A refusal never arrives bare. */
+    unknown: GateUnknown | null;
+    before: ArmResult;
+    after: ArmResult;
+    /** The 95% interval on (after − before), or null. */
+    difference: {
+        point: number;
+        low: number;
+        high: number;
+    } | null;
+    /**
+     * Everything found that means the prompt is not the only variable.
+     *
+     * Non-empty forces `cannot-tell`. Reported even on a `held` verdict, because
+     * a rate that held while the model changed underneath is not evidence that
+     * the prompt is fine either.
+     */
+    confounders: Confounder[];
+    /**
+     * The cost half — measured, and kept beside the quality half rather than
+     * merged with it. The sentence teams argue about needs both.
+     */
+    cost: {
+        beforeUsdPerCall: number;
+        afterUsdPerCall: number;
+        deltaUsdPerCall: number;
+    } | null;
+    /** Outcomes behind the comparison, so the claim carries its own sample size. */
+    outcomes: {
+        before: number;
+        after: number;
+    };
+}
+export declare function qualityGate(before: GateSide, after: GateSide, vocabulary: OutcomeVocabulary | null): QualityGate;
+//# sourceMappingURL=quality-gate.d.ts.map

package/dist/quality-gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"quality-gate.d.ts","sourceRoot":"","sources":["../src/quality-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAEtD;;;;;;;GAOG;AACH,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,yFAAyF;AACzF,eAAO,MAAM,mBAAmB,MAAM,CAAC;AAEvC;;;;;GAKG;AACH,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAEpC,6FAA6F;AAC7F,eAAO,MAAM,kBAAkB,MAAM,CAAC;AAEtC,MAAM,MAAM,UAAU,GAClB;IAAE,IAAI,EAAE,iBAAiB,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GACzD;IAAE,IAAI,EAAE,cAAc,CAAC;IAAC,WAAW,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,GAChF;IAAE,IAAI,EAAE,gBAAgB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAE9D,MAAM,MAAM,WAAW,GAAG,MAAM,GAAG,SAAS,GAAG,aAAa,CAAC;AAE7D,MAAM,MAAM,WAAW,GACnB,gBAAgB,GAChB,eAAe,GACf,YAAY,GACZ,eAAe,GACf,eAAe,CAAC;AAEpB,2DAA2D;AAC3D,MAAM,WAAW,QAAQ;IACvB,GAAG,EAAE,aAAa,CAAC;IACnB,2EAA2E;IAC3E,KAAK,EAAE,MAAM,CAAC;IACd,+CAA+C;IAC/C,UAAU,EAAE,KAAK,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CACnD;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,WAAW,CAAC;IACrB,4EAA4E;IAC5E,OAAO,EAAE,WAAW,GAAG,IAAI,CAAC;IAC5B,MAAM,EAAE,SAAS,CAAC;IAClB,KAAK,EAAE,SAAS,CAAC;IACjB,qDAAqD;IACrD,UAAU,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAChE;;;;;;OAMG;IACH,WAAW,EAAE,UAAU,EAAE,CAAC;IAC1B;;;OAGG;IACH,IAAI,EAAE;QAAE,gBAAgB,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC;IAC5F,gFAAgF;IAChF,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;CAC7C;AAwBD,wBAAgB,WAAW,CACzB,MAAM,EAAE,QAAQ,EAChB,KAAK,EAAE,QAAQ,EACf,UAAU,EAAE,iBAAiB,GAAG,IAAI,GACnC,WAAW,CAqGb"}

package/dist/quality-gate.js

@@ -0,0 +1,185 @@
+/**
+ * The failure that actually matters, and the four ways of getting it wrong.
+ *
+ * CI has been able to fail a build for tokens since 1.4 and for dollars since
+ * 1.21. A prompt edit that quietly made the product worse has never been
+ * gateable — which means every cost saving this tool has ever recommended went
+ * into a repository with its most important consequence unmeasured.
+ *
+ * ## This is a before-and-after, not an experiment
+ *
+ * Worth saying in the first paragraph, because the arithmetic is the same as
+ * `experiment`'s and the epistemics are not. An experiment splits traffic
+ * randomly, so the two arms differ only in the thing under test. A
+ * before-and-after splits it by **time**, and everything else that changed at
+ * the same time is in the difference too.
+ *
+ * So this module spends most of its code looking for reasons *not* to blame the
+ * prompt, and it reports `cannot-tell` far more readily than a randomised
+ * comparison would. That is not timidity. A gate that blames the prompt because
+ * the prompt is the thing it can see will be switched off within a month, and a
+ * switched-off gate catches nothing at all.
+ *
+ * ## The three confounders it can actually see
+ *
+ * **The model moved.** If the mix of models shifted across the boundary, the
+ * prompt is not the only variable, and the drop may be entirely somebody else's
+ * migration.
+ *
+ * **The volume moved.** A workload whose traffic tripled is usually a workload
+ * whose *population* changed — a new surface, a new customer, a marketing
+ * campaign — and the questions being asked are not the questions from before.
+ *
+ * **The coverage moved.** The subtle one, and the one nobody thinks of: if the
+ * share of calls recording an outcome changed, the two rates describe different
+ * populations. A team that started instrumenting its hard cases will see its
+ * measured rate fall without anything getting worse.
+ *
+ * Any of them present, and the verdict is `cannot-tell` with the confounder
+ * **named**. Not a hedge attached to a blame — a refusal to blame.
+ *
+ * ## What it cannot see, and says so
+ *
+ * Everything else deployed that day. This module has one label's numbers and no
+ * knowledge of the world, and it never pretends otherwise: a `dropped` verdict
+ * is a statement that the rate fell and the three things it can check did not
+ * move, which is a smaller claim than "the prompt did it" and is the largest
+ * claim the evidence supports.
+ */
+import { runExperiment } from './experiment.js';
+/**
+ * Outcomes each side of the change needs before a verdict is offered.
+ *
+ * A hundred rather than the ten used for a rate elsewhere, because this one
+ * fails builds. The cost of a wrong `dropped` is somebody reverting a good
+ * change and losing the saving; the cost of a wrong `cannot-tell` is waiting a
+ * day. Those are not symmetric and the threshold is not either.
+ */
+export const MIN_OUTCOMES_EACH_SIDE = 100;
+/** How far the model mix may move before the comparison stops being about the prompt. */
+export const MAX_MODEL_MIX_DRIFT = 0.1;
+/**
+ * How far the call volume may move, as a ratio.
+ *
+ * Half again, either way. Ordinary week-to-week traffic moves less than that;
+ * a workload that doubled is usually a workload whose population changed.
+ */
+export const MAX_VOLUME_RATIO = 1.5;
+/** How far outcome coverage may move before the two rates describe different populations. */
+export const MAX_COVERAGE_DRIFT = 0.1;
+function mixDrift(before, after) {
+    const share = (rows) => {
+        const total = rows.reduce((sum, row) => sum + row.usd, 0);
+        const out = new Map();
+        if (total <= 0)
+            return out;
+        for (const row of rows)
+            out.set(row.model, (out.get(row.model) ?? 0) + row.usd / total);
+        return out;
+    };
+    const a = share(before);
+    const b = share(after);
+    let worst = null;
+    for (const model of new Set([...a.keys(), ...b.keys()])) {
+        const drift = Math.abs((b.get(model) ?? 0) - (a.get(model) ?? 0));
+        if (worst === null || drift > worst.drift)
+            worst = { model, drift };
+    }
+    if (worst === null || worst.drift <= MAX_MODEL_MIX_DRIFT)
+        return null;
+    return { kind: 'model-mix-moved', drift: worst.drift, model: worst.model };
+}
+export function qualityGate(before, after, vocabulary) {
+    /**
+     * The statistics are `experiment`'s, deliberately.
+     *
+     * One implementation of a two-proportion comparison, so a gate and a
+     * deliberate experiment can never disagree about the same two numbers — the
+     * failure mode where a team runs both and gets two answers, then trusts
+     * whichever one they liked.
+     */
+    const stats = runExperiment({ arms: ['before', 'after'], minOutcomesPerArm: MIN_OUTCOMES_EACH_SIDE }, { a: { ...after.arm, name: 'after' }, b: { ...before.arm, name: 'before' } }, vocabulary);
+    const outcomes = { before: stats.b.recorded, after: stats.a.recorded };
+    const cost = before.calls > 0 && after.calls > 0
+        ? {
+            beforeUsdPerCall: before.arm.totalUsd / before.calls,
+            afterUsdPerCall: after.arm.totalUsd / after.calls,
+            deltaUsdPerCall: after.arm.totalUsd / after.calls - before.arm.totalUsd / before.calls,
+        }
+        : null;
+    const confounders = [];
+    const mix = mixDrift(before.usdByModel, after.usdByModel);
+    if (mix !== null)
+        confounders.push(mix);
+    if (before.calls > 0 && after.calls > 0) {
+        const ratio = after.calls / before.calls;
+        if (ratio > MAX_VOLUME_RATIO || ratio < 1 / MAX_VOLUME_RATIO) {
+            confounders.push({
+                kind: 'volume-moved',
+                beforeCalls: before.calls,
+                afterCalls: after.calls,
+                ratio,
+            });
+        }
+        /**
+         * Coverage, the one nobody thinks of.
+         *
+         * A team that starts instrumenting its hard cases sees its measured rate
+         * fall without anything having got worse. Comparing two rates over
+         * differently-selected populations is the most convincing wrong answer
+         * this module could produce.
+         */
+        const coverageBefore = outcomes.before / before.calls;
+        const coverageAfter = outcomes.after / after.calls;
+        if (Math.abs(coverageAfter - coverageBefore) > MAX_COVERAGE_DRIFT) {
+            confounders.push({ kind: 'coverage-moved', before: coverageBefore, after: coverageAfter });
+        }
+    }
+    const shell = {
+        before: stats.b,
+        after: stats.a,
+        difference: stats.difference,
+        confounders,
+        cost,
+        outcomes,
+    };
+    if (vocabulary === null || vocabulary.success.length === 0) {
+        return { ...shell, verdict: 'cannot-tell', unknown: 'no-vocabulary' };
+    }
+    if (outcomes.before < MIN_OUTCOMES_EACH_SIDE) {
+        return { ...shell, verdict: 'cannot-tell', unknown: 'too-few-before' };
+    }
+    if (outcomes.after < MIN_OUTCOMES_EACH_SIDE) {
+        return { ...shell, verdict: 'cannot-tell', unknown: 'too-few-after' };
+    }
+    /**
+     * A confounder outranks the statistics.
+     *
+     * Checked after the sample sizes and before the verdict, so a build is never
+     * failed on a difference that something else could equally explain. A gate
+     * that blames the prompt because the prompt is the thing it can see gets
+     * switched off within a month, and a switched-off gate catches nothing.
+     */
+    if (confounders.length > 0) {
+        return { ...shell, verdict: 'cannot-tell', unknown: 'confounded' };
+    }
+    if (stats.separation === 'not-separable') {
+        /**
+         * Not separable is `cannot-tell`, never `held`.
+         *
+         * "The rate did not measurably fall" and "the rate held" are different
+         * claims, and a gate that spells them the same way passes a real
+         * regression it merely lacked the power to see. `verify` has held this
+         * line since 1.39 and it matters more here, because this one is wired to
+         * an exit code.
+         */
+        return { ...shell, verdict: 'cannot-tell', unknown: 'not-separable' };
+    }
+    // `a` is the *after* arm, so `a-wins` means the rate went up.
+    return {
+        ...shell,
+        verdict: stats.separation === 'a-wins' ? 'held' : 'dropped',
+        unknown: null,
+    };
+}
+//# sourceMappingURL=quality-gate.js.map

package/dist/quality-gate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"quality-gate.js","sourceRoot":"","sources":["../src/quality-gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAIhD;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C,yFAAyF;AACzF,MAAM,CAAC,MAAM,mBAAmB,GAAG,GAAG,CAAC;AAEvC;;;;;GAKG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAEpC,6FAA6F;AAC7F,MAAM,CAAC,MAAM,kBAAkB,GAAG,GAAG,CAAC;AAkDtC,SAAS,QAAQ,CACf,MAA6C,EAC7C,KAA4C;IAE5C,MAAM,KAAK,GAAG,CAAC,IAA2C,EAAuB,EAAE;QACjF,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QAC1D,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;QACtC,IAAI,KAAK,IAAI,CAAC;YAAE,OAAO,GAAG,CAAC;QAC3B,KAAK,MAAM,GAAG,IAAI,IAAI;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC;QACxF,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IACF,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;IACxB,MAAM,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IACvB,IAAI,KAAK,GAA4C,IAAI,CAAC;IAC1D,KAAK,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;QACxD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClE,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,GAAG,KAAK,CAAC,KAAK;YAAE,KAAK,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IACtE,CAAC;IACD,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,KAAK,IAAI,mBAAmB;QAAE,OAAO,IAAI,CAAC;IACtE,OAAO,EAAE,IAAI,EAAE,iBAAiB,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC;AAC7E,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,MAAgB,EAChB,KAAe,EACf,UAAoC;IAEpC;;;;;;;OAOG;IACH,MAAM,KAAK,GAAG,aAAa,CACzB,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,iBAAiB,EAAE,sBAAsB,EAAE,EACxE,EAAE,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,GAAG,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,EAC5E,UAAU,CACX,CAAC;IAEF,MAAM,QAAQ,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;IACvE,MAAM,IAAI,GACR,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC;QACjC,CAAC,CAAC;YACE,gBAAgB,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK;YACpD,eAAe,EAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK;YACjD,eAAe,EAAE,KAAK,CAAC,GAAG,CAAC,QAAQ,GAAG,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK;SACvF;QACH,CAAC,CAAC,IAAI,CAAC;IAEX,MAAM,WAAW,GAAiB,EAAE,CAAC;IACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAC1D,IAAI,GAAG,KAAK,IAAI;QAAE,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAExC,IAAI,MAAM,CAAC,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACzC,IAAI,KAAK,GAAG,gBAAgB,IAAI,KAAK,GAAG,CAAC,GAAG,gBAAgB,EAAE,CAAC;YAC7D,WAAW,CAAC,IAAI,CAAC;gBACf,IAAI,EAAE,cAAc;gBACpB,WAAW,EAAE,MAAM,CAAC,KAAK;gBACzB,UAAU,EAAE,KAAK,CAAC,KAAK;gBACvB,KAAK;aACN,CAAC,CAAC;QACL,CAAC;QACD;;;;;;;WAOG;QACH,MAAM,cAAc,GAAG,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC;QACtD,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC;QACnD,IAAI,IAAI,CAAC,GAAG,CAAC,aAAa,GAAG,cAAc,CAAC,GAAG,kBAAkB,EAAE,CAAC;YAClE,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,aAAa,EAAE,CAAC,CAAC;QAC7F,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG;QACZ,MAAM,EAAE,KAAK,CAAC,CAAC;QACf,KAAK,EAAE,KAAK,CAAC,CAAC;QACd,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,WAAW;QACX,IAAI;QACJ,QAAQ;KACT,CAAC;IAEF,IAAI,UAAU,KAAK,IAAI,IAAI,UAAU,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3D,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;IACxE,CAAC;IACD,IAAI,QAAQ,CAAC,MAAM,GAAG,sBAAsB,EAAE,CAAC;QAC7C,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;IACzE,CAAC;IACD,IAAI,QAAQ,CAAC,KAAK,GAAG,sBAAsB,EAAE,CAAC;QAC5C,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;IACxE,CAAC;IACD;;;;;;;OAOG;IACH,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3B,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,YAAY,EAAE,CAAC;IACrE,CAAC;IACD,IAAI,KAAK,CAAC,UAAU,KAAK,eAAe,EAAE,CAAC;QACzC;;;;;;;;WAQG;QACH,OAAO,EAAE,GAAG,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC;IACxE,CAAC;IACD,8DAA8D;IAC9D,OAAO;QACL,GAAG,KAAK;QACR,OAAO,EAAE,KAAK,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QAC3D,OAAO,EAAE,IAAI;KACd,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@trazum/core",
-  "version": "1.50.6",
+  "version": "1.50.8",
   "description": "Trazum core: priced advisories for LLM prompts (caching, model tier, batching, schemas), plus deterministic trimming, token counting and pricing.",
   "license": "MIT",
   "author": "David Mu\u00f1oz Rey",

package/src/experiment.ts

@@ -0,0 +1,275 @@
+/**
+ * Two arms on real traffic, and the three things that stop it being theatre.
+ *
+ * `eval` compares two prompts on cases somebody wrote; `route` compares two
+ * models on the same. Both measure agreement in a laboratory. The traffic is
+ * the only place the real question gets answered — and the moment a comparison
+ * runs on real traffic, three failures become available that a laboratory does
+ * not have.
+ *
+ * ## 1. A winner where there is none
+ *
+ * Two arms always produce two numbers, and one of them is always larger. An
+ * A/B report that names a winner from that is a coin flip with a dashboard.
+ * The verdict here is **three-valued**, the way `verify`'s has been since
+ * 1.39: A wins, B wins, or **not separable on this traffic** — and the third
+ * one comes with the number of outcomes per arm that *would* separate them, so
+ * "run it longer" is a quantified instruction rather than a shrug.
+ *
+ * ## 2. Peeking
+ *
+ * A test stopped on the first afternoon it looked good is not a test. The
+ * stopping rule is declared **before** the experiment starts, and the report
+ * says whether it was honoured. It cannot enforce that — nobody can stop
+ * somebody reading a number early — but it can make an early stop *visible*
+ * to whoever reads the result later, which is the part that matters.
+ *
+ * ## 3. Quality and cost judged apart, then together
+ *
+ * The interesting arm is almost never better *and* cheaper. It is better and
+ * dearer, and the decision needs one figure nobody computes: **what an extra
+ * success costs**. That is the marginal figure — the difference in spend over
+ * the difference in successes — and it is the number a product decision
+ * actually turns on, printed rather than left as an exercise.
+ *
+ * ## The statistics are shown, not asserted
+ *
+ * Wilson score intervals per arm and Newcombe's interval on the difference,
+ * because both behave at the small samples this will actually see. The
+ * intervals are returned, not just the verdict: a reader who disagrees with
+ * the threshold can see the numbers it was applied to, which is the same
+ * discipline `eval` established by running the original twice before judging
+ * anything against it.
+ */
+
+import { judgeOutcome } from './outcome.js';
+import type { OutcomeTally, OutcomeVocabulary } from './outcome.js';
+
+/** z for a two-sided 95% interval, and for 80% power. */
+const Z_95 = 1.959963984540054;
+const Z_POWER_80 = 0.8416212335729143;
+
+/**
+ * The declaration, made before the experiment runs.
+ *
+ * Its whole purpose is to exist *earlier* than the result. A stopping rule
+ * invented after looking at the numbers is not a stopping rule.
+ */
+export interface ExperimentDeclaration {
+  /** Two arms. More would need a multiple-comparison correction nobody asked for. */
+  arms: [string, string];
+  /**
+   * Outcomes each arm must record before the result may be read.
+   *
+   * Declared as a count rather than a duration, because a duration is a proxy
+   * for a count and the proxy breaks the week traffic doubles.
+   */
+  minOutcomesPerArm: number;
+}
+
+/** What one arm actually did. */
+export interface ExperimentArm {
+  name: string;
+  tally: OutcomeTally;
+  /** Everything this arm spent, recorded outcome or not. */
+  totalUsd: number;
+}
+
+export interface ArmResult {
+  name: string;
+  successes: number;
+  /** Calls carrying a *declared* outcome — the denominator. */
+  recorded: number;
+  /** Successes over recorded, or null when nothing was recorded. */
+  rate: number | null;
+  /** Wilson score interval on that rate, or null. */
+  interval: { low: number; high: number } | null;
+  /** Spend on calls carrying a declared outcome. */
+  recordedUsd: number;
+}
+
+export type Separation = 'a-wins' | 'b-wins' | 'not-separable';
+
+export type NotSeparableReason =
+  | 'interval-includes-zero'
+  | 'no-difference-observed'
+  | 'nothing-recorded';
+
+export interface Marginal {
+  /**
+   * What one extra success costs, going from the worse arm to the better one.
+   *
+   * `(dearer spend − cheaper spend) / (more successes − fewer successes)`,
+   * both per call so arms of different sizes compare. Null when the better arm
+   * is also the cheaper one, because then nothing is being bought and a
+   * "cost per extra success" would be a negative number people would quote.
+   */
+  usdPerExtraSuccess: number | null;
+  /** The arm that resolved more, by rate. */
+  better: string;
+  /** Whether that arm also costs more per call. */
+  dearer: boolean;
+}
+
+export interface ExperimentResult {
+  a: ArmResult;
+  b: ArmResult;
+  separation: Separation;
+  /** Why, when not separable. A refusal never arrives bare. */
+  notSeparable: NotSeparableReason | null;
+  /** Newcombe's interval on (rate A − rate B), or null. */
+  difference: { point: number; low: number; high: number } | null;
+  /**
+   * Outcomes **per arm** that would separate the observed difference, or null.
+   *
+   * Null when the arms recorded the same rate: no sample size separates a
+   * difference of zero, and returning a very large number would read as "keep
+   * going" when the honest answer is "there is nothing here to find".
+   */
+  outcomesNeededPerArm: number | null;
+  stopping: {
+    declared: number;
+    /** Whether both arms cleared the declared minimum. */
+    honoured: boolean;
+    /** The arm that has not, when one has not. */
+    short: string | null;
+  };
+  marginal: Marginal | null;
+}
+
+/** Wilson score interval — behaves at the sample sizes this will actually see. */
+function wilson(successes: number, n: number, z = Z_95): { low: number; high: number } | null {
+  if (n <= 0) return null;
+  const p = successes / n;
+  const denominator = 1 + (z * z) / n;
+  const centre = (p + (z * z) / (2 * n)) / denominator;
+  const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / denominator;
+  return { low: Math.max(0, centre - half), high: Math.min(1, centre + half) };
+}
+
+function resultOf(arm: ExperimentArm, vocabulary: OutcomeVocabulary | null): ArmResult {
+  const declared = vocabulary ?? { values: [], success: [] };
+  let successes = 0;
+  let recorded = 0;
+  let recordedUsd = 0;
+  for (const entry of arm.tally.byValue) {
+    // Undeclared values are out of both halves, as everywhere since 1.50.4: a
+    // typo in an exporter must not decide an experiment.
+    if (judgeOutcome(entry.value, declared) === 'undeclared') continue;
+    recorded += entry.calls;
+    recordedUsd += entry.usd;
+    if (declared.success.includes(entry.value)) successes += entry.calls;
+  }
+  return {
+    name: arm.name,
+    successes,
+    recorded,
+    rate: recorded > 0 ? successes / recorded : null,
+    interval: wilson(successes, recorded),
+    recordedUsd,
+  };
+}
+
+export function runExperiment(
+  declaration: ExperimentDeclaration,
+  arms: { a: ExperimentArm; b: ExperimentArm },
+  vocabulary: OutcomeVocabulary | null,
+): ExperimentResult {
+  const a = resultOf(arms.a, vocabulary);
+  const b = resultOf(arms.b, vocabulary);
+
+  const stopping = {
+    declared: declaration.minOutcomesPerArm,
+    honoured: a.recorded >= declaration.minOutcomesPerArm && b.recorded >= declaration.minOutcomesPerArm,
+    short:
+      a.recorded < declaration.minOutcomesPerArm
+        ? a.name
+        : b.recorded < declaration.minOutcomesPerArm
+          ? b.name
+          : null,
+  };
+
+  const bare = (reason: NotSeparableReason): ExperimentResult => ({
+    a,
+    b,
+    separation: 'not-separable',
+    notSeparable: reason,
+    difference: null,
+    outcomesNeededPerArm: null,
+    stopping,
+    marginal: null,
+  });
+
+  if (a.rate === null || b.rate === null || a.interval === null || b.interval === null) {
+    return bare('nothing-recorded');
+  }
+
+  /**
+   * Newcombe's interval on the difference, built from the two Wilson
+   * intervals rather than from a normal approximation to the difference.
+   *
+   * The naive interval is symmetric and can run past 0 or 1, which at the
+   * sample sizes a real experiment starts with is not an edge case — it is
+   * most of the first week.
+   */
+  const point = a.rate - b.rate;
+  const low = point - Math.sqrt((a.rate - a.interval.low) ** 2 + (b.interval.high - b.rate) ** 2);
+  const high = point + Math.sqrt((a.interval.high - a.rate) ** 2 + (b.rate - b.interval.low) ** 2);
+  const difference = { point, low, high };
+
+  /**
+   * How many outcomes per arm would separate the difference *observed so far*.
+   *
+   * A standard two-proportion power calculation at 95% confidence and 80%
+   * power. It is an estimate about a difference that may itself be noise, and
+   * it is offered as "how much longer" rather than as a promise — but a number
+   * somebody can act on beats "not significant", which tells a reader nothing
+   * about whether to wait a day or abandon the idea.
+   */
+  const spread = Math.abs(point);
+  const outcomesNeededPerArm =
+    spread === 0
+      ? null
+      : Math.ceil(
+          (((Z_95 + Z_POWER_80) ** 2) * (a.rate * (1 - a.rate) + b.rate * (1 - b.rate))) /
+            (spread * spread),
+        );
+
+  const perCallA = arms.a.tally.parsed > 0 ? arms.a.totalUsd / arms.a.tally.parsed : 0;
+  const perCallB = arms.b.tally.parsed > 0 ? arms.b.totalUsd / arms.b.tally.parsed : 0;
+  const better = a.rate >= b.rate ? a : b;
+  const worse = better === a ? b : a;
+  const betterPerCall = better === a ? perCallA : perCallB;
+  const worsePerCall = better === a ? perCallB : perCallA;
+  const rateGap = (better.rate as number) - (worse.rate as number);
+
+  const marginal: Marginal = {
+    /**
+     * Per call on both sides, so arms that took different shares of the
+     * traffic compare. Dividing raw totals would report a marginal cost that
+     * moves when the split changes and the behaviour does not.
+     */
+    usdPerExtraSuccess:
+      betterPerCall > worsePerCall && rateGap > 0
+        ? (betterPerCall - worsePerCall) / rateGap
+        : null,
+    better: better.name,
+    dearer: betterPerCall > worsePerCall,
+  };
+
+  if (spread === 0) return { ...bare('no-difference-observed'), difference, marginal };
+  if (low <= 0 && high >= 0) {
+    return { ...bare('interval-includes-zero'), difference, outcomesNeededPerArm, marginal };
+  }
+
+  return {
+    a,
+    b,
+    separation: point > 0 ? 'a-wins' : 'b-wins',
+    notSeparable: null,
+    difference,
+    outcomesNeededPerArm,
+    stopping,
+    marginal,
+  };
+}

package/src/index.ts

@@ -50,6 +50,24 @@ export type { PlanAction, PlanActionKind, PlanAssumption, PlanDocument } from '.
 export { verifyPlan } from './verify.js';
 export { buildHistory, storedReportFrom, MIN_RUN } from './history.js';
 export { outcomeReport, judgeOutcome, OUTCOME_UNLOCKS } from './outcome.js';
+export { runExperiment } from './experiment.js';
+export {
+  qualityGate,
+  MIN_OUTCOMES_EACH_SIDE,
+  MAX_MODEL_MIX_DRIFT,
+  MAX_VOLUME_RATIO,
+  MAX_COVERAGE_DRIFT,
+} from './quality-gate.js';
+export type { Confounder, GateSide, GateUnknown, GateVerdict, QualityGate } from './quality-gate.js';
+export type {
+  ArmResult,
+  ExperimentArm,
+  ExperimentDeclaration,
+  ExperimentResult,
+  Marginal,
+  NotSeparableReason,
+  Separation,
+} from './experiment.js';
 export {
   ladderArithmetic,
   ladderPosition,

package/src/quality-gate.ts

@@ -0,0 +1,253 @@
+/**
+ * The failure that actually matters, and the four ways of getting it wrong.
+ *
+ * CI has been able to fail a build for tokens since 1.4 and for dollars since
+ * 1.21. A prompt edit that quietly made the product worse has never been
+ * gateable — which means every cost saving this tool has ever recommended went
+ * into a repository with its most important consequence unmeasured.
+ *
+ * ## This is a before-and-after, not an experiment
+ *
+ * Worth saying in the first paragraph, because the arithmetic is the same as
+ * `experiment`'s and the epistemics are not. An experiment splits traffic
+ * randomly, so the two arms differ only in the thing under test. A
+ * before-and-after splits it by **time**, and everything else that changed at
+ * the same time is in the difference too.
+ *
+ * So this module spends most of its code looking for reasons *not* to blame the
+ * prompt, and it reports `cannot-tell` far more readily than a randomised
+ * comparison would. That is not timidity. A gate that blames the prompt because
+ * the prompt is the thing it can see will be switched off within a month, and a
+ * switched-off gate catches nothing at all.
+ *
+ * ## The three confounders it can actually see
+ *
+ * **The model moved.** If the mix of models shifted across the boundary, the
+ * prompt is not the only variable, and the drop may be entirely somebody else's
+ * migration.
+ *
+ * **The volume moved.** A workload whose traffic tripled is usually a workload
+ * whose *population* changed — a new surface, a new customer, a marketing
+ * campaign — and the questions being asked are not the questions from before.
+ *
+ * **The coverage moved.** The subtle one, and the one nobody thinks of: if the
+ * share of calls recording an outcome changed, the two rates describe different
+ * populations. A team that started instrumenting its hard cases will see its
+ * measured rate fall without anything getting worse.
+ *
+ * Any of them present, and the verdict is `cannot-tell` with the confounder
+ * **named**. Not a hedge attached to a blame — a refusal to blame.
+ *
+ * ## What it cannot see, and says so
+ *
+ * Everything else deployed that day. This module has one label's numbers and no
+ * knowledge of the world, and it never pretends otherwise: a `dropped` verdict
+ * is a statement that the rate fell and the three things it can check did not
+ * move, which is a smaller claim than "the prompt did it" and is the largest
+ * claim the evidence supports.
+ */
+
+import { runExperiment } from './experiment.js';
+import type { ArmResult, ExperimentArm } from './experiment.js';
+import type { OutcomeVocabulary } from './outcome.js';
+
+/**
+ * Outcomes each side of the change needs before a verdict is offered.
+ *
+ * A hundred rather than the ten used for a rate elsewhere, because this one
+ * fails builds. The cost of a wrong `dropped` is somebody reverting a good
+ * change and losing the saving; the cost of a wrong `cannot-tell` is waiting a
+ * day. Those are not symmetric and the threshold is not either.
+ */
+export const MIN_OUTCOMES_EACH_SIDE = 100;
+
+/** How far the model mix may move before the comparison stops being about the prompt. */
+export const MAX_MODEL_MIX_DRIFT = 0.1;
+
+/**
+ * How far the call volume may move, as a ratio.
+ *
+ * Half again, either way. Ordinary week-to-week traffic moves less than that;
+ * a workload that doubled is usually a workload whose population changed.
+ */
+export const MAX_VOLUME_RATIO = 1.5;
+
+/** How far outcome coverage may move before the two rates describe different populations. */
+export const MAX_COVERAGE_DRIFT = 0.1;
+
+export type Confounder =
+  | { kind: 'model-mix-moved'; drift: number; model: string }
+  | { kind: 'volume-moved'; beforeCalls: number; afterCalls: number; ratio: number }
+  | { kind: 'coverage-moved'; before: number; after: number };
+
+export type GateVerdict = 'held' | 'dropped' | 'cannot-tell';
+
+export type GateUnknown =
+  | 'too-few-before'
+  | 'too-few-after'
+  | 'confounded'
+  | 'no-vocabulary'
+  | 'not-separable';
+
+/** One side of the boundary, as the caller measured it. */
+export interface GateSide {
+  arm: ExperimentArm;
+  /** Every call in the window, outcome or not — the coverage denominator. */
+  calls: number;
+  /** Spend per model, for the mix comparison. */
+  usdByModel: Array<{ model: string; usd: number }>;
+}
+
+export interface QualityGate {
+  verdict: GateVerdict;
+  /** Why, when the verdict is `cannot-tell`. A refusal never arrives bare. */
+  unknown: GateUnknown | null;
+  before: ArmResult;
+  after: ArmResult;
+  /** The 95% interval on (after − before), or null. */
+  difference: { point: number; low: number; high: number } | null;
+  /**
+   * Everything found that means the prompt is not the only variable.
+   *
+   * Non-empty forces `cannot-tell`. Reported even on a `held` verdict, because
+   * a rate that held while the model changed underneath is not evidence that
+   * the prompt is fine either.
+   */
+  confounders: Confounder[];
+  /**
+   * The cost half — measured, and kept beside the quality half rather than
+   * merged with it. The sentence teams argue about needs both.
+   */
+  cost: { beforeUsdPerCall: number; afterUsdPerCall: number; deltaUsdPerCall: number } | null;
+  /** Outcomes behind the comparison, so the claim carries its own sample size. */
+  outcomes: { before: number; after: number };
+}
+
+function mixDrift(
+  before: Array<{ model: string; usd: number }>,
+  after: Array<{ model: string; usd: number }>,
+): Confounder | null {
+  const share = (rows: Array<{ model: string; usd: number }>): Map<string, number> => {
+    const total = rows.reduce((sum, row) => sum + row.usd, 0);
+    const out = new Map<string, number>();
+    if (total <= 0) return out;
+    for (const row of rows) out.set(row.model, (out.get(row.model) ?? 0) + row.usd / total);
+    return out;
+  };
+  const a = share(before);
+  const b = share(after);
+  let worst: { model: string; drift: number } | null = null;
+  for (const model of new Set([...a.keys(), ...b.keys()])) {
+    const drift = Math.abs((b.get(model) ?? 0) - (a.get(model) ?? 0));
+    if (worst === null || drift > worst.drift) worst = { model, drift };
+  }
+  if (worst === null || worst.drift <= MAX_MODEL_MIX_DRIFT) return null;
+  return { kind: 'model-mix-moved', drift: worst.drift, model: worst.model };
+}
+
+export function qualityGate(
+  before: GateSide,
+  after: GateSide,
+  vocabulary: OutcomeVocabulary | null,
+): QualityGate {
+  /**
+   * The statistics are `experiment`'s, deliberately.
+   *
+   * One implementation of a two-proportion comparison, so a gate and a
+   * deliberate experiment can never disagree about the same two numbers — the
+   * failure mode where a team runs both and gets two answers, then trusts
+   * whichever one they liked.
+   */
+  const stats = runExperiment(
+    { arms: ['before', 'after'], minOutcomesPerArm: MIN_OUTCOMES_EACH_SIDE },
+    { a: { ...after.arm, name: 'after' }, b: { ...before.arm, name: 'before' } },
+    vocabulary,
+  );
+
+  const outcomes = { before: stats.b.recorded, after: stats.a.recorded };
+  const cost =
+    before.calls > 0 && after.calls > 0
+      ? {
+          beforeUsdPerCall: before.arm.totalUsd / before.calls,
+          afterUsdPerCall: after.arm.totalUsd / after.calls,
+          deltaUsdPerCall: after.arm.totalUsd / after.calls - before.arm.totalUsd / before.calls,
+        }
+      : null;
+
+  const confounders: Confounder[] = [];
+  const mix = mixDrift(before.usdByModel, after.usdByModel);
+  if (mix !== null) confounders.push(mix);
+
+  if (before.calls > 0 && after.calls > 0) {
+    const ratio = after.calls / before.calls;
+    if (ratio > MAX_VOLUME_RATIO || ratio < 1 / MAX_VOLUME_RATIO) {
+      confounders.push({
+        kind: 'volume-moved',
+        beforeCalls: before.calls,
+        afterCalls: after.calls,
+        ratio,
+      });
+    }
+    /**
+     * Coverage, the one nobody thinks of.
+     *
+     * A team that starts instrumenting its hard cases sees its measured rate
+     * fall without anything having got worse. Comparing two rates over
+     * differently-selected populations is the most convincing wrong answer
+     * this module could produce.
+     */
+    const coverageBefore = outcomes.before / before.calls;
+    const coverageAfter = outcomes.after / after.calls;
+    if (Math.abs(coverageAfter - coverageBefore) > MAX_COVERAGE_DRIFT) {
+      confounders.push({ kind: 'coverage-moved', before: coverageBefore, after: coverageAfter });
+    }
+  }
+
+  const shell = {
+    before: stats.b,
+    after: stats.a,
+    difference: stats.difference,
+    confounders,
+    cost,
+    outcomes,
+  };
+
+  if (vocabulary === null || vocabulary.success.length === 0) {
+    return { ...shell, verdict: 'cannot-tell', unknown: 'no-vocabulary' };
+  }
+  if (outcomes.before < MIN_OUTCOMES_EACH_SIDE) {
+    return { ...shell, verdict: 'cannot-tell', unknown: 'too-few-before' };
+  }
+  if (outcomes.after < MIN_OUTCOMES_EACH_SIDE) {
+    return { ...shell, verdict: 'cannot-tell', unknown: 'too-few-after' };
+  }
+  /**
+   * A confounder outranks the statistics.
+   *
+   * Checked after the sample sizes and before the verdict, so a build is never
+   * failed on a difference that something else could equally explain. A gate
+   * that blames the prompt because the prompt is the thing it can see gets
+   * switched off within a month, and a switched-off gate catches nothing.
+   */
+  if (confounders.length > 0) {
+    return { ...shell, verdict: 'cannot-tell', unknown: 'confounded' };
+  }
+  if (stats.separation === 'not-separable') {
+    /**
+     * Not separable is `cannot-tell`, never `held`.
+     *
+     * "The rate did not measurably fall" and "the rate held" are different
+     * claims, and a gate that spells them the same way passes a real
+     * regression it merely lacked the power to see. `verify` has held this
+     * line since 1.39 and it matters more here, because this one is wired to
+     * an exit code.
+     */
+    return { ...shell, verdict: 'cannot-tell', unknown: 'not-separable' };
+  }
+  // `a` is the *after* arm, so `a-wins` means the rate went up.
+  return {
+    ...shell,
+    verdict: stats.separation === 'a-wins' ? 'held' : 'dropped',
+    unknown: null,
+  };
+}