@aid-framework/runtime 0.0.0-stage → 0.1.39

package/dist/evals/dataset.js

@@ -0,0 +1,147 @@
+/**
+ * Eval datasets: the probabilistic analogue of a golden fixture.
+ *
+ * A dataset is versioned and committed. The version matters as much as the cases:
+ * comparing a run against a baseline recorded on a *different* dataset version is the
+ * most common way an eval gate becomes noise, so the gate refuses it.
+ */
+import { readFileSync } from 'node:fs';
+// biome-ignore lint/suspicious/noShadowRestrictedNames: the framework's own error name, matching the Python runtime's public surface
+import { EvalError } from '../errors.js';
+import { dumpsSorted } from '../internal/json.js';
+/** One input, its expected output, and any tags the metrics key off. */
+export class EvalCase {
+    id;
+    input;
+    expected;
+    tags;
+    metadata;
+    constructor(init) {
+        if (init.id.length === 0)
+            throw new RangeError('case id must not be empty');
+        this.id = init.id;
+        this.input = init.input;
+        this.expected = init.expected ?? null;
+        this.tags = init.tags ?? [];
+        this.metadata = init.metadata ?? {};
+    }
+    toJSON() {
+        return {
+            id: this.id,
+            input: this.input,
+            expected: this.expected,
+            tags: [...this.tags],
+            metadata: { ...this.metadata },
+        };
+    }
+}
+function parseCase(raw, where) {
+    if (typeof raw !== 'object' || raw === null || Array.isArray(raw)) {
+        throw new EvalError(`dataset ${where} is not an object`);
+    }
+    const record = raw;
+    const id = record.id;
+    const input = record.input;
+    if (typeof id !== 'string' || id.length === 0) {
+        throw new EvalError(`dataset ${where} needs a non-empty 'id'`);
+    }
+    if (typeof input !== 'string')
+        throw new EvalError(`case ${JSON.stringify(id)} needs an 'input'`);
+    const expected = record.expected;
+    if (expected !== undefined && expected !== null && typeof expected !== 'string') {
+        throw new EvalError(`case ${JSON.stringify(id)} has a non-string 'expected'`);
+    }
+    const tags = record.tags;
+    const metadata = record.metadata;
+    return new EvalCase({
+        id,
+        input,
+        expected: typeof expected === 'string' ? expected : null,
+        tags: Array.isArray(tags) ? tags.filter((tag) => typeof tag === 'string') : [],
+        metadata: typeof metadata === 'object' && metadata !== null && !Array.isArray(metadata)
+            ? metadata
+            : {},
+    });
+}
+function parseDataset(payload) {
+    if (typeof payload !== 'object' || payload === null || Array.isArray(payload)) {
+        throw new EvalError('a dataset must be a JSON object');
+    }
+    const record = payload;
+    const name = record.name;
+    const version = record.version;
+    const cases = record.cases;
+    if (typeof name !== 'string' || name.length === 0) {
+        throw new EvalError("a dataset needs a non-empty 'name'");
+    }
+    if (typeof version !== 'string' || version.length === 0) {
+        throw new EvalError("a dataset needs a non-empty 'version'");
+    }
+    if (!Array.isArray(cases) || cases.length === 0) {
+        throw new EvalError('a dataset needs at least one case');
+    }
+    return new Dataset({
+        name,
+        version,
+        cases: cases.map((entry, index) => parseCase(entry, `case ${index}`)),
+    });
+}
+/** An ordered, versioned set of cases. */
+export class Dataset {
+    name;
+    version;
+    cases;
+    constructor(init) {
+        if (init.name.length === 0)
+            throw new RangeError('dataset name must not be empty');
+        if (init.version.length === 0)
+            throw new RangeError('dataset version must not be empty');
+        if (init.cases.length === 0)
+            throw new RangeError('a dataset needs at least one case');
+        this.name = init.name;
+        this.version = init.version;
+        this.cases = init.cases;
+    }
+    get length() {
+        return this.cases.length;
+    }
+    ids() {
+        return this.cases.map((entry) => entry.id);
+    }
+    case(caseId) {
+        for (const candidate of this.cases) {
+            if (candidate.id === caseId)
+                return candidate;
+        }
+        throw new EvalError(`dataset ${JSON.stringify(this.name)} has no case ${JSON.stringify(caseId)}`);
+    }
+    byTag(tag) {
+        return this.cases.filter((entry) => entry.tags.includes(tag));
+    }
+    static fromRecords(name, version, records) {
+        return parseDataset({ name, version, cases: records });
+    }
+    static fromJson(payload) {
+        let parsed;
+        try {
+            parsed = JSON.parse(payload);
+        }
+        catch (error) {
+            throw new EvalError(`a dataset must be JSON: ${error instanceof Error ? error.message : String(error)}`);
+        }
+        return parseDataset(parsed);
+    }
+    static fromPath(path) {
+        return Dataset.fromJson(readFileSync(path, 'utf8'));
+    }
+    /** Canonical JSON: sorted keys, so a dataset diff is reviewable. */
+    toJson() {
+        const payload = {
+            name: this.name,
+            version: this.version,
+            cases: this.cases.map((entry) => entry.toJSON()),
+        };
+        return `${dumpsSorted(payload)}\n`;
+    }
+}
+//# sourceMappingURL=dataset.js.map

package/dist/evals/dataset.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dataset.js","sourceRoot":"","sources":["../../src/evals/dataset.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AAEvC,qIAAqI;AACrI,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAUlD,wEAAwE;AACxE,MAAM,OAAO,QAAQ;IACV,EAAE,CAAS;IACX,KAAK,CAAS;IACd,QAAQ,CAAgB;IACxB,IAAI,CAAoB;IACxB,QAAQ,CAAmC;IAEpD,YAAY,IAAkB;QAC5B,IAAI,IAAI,CAAC,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,2BAA2B,CAAC,CAAC;QAC5E,IAAI,CAAC,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC;QAClB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;QACtC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC;QAC5B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC;IACtC,CAAC;IAED,MAAM;QACJ,OAAO;YACL,EAAE,EAAE,IAAI,CAAC,EAAE;YACX,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;YACpB,QAAQ,EAAE,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE;SAC/B,CAAC;IACJ,CAAC;CACF;AAED,SAAS,SAAS,CAAC,GAAY,EAAE,KAAa;IAC5C,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,SAAS,CAAC,WAAW,KAAK,mBAAmB,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,MAAM,GAAG,GAA8B,CAAC;IAC9C,MAAM,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC;IACrB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,IAAI,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,SAAS,CAAC,WAAW,KAAK,yBAAyB,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,mBAAmB,CAAC,CAAC;IAClG,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IACjC,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,IAAI,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAChF,MAAM,IAAI,SAAS,CAAC,QAAQ,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,8BAA8B,CAAC,CAAC;IAChF,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACzB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC;IACjC,OAAO,IAAI,QAAQ,CAAC;QAClB,EAAE;QACF,KAAK;QACL,QAAQ,EAAE,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI;QACxD,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAiB,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE;QAC7F,QAAQ,EACN,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;YAC3E,CAAC,CAAE,QAAmC;YACtC,CAAC,CAAC,EAAE;KACT,CAAC,CAAC;AACL,CAAC;AAED,SAAS,YAAY,CAAC,OAAgB;IACpC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC9E,MAAM,IAAI,SAAS,CAAC,iCAAiC,CAAC,CAAC;IACzD,CAAC;IACD,MAAM,MAAM,GAAG,OAAkC,CAAC;IAClD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACzB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;IAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAC3B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,SAAS,CAAC,oCAAoC,CAAC,CAAC;IAC5D,CAAC;IACD,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxD,MAAM,IAAI,SAAS,CAAC,uCAAuC,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,SAAS,CAAC,mCAAmC,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,IAAI,OAAO,CAAC;QACjB,IAAI;QACJ,OAAO;QACP,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,KAAK,EAAE,QAAQ,KAAK,EAAE,CAAC,CAAC;KACtE,CAAC,CAAC;AACL,CAAC;AAED,0CAA0C;AAC1C,MAAM,OAAO,OAAO;IACT,IAAI,CAAS;IACb,OAAO,CAAS;IAChB,KAAK,CAAsB;IAEpC,YAAY,IAIX;QACC,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,gCAAgC,CAAC,CAAC;QACnF,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,mCAAmC,CAAC,CAAC;QACzF,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,UAAU,CAAC,mCAAmC,CAAC,CAAC;QACvF,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;QACtB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IAC1B,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED,GAAG;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC7C,CAAC;IAED,IAAI,CAAC,MAAc;QACjB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACnC,IAAI,SAAS,CAAC,EAAE,KAAK,MAAM;gBAAE,OAAO,SAAS,CAAC;QAChD,CAAC;QACD,MAAM,IAAI,SAAS,CACjB,WAAW,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,CAC7E,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAW;QACf,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,CAAC,WAAW,CAChB,IAAY,EACZ,OAAe,EACf,OAAqD;QAErD,OAAO,YAAY,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,CAAC,QAAQ,CAAC,OAAe;QAC7B,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,SAAS,CACjB,2BAA2B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACpF,CAAC;QACJ,CAAC;QACD,OAAO,YAAY,CAAC,MAAM,CAAC,CAAC;IAC9B,CAAC;IAED,MAAM,CAAC,QAAQ,CAAC,IAAkB;QAChC,OAAO,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC;IACtD,CAAC;IAED,oEAAoE;IACpE,MAAM;QACJ,MAAM,OAAO,GAAG;YACd,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;SACjD,CAAC;QACF,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC;IACrC,CAAC;CACF"}

package/dist/evals/gate.d.ts

@@ -0,0 +1,236 @@
+/**
+ * N-run sampling and the statistical baseline gate.
+ *
+ * The ratified Phase 0 decision is that gating uses **N-run sampling with statistical
+ * thresholds compared against a stored baseline**, in the `maxRegression` style --
+ * *not* strict equality. Non-determinism is expected and must not be treated as a bug.
+ *
+ * Concretely, for each metric we:
+ *
+ * 1. Run the dataset `runs` times, giving one mean score per run. Those means are the
+ *    *sample*, and their spread is the honest measure of how much this system varies
+ *    when nothing changed.
+ * 2. Compare the sample against a stored baseline (its own mean, spread and size).
+ * 3. Fail only when the drop exceeds `maxRegression` **and** that drop is larger than
+ *    the standard error of the difference could explain, at the configured
+ *    significance level.
+ *
+ * Step 3 is what makes the gate trustworthy in both directions. A gate that fires on
+ * any decrease is a flaky gate; a gate that only compares means can be fooled by a
+ * noisy run. Requiring both a material *and* a statistically distinguishable drop
+ * means the gate reports a regression when the evidence says there is one, and stays
+ * quiet otherwise.
+ *
+ * The z critical values are a table rather than a library call because adding a
+ * statistics dependency to enforce one one-sided test is not a trade worth making in a
+ * runtime that must stay small.
+ *
+ * ## Parity note
+ *
+ * The arithmetic here is a direct transcription of `aid_runtime.evals.gate`, including
+ * the z-critical table, the verdict ordering, and the six-decimal/four-decimal/three-
+ * decimal formatting of the rationales. Only the *inputs* differ between the two
+ * runtimes, because their fake providers draw their accuracy samples differently.
+ */
+import type { Metric } from './metrics.js';
+/**
+ * The one-sided z critical value at `alpha`, rounded to a stricter level.
+ *
+ * `alpha` is rounded to the next *stricter* tabulated level -- a smaller alpha, hence a
+ * larger z, hence a gate that demands slightly more evidence than asked for. That is
+ * the safe direction for a regression gate: a gate that fires on borderline evidence
+ * is a gate that gets switched off. An `alpha` below the strictest tabulated level
+ * gets the strictest value, for the same reason.
+ */
+export declare function zCritical(alpha: number): number;
+/**
+ * One-sided z for "the current mean is lower than the baseline mean".
+ *
+ * Uses the standard error of the difference of two means, so *both* spreads
+ * contribute: a noisy baseline is a weak claim to regress against, and the gate should
+ * demand more evidence before rejecting it.
+ */
+export declare function regressionZ(init: {
+    readonly currentMean: number;
+    readonly currentStdev: number;
+    readonly currentN: number;
+    readonly baselineMean: number;
+    readonly baselineStdev: number;
+    readonly baselineN: number;
+}): number;
+/** Per-run, per-metric means: one value per run of the dataset. */
+export declare class MetricRun {
+    readonly metric: string;
+    readonly values: readonly number[];
+    constructor(init: {
+        readonly metric: string;
+        readonly values: readonly number[];
+    });
+    toJSON(): Record<string, unknown>;
+}
+/** The sampled result of running a dataset N times. */
+export declare class EvalRun {
+    readonly dataset: string;
+    readonly datasetVersion: string;
+    readonly runs: number;
+    readonly metrics: readonly MetricRun[];
+    constructor(init: {
+        readonly dataset: string;
+        readonly datasetVersion: string;
+        readonly runs: number;
+        readonly metrics: readonly MetricRun[];
+    });
+    means(): Record<string, number>;
+    spread(): Record<string, number>;
+    valuesFor(metric: string): readonly number[];
+    /**
+     * Whether any metric produced different scores across runs.
+     *
+     * Exposed because "the run varied" is a *fact about the system*, not a failure: a
+     * test asserts on it to prove the gate tolerates noise rather than assuming a
+     * deterministic system.
+     */
+    isVarying(): boolean;
+    toJson(): string;
+}
+/** A stored metric distribution, recorded from a run that was accepted. */
+export declare class MetricBaseline {
+    readonly metric: string;
+    readonly runs: number;
+    readonly mean: number;
+    readonly stdev: number;
+    constructor(init: {
+        readonly metric: string;
+        readonly runs: number;
+        readonly mean: number;
+        readonly stdev: number;
+    });
+    static fromValues(metric: string, values: readonly number[]): MetricBaseline;
+    toJSON(): Record<string, unknown>;
+}
+/** A committed baseline: the accepted distribution for a dataset and model. */
+export declare class EvalBaseline {
+    readonly dataset: string;
+    readonly datasetVersion: string;
+    readonly model: string;
+    readonly metrics: readonly MetricBaseline[];
+    readonly note: string;
+    constructor(init: {
+        readonly dataset: string;
+        readonly datasetVersion: string;
+        readonly model: string;
+        readonly metrics: readonly MetricBaseline[];
+        readonly note?: string;
+    });
+    static fromRun(run: EvalRun, options: {
+        readonly model: string;
+        readonly note?: string;
+    }): EvalBaseline;
+    metric(name: string): MetricBaseline;
+    /** Parse a committed baseline. Accepts `snake_case` and `camelCase` keys. */
+    static fromJson(payload: string): EvalBaseline;
+    static fromPath(path: string | URL): EvalBaseline;
+    toJson(): string;
+}
+/** The acceptance rule for one metric. */
+export declare class Threshold {
+    readonly metric: string;
+    /**
+     * Absolute floor on the mean. Independent of the baseline: it says what "good
+     * enough" means, which no comparison to a past run can express.
+     */
+    readonly minMean: number | null;
+    /** Largest mean drop, versus the baseline, still considered acceptable. */
+    readonly maxRegression: number;
+    /** Significance level for the one-sided regression test. */
+    readonly alpha: number;
+    constructor(init: {
+        readonly metric: string;
+        readonly minMean?: number | null;
+        readonly maxRegression?: number;
+        readonly alpha?: number;
+    });
+}
+/** Why one metric passed or failed. The rationale is the product. */
+export declare class MetricGateResult {
+    readonly metric: string;
+    readonly passed: boolean;
+    readonly currentMean: number;
+    readonly currentStdev: number;
+    readonly baselineMean: number;
+    readonly baselineStdev: number;
+    readonly delta: number;
+    readonly z: number;
+    readonly alpha: number;
+    readonly maxRegression: number;
+    readonly rationale: string;
+    constructor(init: {
+        readonly metric: string;
+        readonly passed: boolean;
+        readonly currentMean: number;
+        readonly currentStdev: number;
+        readonly baselineMean: number;
+        readonly baselineStdev: number;
+        readonly delta: number;
+        readonly z: number;
+        readonly alpha: number;
+        readonly maxRegression: number;
+        readonly rationale: string;
+    });
+}
+/** The gate's verdict over every metric. */
+export declare class GateResult {
+    readonly passed: boolean;
+    readonly dataset: string;
+    readonly datasetVersion: string;
+    readonly runs: number;
+    readonly results: readonly MetricGateResult[];
+    readonly varying: boolean;
+    constructor(init: {
+        readonly passed: boolean;
+        readonly dataset: string;
+        readonly datasetVersion: string;
+        readonly runs: number;
+        readonly results: readonly MetricGateResult[];
+        readonly varying?: boolean;
+    });
+    failures(): MetricGateResult[];
+    summary(): string;
+}
+export interface BaselineGateOptions {
+    readonly metrics?: readonly Metric[];
+    readonly allowNondeterministic?: boolean;
+}
+/** Compares a sampled run against a stored baseline. */
+export declare class BaselineGate {
+    private readonly _thresholds;
+    private readonly _metrics;
+    private readonly _allowNondeterministic;
+    constructor(thresholds: readonly Threshold[], options?: BaselineGateOptions);
+    get thresholds(): readonly Threshold[];
+    /** Decide whether `run` regressed against `baseline`. */
+    evaluate(run: EvalRun, baseline: EvalBaseline): GateResult;
+    private evaluateMetric;
+    /**
+     * The pass/fail call and the sentence explaining it.
+     *
+     * Three ways to pass, and the distinction between them is the whole point: no floor
+     * breached, no material drop, or a drop that is material but within what the two
+     * runs' own variation can explain. Only a drop that is both material *and* beyond the
+     * noise threshold counts as a regression.
+     */
+    private verdict;
+    /**
+     * Refuse thresholds that a non-deterministic metric would undermine.
+     *
+     * The rule is enforced here, at construction, rather than documented and hoped for: a
+     * gate configured on a model-backed metric is not a gate, and refusing at
+     * configuration time is the only moment that is cheap to fix.
+     */
+    private checkMetricEligibility;
+}
+/** Build a gate from parsed JSON/YAML, so CI config stays declarative. */
+export declare function gateFromConfig(config: Readonly<Record<string, unknown>>, options?: {
+    readonly metrics?: readonly Metric[];
+}): BaselineGate;
+//# sourceMappingURL=gate.d.ts.map

package/dist/evals/gate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gate.d.ts","sourceRoot":"","sources":["../../src/evals/gate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAOH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAY3C;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAO/C;AAkCD;;;;;;GAMG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B,GAAG,MAAM,CAWT;AAED,mEAAmE;AACnE,qBAAa,SAAS;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IAEnC,YAAY,IAAI,EAAE;QAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,EAIhF;IAED,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAEhC;CACF;AAED,uDAAuD;AACvD,qBAAa,OAAO;IAClB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,SAAS,SAAS,EAAE,CAAC;IAEvC,YAAY,IAAI,EAAE;QAChB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;QAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,OAAO,EAAE,SAAS,SAAS,EAAE,CAAC;KACxC,EASA;IAED,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAI9B;IAED,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAI/B;IAED,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,CAK3C;IAED;;;;;;OAMG;IACH,SAAS,IAAI,OAAO,CAEnB;IAED,MAAM,IAAI,MAAM,CAQf;CACF;AAED,2EAA2E;AAC3E,qBAAa,cAAc;IACzB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,YAAY,IAAI,EAAE;QAChB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;KACxB,EAYA;IAED,MAAM,CAAC,UAAU,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,cAAc,CAO3E;IAED,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAOhC;CACF;AAsBD,+EAA+E;AAC/E,qBAAa,YAAY;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,SAAS,cAAc,EAAE,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,YAAY,IAAI,EAAE;QAChB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;QAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,OAAO,EAAE,SAAS,cAAc,EAAE,CAAC;QAC5C,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;KACxB,EAOA;IAED,MAAM,CAAC,OAAO,CACZ,GAAG,EAAE,OAAO,EACZ,OAAO,EAAE;QAAE,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,GAC1D,YAAY,CAQd;IAED,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,CAKnC;IAED,6EAA6E;IAC7E,MAAM,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,YAAY,CA6B7C;IAED,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,GAAG,GAAG,YAAY,CAEhD;IAED,MAAM,IAAI,MAAM,CASf;CACF;AAED,0CAA0C;AAC1C,qBAAa,SAAS;IACpB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB;;;OAGG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,2EAA2E;IAC3E,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,4DAA4D;IAC5D,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,YAAY,IAAI,EAAE;QAChB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;QACjC,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;QAChC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;KACzB,EAgBA;CACF;AAED,qEAAqE;AACrE,qBAAa,gBAAgB;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAE3B,YAAY,IAAI,EAAE;QAChB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;QACzB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;QAC7B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;QAC9B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;QAC9B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;QAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;QACnB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;QAC/B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;KAC5B,EAYA;CACF;AAED,4CAA4C;AAC5C,qBAAa,UAAU;IACrB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,SAAS,gBAAgB,EAAE,CAAC;IAC9C,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAE1B,YAAY,IAAI,EAAE;QAChB,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC;QACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;QAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,OAAO,EAAE,SAAS,gBAAgB,EAAE,CAAC;QAC9C,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;KAC5B,EAQA;IAED,QAAQ,IAAI,gBAAgB,EAAE,CAE7B;IAED,OAAO,IAAI,MAAM,CAUhB;CACF;AAED,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,qBAAqB,CAAC,EAAE,OAAO,CAAC;CAC1C;AAED,wDAAwD;AACxD,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuB;IACnD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA8B;IACvD,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAU;IAEjD,YAAY,UAAU,EAAE,SAAS,SAAS,EAAE,EAAE,OAAO,GAAE,mBAAwB,EAe9E;IAED,IAAI,UAAU,IAAI,SAAS,SAAS,EAAE,CAErC;IAED,yDAAyD;IACzD,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,QAAQ,EAAE,YAAY,GAAG,UAAU,CAmBzD;IAED,OAAO,CAAC,cAAc;IAmCtB;;;;;;;OAOG;IACH,OAAO,CAAC,OAAO;IA4Cf;;;;;;OAMG;IACH,OAAO,CAAC,sBAAsB;CAkB/B;AAED,0EAA0E;AAC1E,wBAAgB,cAAc,CAC5B,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,EACzC,OAAO,GAAE;IAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CAAO,GACrD,YAAY,CA4Bd"}