@aid-framework/runtime 0.0.0-stage → 0.1.39

package/dist/evals/predicate.js

@@ -0,0 +1,178 @@
+/**
+ * The PredicateExpr evaluator an `assert` metric runs.
+ *
+ * The AST is data, not code: the IR parses and validates it, the generator emits it
+ * as a plain object literal, and this module walks it. There is no `eval` and no
+ * `new Function` anywhere on that path, so a predicate can only ever do what this
+ * file implements -- read the prediction, the case id and one metadata key, call
+ * four total string builtins, and compare. Anything else is a node kind this walker
+ * does not know, and an unknown node throws rather than passing.
+ *
+ * Throwing matters more than it looks. A false assertion and an *undecidable* one
+ * are different outcomes: `false` is a legitimate result the gate must see, while a
+ * missing metadata key or a string where a number belongs means the eval itself is
+ * misconfigured. Returning `false` for either would report a broken eval as a
+ * failing candidate, and returning `true` would hide it, so both throw `EvalError`.
+ *
+ * Both sides of `and`/`or` are always evaluated. Short-circuiting would let
+ * `case.metadata.missing == "x" or true` pass silently, which is the one failure
+ * mode this whole design exists to prevent.
+ */
+// biome-ignore lint/suspicious/noShadowRestrictedNames: the framework's own error name, matching the Python runtime's public surface
+import { EvalError } from '../errors.js';
+const ORDERING = new Set(['<', '<=', '>', '>=']);
+/** The name a diagnostic uses for a value's type. */
+export function typeName(value) {
+    if (typeof value === 'boolean')
+        return 'boolean';
+    if (typeof value === 'number')
+        return 'number';
+    return 'string';
+}
+/**
+ * Evaluate `node` against one case and prediction, or throw if it cannot be.
+ *
+ * A non-boolean result is an error rather than a truthiness test: the expression's
+ * type is fixed at build time, so a non-boolean here means the AST and the walker
+ * disagree, and guessing which one is right is exactly the kind of guess that makes
+ * a gate untrustworthy.
+ */
+export function evaluatePredicate(node, context) {
+    const value = evaluateValue(node, context);
+    if (typeof value !== 'boolean') {
+        throw new EvalError(`predicate expected a boolean, got ${typeName(value)}`);
+    }
+    return value;
+}
+function evaluateValue(node, context) {
+    if (node === undefined || node === null || typeof node !== 'object') {
+        throw new EvalError('predicate node must be an object');
+    }
+    switch (node.kind) {
+        case 'string':
+            return literalString(node);
+        case 'number':
+            return literalNumber(node);
+        case 'boolean':
+            return literalBoolean(node);
+        case 'invalid':
+            // Unreachable through a built metric -- the build drops an invalid
+            // expression -- but a hand-written AST must not become a silent pass.
+            throw new EvalError(`invalid predicate: ${node.reason || 'unknown reason'}`);
+        case 'path':
+            return path(node, context);
+        case 'call':
+            return call(node, context);
+        case 'comparison':
+            return compare(node, context);
+        case 'not':
+            return !booleanAt(node.operand, context);
+        case 'and': {
+            const left = booleanAt(node.left, context);
+            const right = booleanAt(node.right, context);
+            return left && right;
+        }
+        case 'or': {
+            const left = booleanAt(node.left, context);
+            const right = booleanAt(node.right, context);
+            return left || right;
+        }
+        default:
+            throw new EvalError(`unknown predicate node '${String(node.kind)}'`);
+    }
+}
+function literalString(node) {
+    if (typeof node.value !== 'string') {
+        throw new EvalError(`predicate string literal must be a string, got ${typeName(node.value)}`);
+    }
+    return node.value;
+}
+function literalNumber(node) {
+    if (typeof node.value !== 'number') {
+        throw new EvalError(`predicate number literal must be a number, got ${typeName(node.value)}`);
+    }
+    return node.value;
+}
+function literalBoolean(node) {
+    if (typeof node.value !== 'boolean') {
+        throw new EvalError(`predicate boolean literal must be a boolean, got ${typeName(node.value)}`);
+    }
+    return node.value;
+}
+function path(node, context) {
+    if (node.path === 'prediction')
+        return context.prediction;
+    if (node.path === 'case.id')
+        return context.entry.id;
+    if (node.path === 'case.metadata') {
+        if (typeof node.name !== 'string') {
+            throw new EvalError('case.metadata path must name a key');
+        }
+        if (!(node.name in context.entry.metadata)) {
+            throw new EvalError(`unbound case.metadata key '${node.name}'`);
+        }
+        return context.entry.metadata[node.name];
+    }
+    throw new EvalError(`unknown predicate path '${node.path}'`);
+}
+function call(node, context) {
+    const values = node.args.map((arg) => evaluateValue(arg, context));
+    if (node.name === 'length') {
+        arity(node.name, values, 1);
+        return stringFor(node.name, values[0]).length;
+    }
+    if (node.name === 'contains' || node.name === 'startswith' || node.name === 'endswith') {
+        arity(node.name, values, 2);
+        const text = stringFor(node.name, values[0]);
+        const needle = stringFor(node.name, values[1]);
+        if (node.name === 'contains')
+            return text.includes(needle);
+        if (node.name === 'startswith')
+            return text.startsWith(needle);
+        return text.endsWith(needle);
+    }
+    throw new EvalError(`unknown predicate builtin '${node.name}'`);
+}
+function arity(name, values, wanted) {
+    if (values.length !== wanted) {
+        throw new EvalError(`${name}() takes ${wanted} argument(s), got ${values.length}`);
+    }
+}
+function stringFor(name, value) {
+    if (typeof value !== 'string') {
+        throw new EvalError(`${name}() requires strings, got ${typeName(value)}`);
+    }
+    return value;
+}
+function compare(node, context) {
+    const left = evaluateValue(node.left, context);
+    const right = evaluateValue(node.right, context);
+    if (ORDERING.has(node.op)) {
+        if (typeof left !== 'number' || typeof right !== 'number') {
+            throw new EvalError(`ordering comparison '${node.op}' requires two numbers, got ${typeName(left)} and ${typeName(right)}`);
+        }
+        if (node.op === '<')
+            return left < right;
+        if (node.op === '<=')
+            return left <= right;
+        if (node.op === '>')
+            return left > right;
+        return left >= right;
+    }
+    if (node.op !== '==' && node.op !== '!=') {
+        throw new EvalError(`unknown predicate operator '${node.op}'`);
+    }
+    if (typeName(left) !== typeName(right)) {
+        throw new EvalError(`type mismatch: cannot apply '${node.op}' to ${typeName(left)} and ${typeName(right)}`);
+    }
+    const equal = left === right;
+    return node.op === '==' ? equal : !equal;
+}
+function booleanAt(node, context) {
+    const value = evaluateValue(node, context);
+    if (typeof value !== 'boolean') {
+        throw new EvalError(`predicate expected a boolean, got ${typeName(value)}`);
+    }
+    return value;
+}
+//# sourceMappingURL=predicate.js.map

package/dist/evals/predicate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"predicate.js","sourceRoot":"","sources":["../../src/evals/predicate.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,qIAAqI;AACrI,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AA8BzC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC;AAEjD,qDAAqD;AACrD,MAAM,UAAU,QAAQ,CAAC,KAAqB;IAC5C,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACjD,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,QAAQ,CAAC;IAC/C,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,iBAAiB,CAC/B,IAA+B,EAC/B,OAAkE;IAElE,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC3C,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,SAAS,CAAC,qCAAqC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAOD,SAAS,aAAa,CAAC,IAA+B,EAAE,OAAgB;IACtE,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QACpE,MAAM,IAAI,SAAS,CAAC,kCAAkC,CAAC,CAAC;IAC1D,CAAC;IACD,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC;QAClB,KAAK,QAAQ;YACX,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,QAAQ;YACX,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;QAC7B,KAAK,SAAS;YACZ,OAAO,cAAc,CAAC,IAAI,CAAC,CAAC;QAC9B,KAAK,SAAS;YACZ,mEAAmE;YACnE,sEAAsE;YACtE,MAAM,IAAI,SAAS,CAAC,sBAAsB,IAAI,CAAC,MAAM,IAAI,gBAAgB,EAAE,CAAC,CAAC;QAC/E,KAAK,MAAM;YACT,OAAO,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC7B,KAAK,MAAM;YACT,OAAO,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAC7B,KAAK,YAAY;YACf,OAAO,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAChC,KAAK,KAAK;YACR,OAAO,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAC3C,KAAK,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC3C,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC7C,OAAO,IAAI,IAAI,KAAK,CAAC;QACvB,CAAC;QACD,KAAK,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC3C,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAC7C,OAAO,IAAI,IAAI,KAAK,CAAC;QACvB,CAAC;QACD;YACE,MAAM,IAAI,SAAS,CAAC,2BAA2B,MAAM,CAAE,IAA2B,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACjG,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,IAAyD;IAC9E,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,IAAI,SAAS,CAAC,kDAAkD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC;AACpB,CAAC;AAED,SAAS,aAAa,CAAC,IAAyD;IAC9E,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,IAAI,SAAS,CAAC,kDAAkD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC;AACpB,CAAC;AAED,SAAS,cAAc,CAAC,IAA2D;IACjF,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,IAAI,SAAS,CAAC,oDAAoD,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAClG,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC;AACpB,CAAC;AAED,SAAS,IAAI,CACX,IAA8E,EAC9E,OAAgB;IAEhB,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY;QAAE,OAAO,OAAO,CAAC,UAAU,CAAC;IAC1D,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;IACrD,IAAI,IAAI,CAAC,IAAI,KAAK,eAAe,EAAE,CAAC;QAClC,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAClC,MAAM,IAAI,SAAS,CAAC,oCAAoC,CAAC,CAAC;QAC5D,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3C,MAAM,IAAI,SAAS,CAAC,8BAA8B,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;QAClE,CAAC;QACD,OAAO,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAW,CAAC;IACrD,CAAC;IACD,MAAM,IAAI,SAAS,CAAC,2BAA2B,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;AAC/D,CAAC;AAED,SAAS,IAAI,CACX,IAA+F,EAC/F,OAAgB;IAEhB,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;IACnE,IAAI,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC3B,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAmB,CAAC,CAAC,MAAM,CAAC;IAClE,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QACvF,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAmB,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAmB,CAAC,CAAC;QACjE,IAAI,IAAI,CAAC,IAAI,KAAK,UAAU;YAAE,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC3D,IAAI,IAAI,CAAC,IAAI,KAAK,YAAY;YAAE,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC/D,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IACD,MAAM,IAAI,SAAS,CAAC,8BAA8B,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,KAAK,CAAC,IAAY,EAAE,MAAiC,EAAE,MAAc;IAC5E,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,YAAY,MAAM,qBAAqB,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACrF,CAAC;AACH,CAAC;AAED,SAAS,SAAS,CAAC,IAAY,EAAE,KAAqB;IACpD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,SAAS,CAAC,GAAG,IAAI,4BAA4B,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC5E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,OAAO,CACd,IAKC,EACD,OAAgB;IAEhB,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACjD,IAAI,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;QAC1B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC1D,MAAM,IAAI,SAAS,CACjB,wBAAwB,IAAI,CAAC,EAAE,+BAA+B,QAAQ,CAAC,IAAI,CAAC,QAAQ,QAAQ,CAAC,KAAK,CAAC,EAAE,CACtG,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,EAAE,KAAK,GAAG;YAAE,OAAO,IAAI,GAAG,KAAK,CAAC;QACzC,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI;YAAE,OAAO,IAAI,IAAI,KAAK,CAAC;QAC3C,IAAI,IAAI,CAAC,EAAE,KAAK,GAAG;YAAE,OAAO,IAAI,GAAG,KAAK,CAAC;QACzC,OAAO,IAAI,IAAI,KAAK,CAAC;IACvB,CAAC;IACD,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,EAAE,CAAC;QACzC,MAAM,IAAI,SAAS,CAAC,+BAA+B,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC;IACjE,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,MAAM,IAAI,SAAS,CACjB,gCAAgC,IAAI,CAAC,EAAE,QAAQ,QAAQ,CAAC,IAAI,CAAC,QAAQ,QAAQ,CAAC,KAAK,CAAC,EAAE,CACvF,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,KAAK,KAAK,CAAC;IAC7B,OAAO,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;AAC3C,CAAC;AAED,SAAS,SAAS,CAAC,IAA+B,EAAE,OAAgB;IAClE,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC3C,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,MAAM,IAAI,SAAS,CAAC,qCAAqC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/evals/runner.d.ts

@@ -0,0 +1,94 @@
+/**
+ * Running a dataset through a gateway, once or N times.
+ *
+ * The split with {@link import('./gate.js')} is deliberate: this module *produces*
+ * numbers, the gate *judges* them. Keeping them apart is what lets the sampling loop
+ * stay boring and the decision logic stay testable without a provider.
+ */
+import type { ModelGateway } from '../gateway.js';
+import { CompletionRequest } from '../types.js';
+import type { Dataset, EvalCase } from './dataset.js';
+import { EvalRun } from './gate.js';
+import type { Metric, MetricResult } from './metrics.js';
+/** Anything that turns a case into a prediction. */
+export type Runner = (entry: EvalCase) => Promise<string>;
+/** One case's prediction and its per-metric scores, for inspection. */
+export declare class CaseOutcome {
+    readonly caseId: string;
+    readonly prediction: string;
+    readonly scores: readonly MetricResult[];
+    constructor(init: {
+        readonly caseId: string;
+        readonly prediction: string;
+        readonly scores?: readonly MetricResult[];
+    });
+    get passed(): boolean;
+}
+/** A single pass over a dataset, keeping per-case detail. */
+export declare class DatasetRun {
+    readonly dataset: string;
+    readonly datasetVersion: string;
+    readonly outcomes: readonly CaseOutcome[];
+    constructor(init: {
+        readonly dataset: string;
+        readonly datasetVersion: string;
+        readonly outcomes: readonly CaseOutcome[];
+    });
+    /** Mean score per metric across the cases of this pass. */
+    means(): Record<string, number>;
+    failures(): CaseOutcome[];
+    outcome(caseId: string): CaseOutcome;
+    summary(): string;
+}
+export interface GatewayRunnerOptions {
+    readonly system?: string;
+    readonly temperature?: number;
+    readonly maxOutputTokens?: number;
+    /** Supply a prompt-registry-backed request instead of the default shape. */
+    readonly requestFor?: (entry: EvalCase) => CompletionRequest;
+    readonly fallbacks?: readonly string[];
+}
+/**
+ * Adapts a {@link ModelGateway} into a {@link Runner}.
+ *
+ * `requestFor` exists so a caller can supply a prompt-registry-backed request (with its
+ * own system message and version) instead of the default one-message-for-`case.input`
+ * shape. Keeping it an option rather than a subclass is enough: the only thing that
+ * varies is how a case becomes a request.
+ */
+export declare class GatewayRunner {
+    private readonly _gateway;
+    private readonly _alias;
+    private readonly _fallbacks;
+    private readonly _temperature;
+    private readonly _maxOutputTokens;
+    private readonly _system;
+    private readonly _requestFor;
+    constructor(gateway: ModelGateway, alias: string, options?: GatewayRunnerOptions);
+    get alias(): string;
+    get costUsd(): number;
+    run(entry: EvalCase): Promise<string>;
+    private defaultRequest;
+}
+/**
+ * A judge call backed by the gateway, for {@link import('./metrics.js').LlmJudge}.
+ *
+ * Note what this does *not* do: it does not make the judge deterministic. It is still
+ * unfit for a CI threshold.
+ */
+export declare function gatewayJudge(gateway: ModelGateway, alias: string, options?: {
+    readonly system?: string;
+}): (prompt: string) => Promise<string>;
+/** One pass over `dataset`, scoring every prediction with every metric. */
+export declare function evaluateOnce(dataset: Dataset, runner: Runner, metrics: readonly Metric[]): Promise<DatasetRun>;
+/**
+ * Sample `dataset` `runs` times, recording one mean per metric per run.
+ *
+ * Every run makes fresh calls, which is the point: the spread across runs is the
+ * measurement of how much this system varies when nothing has changed. Without it there
+ * is nothing to compare a "regression" against.
+ */
+export declare function runEvaluation(dataset: Dataset, runner: Runner, metrics: readonly Metric[], options?: {
+    readonly runs?: number;
+}): Promise<EvalRun>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/evals/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAElD,OAAO,EAAE,iBAAiB,EAAW,MAAM,aAAa,CAAC;AACzD,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AACtD,OAAO,EAAE,OAAO,EAAa,MAAM,WAAW,CAAC;AAC/C,OAAO,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAEzD,oDAAoD;AACpD,MAAM,MAAM,MAAM,GAAG,CAAC,KAAK,EAAE,QAAQ,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;AAE1D,uEAAuE;AACvE,qBAAa,WAAW;IACtB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,MAAM,EAAE,SAAS,YAAY,EAAE,CAAC;IAEzC,YAAY,IAAI,EAAE;QAChB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;QACxB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;QAC5B,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,YAAY,EAAE,CAAC;KAC3C,EAIA;IAED,IAAI,MAAM,IAAI,OAAO,CAEpB;CACF;AAQD,6DAA6D;AAC7D,qBAAa,UAAU;IACrB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,CAAC;IAE1C,YAAY,IAAI,EAAE;QAChB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;QAChC,QAAQ,CAAC,QAAQ,EAAE,SAAS,WAAW,EAAE,CAAC;KAC3C,EAMA;IAED,2DAA2D;IAC3D,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAY9B;IAED,QAAQ,IAAI,WAAW,EAAE,CAExB;IAED,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAKnC;IAED,OAAO,IAAI,MAAM,CAgBhB;CACF;AAED,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,4EAA4E;IAC5E,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,QAAQ,KAAK,iBAAiB,CAAC;IAC7D,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;CACxC;AAED;;;;;;;GAOG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAe;IACxC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAoB;IAC/C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAqB;IAClD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IACtD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAyC;IAErE,YAAY,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,oBAAyB,EASnF;IAED,IAAI,KAAK,IAAI,MAAM,CAElB;IAED,IAAI,OAAO,IAAI,MAAM,CAEpB;IAEK,GAAG,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAM1C;IAED,OAAO,CAAC,cAAc;CAUvB;AAED;;;;;GAKG;AACH,wBAAgB,YAAY,CAC1B,OAAO,EAAE,YAAY,EACrB,KAAK,EAAE,MAAM,EACb,OAAO,GAAE;IAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GACzC,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAKrC;AAED,2EAA2E;AAC3E,wBAAsB,YAAY,CAChC,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,SAAS,MAAM,EAAE,GACzB,OAAO,CAAC,UAAU,CAAC,CAcrB;AAED;;;;;;GAMG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,SAAS,MAAM,EAAE,EAC1B,OAAO,GAAE;IAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAO,GACvC,OAAO,CAAC,OAAO,CAAC,CAwBlB"}

package/dist/evals/runner.js

@@ -0,0 +1,202 @@
+/**
+ * Running a dataset through a gateway, once or N times.
+ *
+ * The split with {@link import('./gate.js')} is deliberate: this module *produces*
+ * numbers, the gate *judges* them. Keeping them apart is what lets the sampling loop
+ * stay boring and the decision logic stay testable without a provider.
+ */
+// biome-ignore lint/suspicious/noShadowRestrictedNames: the framework's own error name, matching the Python runtime's public surface
+import { EvalError } from '../errors.js';
+import { repr } from '../internal/json.js';
+import { CompletionRequest, Message } from '../types.js';
+import { EvalRun, MetricRun } from './gate.js';
+/** One case's prediction and its per-metric scores, for inspection. */
+export class CaseOutcome {
+    caseId;
+    prediction;
+    scores;
+    constructor(init) {
+        this.caseId = init.caseId;
+        this.prediction = init.prediction;
+        this.scores = init.scores ?? [];
+    }
+    get passed() {
+        return this.scores.every((score) => score.passed);
+    }
+}
+function mean(values) {
+    let total = 0;
+    for (const value of values)
+        total += value;
+    return total / values.length;
+}
+/** A single pass over a dataset, keeping per-case detail. */
+export class DatasetRun {
+    dataset;
+    datasetVersion;
+    outcomes;
+    constructor(init) {
+        if (init.outcomes.length === 0)
+            throw new RangeError('a dataset run needs at least one outcome');
+        this.dataset = init.dataset;
+        this.datasetVersion = init.datasetVersion;
+        this.outcomes = init.outcomes;
+    }
+    /** Mean score per metric across the cases of this pass. */
+    means() {
+        const collected = new Map();
+        for (const outcome of this.outcomes) {
+            for (const score of outcome.scores) {
+                const bucket = collected.get(score.metric);
+                if (bucket === undefined)
+                    collected.set(score.metric, [score.score]);
+                else
+                    bucket.push(score.score);
+            }
+        }
+        const out = {};
+        for (const [name, values] of collected)
+            out[name] = mean(values);
+        return out;
+    }
+    failures() {
+        return this.outcomes.filter((outcome) => !outcome.passed);
+    }
+    outcome(caseId) {
+        for (const outcome of this.outcomes) {
+            if (outcome.caseId === caseId)
+                return outcome;
+        }
+        throw new EvalError(`no outcome recorded for case ${repr(caseId)}`);
+    }
+    summary() {
+        const failed = this.failures();
+        const lines = [
+            `${this.dataset}@${this.datasetVersion}: ${this.outcomes.length - failed.length}/${this.outcomes.length} cases passed`,
+        ];
+        for (const [name, value] of Object.entries(this.means()).sort(([a], [b]) => (a < b ? -1 : 1))) {
+            lines.push(`  ${name}: ${value.toFixed(4)}`);
+        }
+        for (const outcome of failed) {
+            const detail = outcome.scores
+                .filter((score) => !score.passed)
+                .map((score) => `${score.metric}: ${score.detail}`)
+                .join('; ');
+            lines.push(`  FAIL ${outcome.caseId}: ${detail}`);
+        }
+        return lines.join('\n');
+    }
+}
+/**
+ * Adapts a {@link ModelGateway} into a {@link Runner}.
+ *
+ * `requestFor` exists so a caller can supply a prompt-registry-backed request (with its
+ * own system message and version) instead of the default one-message-for-`case.input`
+ * shape. Keeping it an option rather than a subclass is enough: the only thing that
+ * varies is how a case becomes a request.
+ */
+export class GatewayRunner {
+    _gateway;
+    _alias;
+    _fallbacks;
+    _temperature;
+    _maxOutputTokens;
+    _system;
+    _requestFor;
+    constructor(gateway, alias, options = {}) {
+        gateway.resolve(alias);
+        this._gateway = gateway;
+        this._alias = alias;
+        this._fallbacks = options.fallbacks ?? [];
+        this._temperature = options.temperature;
+        this._maxOutputTokens = options.maxOutputTokens;
+        this._system = options.requestFor === undefined ? options.system : undefined;
+        this._requestFor = options.requestFor ?? ((entry) => this.defaultRequest(entry));
+    }
+    get alias() {
+        return this._alias;
+    }
+    get costUsd() {
+        return this._gateway.ledger.costUsd;
+    }
+    async run(entry) {
+        const request = this._requestFor(entry);
+        const result = await this._gateway.generate(this._alias, request, {
+            fallbacks: this._fallbacks,
+        });
+        return result.text;
+    }
+    defaultRequest(entry) {
+        const messages = [];
+        if (this._system !== undefined)
+            messages.push(Message.system(this._system));
+        messages.push(Message.user(entry.input));
+        return new CompletionRequest({
+            messages,
+            temperature: this._temperature,
+            maxOutputTokens: this._maxOutputTokens,
+        });
+    }
+}
+/**
+ * A judge call backed by the gateway, for {@link import('./metrics.js').LlmJudge}.
+ *
+ * Note what this does *not* do: it does not make the judge deterministic. It is still
+ * unfit for a CI threshold.
+ */
+export function gatewayJudge(gateway, alias, options = {}) {
+    return async (prompt) => {
+        const result = await gateway.generate(alias, CompletionRequest.prompt(prompt, options.system));
+        return result.text;
+    };
+}
+/** One pass over `dataset`, scoring every prediction with every metric. */
+export async function evaluateOnce(dataset, runner, metrics) {
+    if (metrics.length === 0)
+        throw new RangeError('at least one metric is required');
+    const outcomes = [];
+    for (const entry of dataset.cases) {
+        const prediction = await runner(entry);
+        const scores = [];
+        for (const metric of metrics)
+            scores.push(await metric.score(entry, prediction));
+        outcomes.push(new CaseOutcome({ caseId: entry.id, prediction, scores }));
+    }
+    return new DatasetRun({
+        dataset: dataset.name,
+        datasetVersion: dataset.version,
+        outcomes,
+    });
+}
+/**
+ * Sample `dataset` `runs` times, recording one mean per metric per run.
+ *
+ * Every run makes fresh calls, which is the point: the spread across runs is the
+ * measurement of how much this system varies when nothing has changed. Without it there
+ * is nothing to compare a "regression" against.
+ */
+export async function runEvaluation(dataset, runner, metrics, options = {}) {
+    const runs = options.runs ?? 5;
+    if (runs < 1)
+        throw new RangeError('runs must be at least 1');
+    if (metrics.length === 0)
+        throw new RangeError('at least one metric is required');
+    const collected = new Map(metrics.map((metric) => [metric.name, []]));
+    for (let pass = 0; pass < runs; pass += 1) {
+        const result = await evaluateOnce(dataset, runner, metrics);
+        const means = result.means();
+        for (const metric of metrics) {
+            const bucket = collected.get(metric.name);
+            if (bucket === undefined)
+                continue;
+            bucket.push(means[metric.name] ?? 0);
+        }
+    }
+    return new EvalRun({
+        dataset: dataset.name,
+        datasetVersion: dataset.version,
+        runs,
+        metrics: metrics.map((metric) => new MetricRun({ metric: metric.name, values: collected.get(metric.name) ?? [] })),
+    });
+}
+//# sourceMappingURL=runner.js.map

package/dist/evals/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/evals/runner.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,qIAAqI;AACrI,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,IAAI,EAAE,MAAM,qBAAqB,CAAC;AAC3C,OAAO,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAEzD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAM/C,uEAAuE;AACvE,MAAM,OAAO,WAAW;IACb,MAAM,CAAS;IACf,UAAU,CAAS;IACnB,MAAM,CAA0B;IAEzC,YAAY,IAIX;QACC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC;QAClC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;IAClC,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;CACF;AAED,SAAS,IAAI,CAAC,MAAyB;IACrC,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,KAAK,IAAI,MAAM;QAAE,KAAK,IAAI,KAAK,CAAC;IAC3C,OAAO,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC;AAC/B,CAAC;AAED,6DAA6D;AAC7D,MAAM,OAAO,UAAU;IACZ,OAAO,CAAS;IAChB,cAAc,CAAS;IACvB,QAAQ,CAAyB;IAE1C,YAAY,IAIX;QACC,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC;YAC5B,MAAM,IAAI,UAAU,CAAC,0CAA0C,CAAC,CAAC;QACnE,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAC5B,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,CAAC;QAC1C,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;IAChC,CAAC;IAED,2DAA2D;IAC3D,KAAK;QACH,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoB,CAAC;QAC9C,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC3C,IAAI,MAAM,KAAK,SAAS;oBAAE,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;;oBAChE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QACD,MAAM,GAAG,GAA2B,EAAE,CAAC;QACvC,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,SAAS;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QACjE,OAAO,GAAG,CAAC;IACb,CAAC;IAED,QAAQ;QACN,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5D,CAAC;IAED,OAAO,CAAC,MAAc;QACpB,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM;gBAAE,OAAO,OAAO,CAAC;QAChD,CAAC;QACD,MAAM,IAAI,SAAS,CAAC,gCAAgC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,OAAO;QACL,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG;YACZ,GAAG,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,cAAc,KAAK,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,eAAe;SACvH,CAAC;QACF,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9F,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/C,CAAC;QACD,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM;iBAC1B,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC;iBAChC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;iBAClD,IAAI,CAAC,IAAI,CAAC,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;CACF;AAWD;;;;;;;GAOG;AACH,MAAM,OAAO,aAAa;IACP,QAAQ,CAAe;IACvB,MAAM,CAAS;IACf,UAAU,CAAoB;IAC9B,YAAY,CAAqB;IACjC,gBAAgB,CAAqB;IACrC,OAAO,CAAqB;IAC5B,WAAW,CAAyC;IAErE,YAAY,OAAqB,EAAE,KAAa,EAAE,OAAO,GAAyB,EAAE;QAClF,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QACxB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,WAAW,CAAC;QACxC,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC;QAChD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QAC7E,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;IACnF,CAAC;IAED,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,IAAI,OAAO;QACT,OAAO,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,KAAe;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACxC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE;YAChE,SAAS,EAAE,IAAI,CAAC,UAAU;SAC3B,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAEO,cAAc,CAAC,KAAe;QACpC,MAAM,QAAQ,GAAc,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;YAAE,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC;QAC5E,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;QACzC,OAAO,IAAI,iBAAiB,CAAC;YAC3B,QAAQ;YACR,WAAW,EAAE,IAAI,CAAC,YAAY;YAC9B,eAAe,EAAE,IAAI,CAAC,gBAAgB;SACvC,CAAC,CAAC;IACL,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAC1B,OAAqB,EACrB,KAAa,EACb,OAAO,GAAiC,EAAE;IAE1C,OAAO,KAAK,EAAE,MAAc,EAAmB,EAAE;QAC/C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC;QAC/F,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC,CAAC;AACJ,CAAC;AAED,2EAA2E;AAC3E,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAgB,EAChB,MAAc,EACd,OAA0B;IAE1B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,CAAC,CAAC;IAClF,MAAM,QAAQ,GAAkB,EAAE,CAAC;IACnC,KAAK,MAAM,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,UAAU,GAAG,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC;QACvC,MAAM,MAAM,GAAmB,EAAE,CAAC;QAClC,KAAK,MAAM,MAAM,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC,CAAC;QACjF,QAAQ,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,IAAI,UAAU,CAAC;QACpB,OAAO,EAAE,OAAO,CAAC,IAAI;QACrB,cAAc,EAAE,OAAO,CAAC,OAAO;QAC/B,QAAQ;KACT,CAAC,CAAC;AACL,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAgB,EAChB,MAAc,EACd,OAA0B,EAC1B,OAAO,GAA+B,EAAE;IAExC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;IAC/B,IAAI,IAAI,GAAG,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,yBAAyB,CAAC,CAAC;IAC9D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,MAAM,IAAI,UAAU,CAAC,iCAAiC,CAAC,CAAC;IAElF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAmB,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC;IACxF,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,IAAI,EAAE,IAAI,IAAI,CAAC,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAC5D,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,EAAE,CAAC;QAC7B,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,MAAM,KAAK,SAAS;gBAAE,SAAS;YACnC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IAED,OAAO,IAAI,OAAO,CAAC;QACjB,OAAO,EAAE,OAAO,CAAC,IAAI;QACrB,cAAc,EAAE,OAAO,CAAC,OAAO;QAC/B,IAAI;QACJ,OAAO,EAAE,OAAO,CAAC,GAAG,CAClB,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC,CAC7F;KACF,CAAC,CAAC;AACL,CAAC"}

package/dist/gateway.d.ts

@@ -0,0 +1,118 @@
+/**
+ * The model gateway: alias resolution, capability negotiation, cost accounting.
+ *
+ * Three responsibilities, in this order:
+ *
+ * 1. **Pinning.** An alias resolves to an exact provider model id. Unknown aliases are
+ *    an error; there is no fall-through to a provider default, because a provider
+ *    default is precisely the floating version a stored eval baseline cannot survive.
+ * 2. **Ordered fallback.** A caller may name fallback aliases. They are tried in the
+ *    order given, and the aliases actually attempted are reported on the result.
+ *    Escalation routing -- "use the strong model if the cheap one is unsure" -- is
+ *    *not* implemented: "unsure" needs an agent loop to define.
+ * 3. **Budgets and tracing.** Every charged call opens a GenAI span and updates the
+ *    ledger.
+ */
+import type { ConcurrencyLimiter } from './concurrency.js';
+import { type Budget, CostLedger, type LedgerEntry } from './cost.js';
+import type { Provider } from './providers/base.js';
+import { type Capability, type CompletionRequest, type CompletionResponse, GenerationResult, type ModelSpec, PinnedModel, TokenUsage } from './types.js';
+/** Called after every successfully charged call. */
+export type CallHook = (entry: LedgerEntry) => void;
+/** An alias table entry: a bare spec, or one already pinned to an alias. */
+export type AliasEntry = ModelSpec | PinnedModel;
+export interface ModelGatewayOptions {
+    readonly providers: Readonly<Record<string, Provider>>;
+    readonly aliases: Readonly<Record<string, AliasEntry>>;
+    readonly ledger?: CostLedger;
+    readonly budget?: Budget;
+    readonly onCall?: readonly CallHook[];
+    readonly tracerName?: string;
+    /**
+     * The application's single limiter, from `deployments.concurrency`.
+     *
+     * A gateway constructed without one is unbounded, which keeps plain request/response
+     * usage free of concurrency surprises. Sharing the instance with the tool executor is
+     * what makes "one limiter, not two" true.
+     */
+    readonly limiter?: ConcurrencyLimiter;
+}
+/** Resolves aliases to pinned models and executes calls within a budget. */
+export declare class ModelGateway {
+    private readonly _providers;
+    private readonly _aliases;
+    private readonly _ledger;
+    private readonly _budget;
+    private readonly _hooks;
+    private readonly _tracerName;
+    private readonly _limiter;
+    constructor(options: ModelGatewayOptions);
+    /** The pinned aliases, sorted. */
+    get aliases(): string[];
+    get ledger(): CostLedger;
+    get budget(): Budget | undefined;
+    /** Resolve an alias to its pinned model, or fail loudly. */
+    resolve(alias: string): PinnedModel;
+    /** The adapter that serves `alias`. Raises for an unknown alias. */
+    providerFor(alias: string): Provider;
+    /**
+     * Assert a pinned model *and* its adapter advertise every capability.
+     *
+     * Both sides are checked: a model that supports tool calling behind an adapter that
+     * does not serialise tool definitions cannot tool-call.
+     */
+    require(alias: string, capabilities: Iterable<Capability>): PinnedModel;
+    /**
+     * Run one call, falling back through `fallbacks` in order.
+     *
+     * A fallback is only tried for a failure another model could plausibly survive: a
+     * provider error or a missing capability. A {@link BudgetExceededError} propagates
+     * immediately -- retrying a different model cannot make an exhausted budget
+     * affordable.
+     */
+    generate(alias: string, request: CompletionRequest, options?: {
+        readonly fallbacks?: readonly string[];
+        readonly budget?: Budget;
+    }): Promise<GenerationResult>;
+    /** Declared but not implemented; see the module docstring. */
+    stream(_alias: string, _request: CompletionRequest): Promise<never>;
+    close(): Promise<void>;
+    /**
+     * What this request's prompt would cost on top of what is already spent.
+     *
+     * Public because a routing decision needs the same number the preflight enforces: an
+     * agent stepping down a price ladder asks whether the *cheaper* alias fits before
+     * switching, and a second estimator would let the two disagree about the same ceiling.
+     */
+    projectedCostUsd(alias: string, request: CompletionRequest): number;
+    /**
+     * Whether this request still fits the budget. A question, never a charge.
+     *
+     * An unset ceiling always fits -- there is nothing to exceed. An *already* exceeded
+     * budget does not fit whatever the prompt, which is what makes a degraded agent stop
+     * rather than step down for every remaining step.
+     */
+    fits(alias: string, request: CompletionRequest, budget: Budget | undefined): boolean;
+    private attempt;
+    /**
+     * Acquire the shared concurrency slot around one provider call.
+     *
+     * The slot wraps the call, not the surrounding bookkeeping, so a cache hit or a
+     * refused call never consumes one, and the slot is released on completion and on
+     * failure alike.
+     */
+    private limit;
+    /**
+     * Refuse a call whose *prompt* alone already breaches the budget.
+     *
+     * Only the prompt is priced, and deliberately over-estimated, because the response
+     * is unknowable before the call. This catches gross overruns early; the post-call
+     * charge is what enforces the ceiling exactly.
+     */
+    private preflight;
+}
+/** Token estimate for a set of strings, at the usual four characters per token. */
+export declare function estimateUsage(texts: string | readonly string[]): TokenUsage;
+/** Re-exported so callers naming a response type do not reach across modules. */
+export type { CompletionResponse };
+//# sourceMappingURL=gateway.d.ts.map

package/dist/gateway.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"gateway.d.ts","sourceRoot":"","sources":["../src/gateway.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,KAAK,MAAM,EAAE,UAAU,EAAyB,KAAK,WAAW,EAAE,MAAM,WAAW,CAAC;AAS7F,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAQpD,OAAO,EACL,KAAK,UAAU,EACf,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,gBAAgB,EAChB,KAAK,SAAS,EACd,WAAW,EACX,UAAU,EACX,MAAM,YAAY,CAAC;AAEpB,oDAAoD;AACpD,MAAM,MAAM,QAAQ,GAAG,CAAC,KAAK,EAAE,WAAW,KAAK,IAAI,CAAC;AAEpD,4EAA4E;AAC5E,MAAM,MAAM,UAAU,GAAG,SAAS,GAAG,WAAW,CAAC;AAEjD,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC,CAAC;IACvD,QAAQ,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC;IACvD,QAAQ,CAAC,MAAM,CAAC,EAAE,UAAU,CAAC;IAC7B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,MAAM,CAAC,EAAE,SAAS,QAAQ,EAAE,CAAC;IACtC,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B;;;;;;OAMG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,kBAAkB,CAAC;CACvC;AAED,4EAA4E;AAC5E,qBAAa,YAAY;IACvB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAgC;IAC3D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiC;IAC1D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAa;IACrC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;IAC7C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiC;IAE1D,YAAY,OAAO,EAAE,mBAAmB,EA4BvC;IAED,kCAAkC;IAClC,IAAI,OAAO,IAAI,MAAM,EAAE,CAEtB;IAED,IAAI,MAAM,IAAI,UAAU,CAEvB;IAED,IAAI,MAAM,IAAI,MAAM,GAAG,SAAS,CAE/B;IAED,4DAA4D;IAC5D,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,WAAW,CAIlC;IAED,oEAAoE;IACpE,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,QAAQ,CAInC;IAED;;;;;OAKG;IACH,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,WAAW,CAWtE;IAED;;;;;;;OAOG;IACG,QAAQ,CACZ,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,iBAAiB,EAC1B,OAAO,GAAE;QAAE,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;QAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAO,GACjF,OAAO,CAAC,gBAAgB,CAAC,CAuB3B;IAED,8DAA8D;IACxD,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,GAAG,OAAO,CAAC,KAAK,CAAC,CAKxE;IAEK,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAI3B;IAED;;;;;;OAMG;IACH,gBAAgB,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,MAAM,CAOlE;IAED;;;;;;OAMG;IACH,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,CAInF;YAEa,OAAO;IA2DrB;;;;;;OAMG;YACW,KAAK;IAKnB;;;;;;OAMG;IACH,OAAO,CAAC,SAAS;CAOlB;AAED,mFAAmF;AACnF,wBAAgB,aAAa,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,MAAM,EAAE,GAAG,UAAU,CAK3E;AAED,iFAAiF;AACjF,YAAY,EAAE,kBAAkB,EAAE,CAAC"}