@adaptic/utils 0.0.1014 → 0.0.1016

package/dist/types/llm/eval/comparators.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparators.d.ts","sourceRoot":"","sources":["../../../../src/llm/eval/comparators.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,SAAS,CAAC;AAEhE;;;;;;GAMG;AACH,eAAO,MAAM,sBAAsB,IAAI,CAAC;AAExC;;;;;;;GAOG;AACH,eAAO,MAAM,wBAAwB,OAAO,CAAC;AAE7C,8FAA8F;AAC9F,eAAO,MAAM,cAAc,MAAM,CAAC;AAiBlC,qDAAqD;AACrD,MAAM,WAAW,eAAe;IAC9B,oFAAoF;IACpF,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,wFAAwF;IACxF,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IACvC,4CAA4C;IAC5C,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,yDAAyD;AACzD,MAAM,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,eAAe,KAAK,OAAO,CAAC;AAkF7D;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB,EAAE,UACsB,CAAC;AAE5D;;;;;;;;;GASG;AACH,eAAO,MAAM,oBAAoB,EAAE,UACmB,CAAC;AAEvD;;;;;;;;;GASG;AACH,eAAO,MAAM,iBAAiB,EAAE,UA8B/B,CAAC;AAEF;;;;;;;;;GASG;AACH,eAAO,MAAM,iBAAiB,EAAE,UA8B/B,CAAC;AAEF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,iBAAiB,EAAE,UA2B/B,CAAC;AAEF;;;;;;GAMG;AACH,eAAO,MAAM,WAAW,EAAE,QAAQ,CAAC,MAAM,CAAC,aAAa,EAAE,UAAU,CAAC,CAMnE,CAAC;AAEF;;;;;;;GAOG;AACH,eAAO,MAAM,oBAAoB,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,CAAC,CAKrF,CAAC"}

package/dist/types/llm/eval/coverage.d.ts

@@ -0,0 +1,44 @@
+/**
+ * Coverage of the harness itself.
+ *
+ * A gate is only worth its CI minutes if every assertion it claims to enforce
+ * has been demonstrated to fire in both directions. "Implemented" is not the
+ * bar — a comparator that always returns `pass` is implemented. The bar is
+ * EXECUTED EVIDENCE: for each assertion, a fixture that this run graded as
+ * passing and a fixture that this run graded as failing.
+ *
+ * The route table is the other half of the question. An alias whose eval gate
+ * maps to no assertion bundle is an alias nothing grades, and it would sail
+ * through a migration unmeasured, so the coverage check reads the table rather
+ * than a list someone maintained by hand.
+ *
+ * @module llm/eval/coverage
+ */
+import type { LlmRouteTable } from "../types";
+import type { EvalAssertion, EvalStatus } from "./types";
+/** Assertions a run actually demonstrated, in each direction. */
+export interface ObservedAssertions {
+    /** Assertions this run graded as passing on at least one fixture. */
+    readonly green: readonly EvalAssertion[];
+    /** Assertions this run graded as failing on at least one fixture. */
+    readonly red: readonly EvalAssertion[];
+}
+/** The outcome of the coverage check. */
+export interface CoverageReport {
+    /** Assertions required by at least one eval gate in the route table. */
+    readonly required: readonly EvalAssertion[];
+    /** Everything missing; empty is the only passing state. */
+    readonly problems: readonly string[];
+    /** PASSED only when nothing is missing. */
+    readonly status: EvalStatus;
+}
+/**
+ * Check that every assertion the route table requires is implemented and was
+ * demonstrated in both directions by this run.
+ *
+ * @param observed Assertions this run graded as passing and as failing.
+ * @param table The route table to read; defaults to the canonical one.
+ * @returns The coverage report.
+ */
+export declare function computeCoverage(observed: ObservedAssertions, table?: LlmRouteTable): CoverageReport;
+//# sourceMappingURL=coverage.d.ts.map

package/dist/types/llm/eval/coverage.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"coverage.d.ts","sourceRoot":"","sources":["../../../../src/llm/eval/coverage.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAGH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAG9C,OAAO,KAAK,EAAE,aAAa,EAAY,UAAU,EAAE,MAAM,SAAS,CAAC;AAEnE,iEAAiE;AACjE,MAAM,WAAW,kBAAkB;IACjC,qEAAqE;IACrE,QAAQ,CAAC,KAAK,EAAE,SAAS,aAAa,EAAE,CAAC;IACzC,qEAAqE;IACrE,QAAQ,CAAC,GAAG,EAAE,SAAS,aAAa,EAAE,CAAC;CACxC;AAED,yCAAyC;AACzC,MAAM,WAAW,cAAc;IAC7B,wEAAwE;IACxE,QAAQ,CAAC,QAAQ,EAAE,SAAS,aAAa,EAAE,CAAC;IAC5C,2DAA2D;IAC3D,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,2CAA2C;IAC3C,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC7B;AA4BD;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,kBAAkB,EAC5B,KAAK,GAAE,aAA0B,GAChC,cAAc,CAwChB"}

package/dist/types/llm/eval/golden-set.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Parsing and structural validation of golden sets and candidate runs.
+ *
+ * A golden set is evidence, and evidence that is not validated on the way in is
+ * evidence nobody can rely on later. Every field is checked here, at the edge,
+ * so that by the time a comparator sees a number the only remaining question is
+ * the one the comparator exists to answer.
+ *
+ * Validation is strict in one particular direction: a set that declares an
+ * assertion must carry everything that assertion needs. A set asserting
+ * `schema-valid` without a response shape, or `tool-call` without a tool
+ * contract, is rejected rather than quietly graded on the assertions it does
+ * support — a gate that drops the check it cannot perform is a gate that gets
+ * weaker exactly where the evidence is thinnest.
+ *
+ * @module llm/eval/golden-set
+ */
+import type { CandidateRun, GoldenSet } from "./types";
+/** Thrown when a golden set or candidate run is not well formed. */
+export declare class GoldenSetError extends Error {
+    /** Where the malformed document came from, so the message is actionable. */
+    readonly origin: string;
+    /**
+     * @param origin Where the document came from.
+     * @param detail What is wrong with it.
+     */
+    constructor(origin: string, detail: string);
+}
+/**
+ * Parse and validate a golden set.
+ *
+ * @param raw The parsed JSON document.
+ * @param origin Where it came from, quoted in any error.
+ * @returns The validated golden set.
+ * @throws {GoldenSetError} When the document is not a valid golden set.
+ */
+export declare function parseGoldenSet(raw: unknown, origin: string): GoldenSet;
+/**
+ * Parse and validate a candidate run.
+ *
+ * @param raw The parsed JSON document.
+ * @param origin Where it came from, quoted in any error.
+ * @returns The validated candidate run.
+ * @throws {GoldenSetError} When the document is not a valid candidate run.
+ */
+export declare function parseCandidateRun(raw: unknown, origin: string): CandidateRun;
+//# sourceMappingURL=golden-set.d.ts.map

package/dist/types/llm/eval/golden-set.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"golden-set.d.ts","sourceRoot":"","sources":["../../../../src/llm/eval/golden-set.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAKH,OAAO,KAAK,EACV,YAAY,EAIZ,SAAS,EAOV,MAAM,SAAS,CAAC;AAEjB,oEAAoE;AACpE,qBAAa,cAAe,SAAQ,KAAK;IACvC,4EAA4E;IAC5E,SAAgB,MAAM,EAAE,MAAM,CAAC;IAE/B;;;OAGG;gBACgB,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;CAKlD;AA8QD;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,SAAS,CAyJtE;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,GAAG,YAAY,CAa5E"}

package/dist/types/llm/eval/index.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Public surface of the LLM migration eval harness.
+ *
+ * The harness grades a candidate model against a recorded incumbent baseline on
+ * captured golden sets, and renders every verdict by code from recorded data —
+ * PD-6 forbids agent judgement from standing in for a gate, and an exported
+ * surface with no "just tell me if this looks fine" entry point is what makes
+ * that structural rather than aspirational.
+ *
+ * @module llm/eval
+ */
+export { COMPARATORS, EVAL_GATE_ASSERTIONS, JUDGE_TOLERANCE_FRACTION, MATCH_TOLERANCE_POINTS, RATE_TO_POINTS, compareJudgeScore, compareLatencyP95, compareMatchScore, compareSchemaValidRate, compareValidCallRate, } from "./comparators";
+export type { Comparator, ComparatorInput } from "./comparators";
+export { computeCoverage } from "./coverage";
+export type { CoverageReport, ObservedAssertions } from "./coverage";
+export { GoldenSetError, parseCandidateRun, parseGoldenSet } from "./golden-set";
+export { F1_SCALE_POINTS, fieldF1, jsonEquals, leafFields, p95, satisfiesShape } from "./json-shape";
+export { JUDGE_SCORE_MAX, JUDGE_SCORE_MIN, JudgeNotPinnedError, PINNED_JUDGE, assertPinnedJudge, scoreCandidateRun, scoreWithPinnedJudge, } from "./judge";
+export type { JudgeRequest, PinnedJudgeIdentity, ResolvedJudge } from "./judge";
+export { MAX_STRUCTURED_RETRIES, deriveMetrics, isValidToolCall } from "./metrics";
+export type { MetricSample } from "./metrics";
+export { BASELINE_ATTESTATION_TOLERANCE, evaluateRun, evaluateSet, formatRunReport, formatVerdict, } from "./run";
+export type { EvalPair, EvaluateOptions } from "./run";
+export type { BaselineMetrics, CandidateRun, EvalAssertion, EvalGate, EvalStatus, GoldenCase, GoldenSet, IncumbentBaseline, JsonShape, MatchMetric, MetricUnit, RecordedResult, RecordedToolCall, RunReport, SetReport, ToolExpectation, Verdict, } from "./types";
+//# sourceMappingURL=index.d.ts.map

package/dist/types/llm/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/llm/eval/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACL,WAAW,EACX,oBAAoB,EACpB,wBAAwB,EACxB,sBAAsB,EACtB,cAAc,EACd,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,EACjB,sBAAsB,EACtB,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,YAAY,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEjE,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,YAAY,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAErE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEjF,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,GAAG,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAErG,OAAO,EACL,eAAe,EACf,eAAe,EACf,mBAAmB,EACnB,YAAY,EACZ,iBAAiB,EACjB,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,SAAS,CAAC;AACjB,YAAY,EAAE,YAAY,EAAE,mBAAmB,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAEhF,OAAO,EAAE,sBAAsB,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,WAAW,CAAC;AACnF,YAAY,EAAE,YAAY,EAAE,MAAM,WAAW,CAAC;AAE9C,OAAO,EACL,8BAA8B,EAC9B,WAAW,EACX,WAAW,EACX,eAAe,EACf,aAAa,GACd,MAAM,OAAO,CAAC;AACf,YAAY,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,OAAO,CAAC;AAEvD,YAAY,EACV,eAAe,EACf,YAAY,EACZ,aAAa,EACb,QAAQ,EACR,UAAU,EACV,UAAU,EACV,SAAS,EACT,iBAAiB,EACjB,SAAS,EACT,WAAW,EACX,UAAU,EACV,cAAc,EACd,gBAAgB,EAChB,SAAS,EACT,SAAS,EACT,eAAe,EACf,OAAO,GACR,MAAM,SAAS,CAAC"}

package/dist/types/llm/eval/json-shape.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Structural comparison primitives the comparators are built from.
+ *
+ * These are separated from the comparators so that "did this answer parse" and
+ * "was this answer right" stay distinct measurements. Conflating them hides the
+ * most useful diagnostic in a model swap: a candidate can be perfectly
+ * well-formed and consistently wrong, or consistently right and badly
+ * formatted, and those two have opposite remedies.
+ *
+ * The validator honours a documented subset of JSON Schema and adds no
+ * dependency. An unrecognised keyword is ignored rather than treated as
+ * satisfied, so the subset can only ever be stricter than a caller expects,
+ * never more permissive.
+ *
+ * @module llm/eval/json-shape
+ */
+import type { JsonShape } from "./types";
+/** Points on the F1 scale, so a tolerance expressed "in points" has a fixed meaning. */
+export declare const F1_SCALE_POINTS = 100;
+/**
+ * Whether a value satisfies a shape.
+ *
+ * @param value The value to check.
+ * @param shape The shape it must satisfy.
+ * @returns Whether the value satisfies the shape.
+ */
+export declare function satisfiesShape(value: unknown, shape: JsonShape): boolean;
+/**
+ * Structural equality over JSON values.
+ *
+ * Order-sensitive for arrays and order-insensitive for object keys, which is
+ * what JSON itself means: two objects with the same entries are the same
+ * answer, while a reordered list is a different one.
+ *
+ * @param left The first value.
+ * @param right The second value.
+ * @returns Whether the two are structurally equal.
+ */
+export declare function jsonEquals(left: unknown, right: unknown): boolean;
+/**
+ * Flatten a JSON value into dotted leaf paths paired with their serialised values.
+ *
+ * Field-level F1 needs a set of comparable atoms, and a leaf path is the
+ * natural one for extraction output: it credits a candidate for the fields it
+ * got right instead of scoring the whole record all-or-nothing, which is the
+ * difference between a metric that can move by one point and one that can only
+ * move by whole cases.
+ *
+ * @param value The value to flatten.
+ * @param prefix Path prefix used by the recursion.
+ * @returns Leaf paths mapped to their serialised values.
+ */
+export declare function leafFields(value: unknown, prefix?: string): Map<string, string>;
+/**
+ * Field-level F1 between a produced value and the reference, in points.
+ *
+ * @param produced What the model returned.
+ * @param expected The reference answer.
+ * @returns F1 on a 0-100 point scale; 0 when nothing matched.
+ */
+export declare function fieldF1(produced: unknown, expected: unknown): number;
+/**
+ * The p95 of a sample by nearest-rank.
+ *
+ * Nearest-rank rather than an interpolating estimator because a latency gate
+ * must name a value the system actually produced; an interpolated p95 is a
+ * number no request ever took, and a gate is easier to trust when its threshold
+ * is an observation.
+ *
+ * @param values The sample.
+ * @returns The p95 value, or `null` for an empty sample.
+ */
+export declare function p95(values: readonly number[]): number | null;
+//# sourceMappingURL=json-shape.d.ts.map

package/dist/types/llm/eval/json-shape.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-shape.d.ts","sourceRoot":"","sources":["../../../../src/llm/eval/json-shape.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAEzC,wFAAwF;AACxF,eAAO,MAAM,eAAe,MAAM,CAAC;AAEnC;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,OAAO,CA6BxE;AA4BD;;;;;;;;;;GAUG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,CA6BjE;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,EAAE,MAAM,SAAK,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAqB3E;AAED;;;;;;GAMG;AACH,wBAAgB,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,GAAG,MAAM,CAepE;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,GAAG,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,GAAG,IAAI,CAQ5D"}

package/dist/types/llm/eval/judge.d.ts

@@ -0,0 +1,131 @@
+/**
+ * The pinned judge, and the guard that keeps it pinned.
+ *
+ * PD-6 states that the judge model is pinned and never auto-swapped. That is a
+ * stronger requirement than "configured": a judge that could move would make
+ * every gate it decided incomparable with the gates decided before the move,
+ * because the scores either side of it were produced against different
+ * standards. A migration programme whose bar drifts under it cannot demonstrate
+ * anything.
+ *
+ * The pin is therefore recorded in TWO places that must agree — this module and
+ * the route table — and a judged assertion refuses to run when they disagree.
+ * The duplication is deliberate. A pin that lived only in the table would be
+ * satisfied by any model the table happened to name, so changing the table
+ * would silently change the standard; requiring both to move makes a judge swap
+ * an explicit, reviewable act in the same change that re-baselines the sets.
+ *
+ * @module llm/eval/judge
+ */
+import type { LlmAlias, LlmRouteTable } from "../types";
+import type { CandidateRun, GoldenSet } from "./types";
+/** Lowest score the judge rubric may return. */
+export declare const JUDGE_SCORE_MIN = 0;
+/** Highest score the judge rubric may return, matching the point scale the comparators use. */
+export declare const JUDGE_SCORE_MAX = 100;
+/** The identity a judged assertion requires the route table to resolve to. */
+export interface PinnedJudgeIdentity {
+    /** The only alias a judged assertion may be served by. */
+    readonly alias: LlmAlias;
+    /** The provider the judge route must name. */
+    readonly provider: string;
+    /** The exact model id the judge route must name. */
+    readonly modelId: string;
+}
+/**
+ * The pinned judge identity.
+ *
+ * This is the one place in the codebase where naming a vendor model string is
+ * the requirement rather than a violation of it: PD-5 forbids model strings in
+ * APPLICATION code so that routing stays in config, while PD-6 requires the
+ * judge to be pinned so that grading stays comparable. Pinning by anything
+ * looser — a family, a provider, "whatever the table says" — would not pin
+ * anything.
+ */
+export declare const PINNED_JUDGE: PinnedJudgeIdentity;
+/**
+ * Thrown when a judged assertion is asked to run against a judge that is not
+ * the pinned one.
+ *
+ * A refusal rather than a warning or a downgraded score: a judged gate decided
+ * by an unknown judge looks exactly like a judged gate decided by the right
+ * one, so the only safe outcome is to produce no verdict at all.
+ */
+export declare class JudgeNotPinnedError extends Error {
+    /** The alias the caller tried to judge through. */
+    readonly alias: string;
+    /**
+     * @param alias The alias the caller tried to judge through.
+     * @param detail What specifically failed the pin check.
+     */
+    constructor(alias: string, detail: string);
+}
+/** The judge identity a route table actually resolves to. */
+export interface ResolvedJudge {
+    /** The alias that was verified. */
+    readonly alias: LlmAlias;
+    /** The provider the table names for it. */
+    readonly provider: string;
+    /** The model id the table names for it. */
+    readonly modelId: string;
+}
+/**
+ * Verify that an alias is the pinned judge, or refuse.
+ *
+ * @param alias The alias a judged assertion would be served by.
+ * @param table The route table to verify against; defaults to the canonical one.
+ * @returns The resolved judge identity when every pin condition holds.
+ * @throws {JudgeNotPinnedError} When any pin condition fails.
+ */
+export declare function assertPinnedJudge(alias: LlmAlias, table?: LlmRouteTable): ResolvedJudge;
+/** One case put to the judge. */
+export interface JudgeRequest {
+    /** The grading rubric, stated on the golden set rather than improvised per call. */
+    readonly rubric: string;
+    /** The original input the answer responds to. */
+    readonly input: string;
+    /** The reference answer. */
+    readonly reference: string;
+    /** The answer being graded. */
+    readonly answer: string;
+}
+/**
+ * Score one answer with the pinned judge.
+ *
+ * The pin is verified before the call, not after, so a mis-pinned judge costs
+ * nothing and produces nothing rather than producing a score that would have to
+ * be retracted.
+ *
+ * @param request The case to grade.
+ * @param options Overrides used by the harness's own pinning check.
+ * @param options.alias The alias to judge through; only the pinned one is accepted.
+ * @param options.table The route table to verify the pin against.
+ * @returns The score, on the same 0-100 point scale the comparators use.
+ * @throws {JudgeNotPinnedError} When the judge is not the pinned one.
+ */
+export declare function scoreWithPinnedJudge(request: JudgeRequest, options?: {
+    alias?: LlmAlias;
+    table?: LlmRouteTable;
+}): Promise<number>;
+/**
+ * Fill in any missing judge scores on a candidate run, using the pinned judge.
+ *
+ * The pin is verified once before the first call rather than per case, so a
+ * mis-pinned judge cannot grade part of a set before it is caught — a half-
+ * graded set is worse than an ungraded one, because its mean looks like a
+ * measurement.
+ *
+ * @param set The golden set being graded, which supplies the reference answers.
+ * @param candidate The candidate run whose answers need scoring.
+ * @param rubric The grading rubric, stated on the set rather than improvised.
+ * @param options Overrides used by the harness's own pinning check.
+ * @param options.alias The alias to judge through; only the pinned one is accepted.
+ * @param options.table The route table to verify the pin against.
+ * @returns A candidate run with a judge score on every case.
+ * @throws {JudgeNotPinnedError} When the judge is not the pinned one.
+ */
+export declare function scoreCandidateRun(set: GoldenSet, candidate: CandidateRun, rubric: string, options?: {
+    alias?: LlmAlias;
+    table?: LlmRouteTable;
+}): Promise<CandidateRun>;
+//# sourceMappingURL=judge.d.ts.map

package/dist/types/llm/eval/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../../../src/llm/eval/judge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAIH,OAAO,KAAK,EACV,QAAQ,EACR,aAAa,EAEd,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EAAE,YAAY,EAAE,SAAS,EAAkB,MAAM,SAAS,CAAC;AAEvE,gDAAgD;AAChD,eAAO,MAAM,eAAe,IAAI,CAAC;AAEjC,+FAA+F;AAC/F,eAAO,MAAM,eAAe,MAAM,CAAC;AAEnC,8EAA8E;AAC9E,MAAM,WAAW,mBAAmB;IAClC,0DAA0D;IAC1D,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC;IACzB,8CAA8C;IAC9C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,oDAAoD;IACpD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;;;GASG;AACH,eAAO,MAAM,YAAY,EAAE,mBAI1B,CAAC;AAEF;;;;;;;GAOG;AACH,qBAAa,mBAAoB,SAAQ,KAAK;IAC5C,mDAAmD;IACnD,SAAgB,KAAK,EAAE,MAAM,CAAC;IAE9B;;;OAGG;gBACgB,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;CASjD;AAED,6DAA6D;AAC7D,MAAM,WAAW,aAAa;IAC5B,mCAAmC;IACnC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC;IACzB,2CAA2C;IAC3C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,2CAA2C;IAC3C,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAC/B,KAAK,EAAE,QAAQ,EACf,KAAK,GAAE,aAA0B,GAChC,aAAa,CAiDf;AAED,iCAAiC;AACjC,MAAM,WAAW,YAAY;IAC3B,oFAAoF;IACpF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,iDAAiD;IACjD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,4BAA4B;IAC5B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,+BAA+B;IAC/B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAyBD;;;;;;;;;;;;;GAaG;AACH,wBAAsB,oBAAoB,CACxC,OAAO,EAAE,YAAY,EACrB,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,QAAQ,CAAC;IAAC,KAAK,CAAC,EAAE,aAAa,CAAA;CAAO,GACxD,OAAO,CAAC,MAAM,CAAC,CAqBjB;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,iBAAiB,CACrC,GAAG,EAAE,SAAS,EACd,SAAS,EAAE,YAAY,EACvB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE;IAAE,KAAK,CAAC,EAAE,QAAQ,CAAC;IAAC,KAAK,CAAC,EAAE,aAAa,CAAA;CAAO,GACxD,OAAO,CAAC,YAAY,CAAC,CA0BvB"}

package/dist/types/llm/eval/metrics.d.ts

@@ -0,0 +1,51 @@
+/**
+ * Derivation of eval metrics from recorded model answers.
+ *
+ * Both sides of every comparison are measured HERE, by the same code, from
+ * records of the same shape. That symmetry is the point: an incumbent baseline
+ * computed by one routine and a candidate score computed by another would
+ * differ by the routines as much as by the models, and the gate would be
+ * measuring its own implementation.
+ *
+ * A metric that cannot be derived from the records present is returned as
+ * `undefined` rather than as a zero or a default. Zero is a measurement; absence
+ * is not, and the comparators depend on being able to tell them apart in order
+ * to render `indeterminate` instead of a verdict they have not earned.
+ *
+ * @module llm/eval/metrics
+ */
+import type { BaselineMetrics, GoldenCase, GoldenSet, RecordedResult, ToolExpectation } from "./types";
+/**
+ * Structured retries a tool-call site may spend before the answer stops
+ * counting as a valid call.
+ *
+ * Section 6 permits exactly one: the retry exists to recover a malformed call
+ * from a model that otherwise understood the request, and a site that needs
+ * more than one has not produced a valid call — it has been coached into one,
+ * which is a different capability from the one being measured.
+ */
+export declare const MAX_STRUCTURED_RETRIES = 1;
+/** One graded case paired with the answer being scored. */
+export interface MetricSample {
+    /** The golden case. */
+    readonly goldenCase: GoldenCase;
+    /** The answer under measurement — incumbent or candidate. */
+    readonly result: RecordedResult;
+}
+/**
+ * Whether one recorded answer is a valid tool call under the site's contract.
+ *
+ * @param result The recorded answer.
+ * @param expectation The tool contract the site requires.
+ * @returns Whether the answer is a valid call within the permitted retry budget.
+ */
+export declare function isValidToolCall(result: RecordedResult, expectation: ToolExpectation): boolean;
+/**
+ * Derive every metric a set's assertions need, from one side's answers.
+ *
+ * @param set The golden set, which fixes what is measured and how.
+ * @param samples Cases paired with the answers being scored.
+ * @returns The derived metrics; a metric the records cannot support is absent.
+ */
+export declare function deriveMetrics(set: GoldenSet, samples: readonly MetricSample[]): BaselineMetrics;
+//# sourceMappingURL=metrics.d.ts.map

package/dist/types/llm/eval/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../../src/llm/eval/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAGH,OAAO,KAAK,EACV,eAAe,EACf,UAAU,EACV,SAAS,EACT,cAAc,EACd,eAAe,EAChB,MAAM,SAAS,CAAC;AAEjB;;;;;;;;GAQG;AACH,eAAO,MAAM,sBAAsB,IAAI,CAAC;AAExC,2DAA2D;AAC3D,MAAM,WAAW,YAAY;IAC3B,uBAAuB;IACvB,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;IAChC,6DAA6D;IAC7D,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;CACjC;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,cAAc,EACtB,WAAW,EAAE,eAAe,GAC3B,OAAO,CAwBT;AAuBD;;;;;;GAMG;AACH,wBAAgB,aAAa,CAC3B,GAAG,EAAE,SAAS,EACd,OAAO,EAAE,SAAS,YAAY,EAAE,GAC/B,eAAe,CAmDjB"}

package/dist/types/llm/eval/run.d.ts

@@ -0,0 +1,97 @@
+/**
+ * Grading one candidate run against one golden set, and a whole evaluation
+ * against many.
+ *
+ * The orchestration here holds two invariants that the comparators cannot hold
+ * on their own.
+ *
+ * The bar is the ATTESTED baseline, not a number recomputed at grading time —
+ * so the bar a candidate clears is the same bar every earlier candidate cleared,
+ * and a re-baselining is a visible edit to the set rather than a silent
+ * consequence of running the harness again. The recomputation still happens,
+ * as an integrity check: when the attested baseline and the records it claims
+ * to summarise disagree, the set renders `indeterminate` instead of grading
+ * against a bar that has been moved by hand.
+ *
+ * And an incomplete candidate run is never graded. Scoring the subset of cases
+ * that happened to produce an answer silently reweights the set toward whatever
+ * the candidate found easy, which is the most flattering possible error a model
+ * comparison can make.
+ *
+ * @module llm/eval/run
+ */
+import type { LlmAlias, LlmRouteTable } from "../types";
+import type { CandidateRun, GoldenSet, RunReport, SetReport, Verdict } from "./types";
+/**
+ * Absolute agreement required between an attested baseline metric and the same
+ * metric recomputed from the set's own records.
+ *
+ * Tight enough that only floating-point representation error fits inside it: the
+ * check exists to catch a baseline edited independently of its evidence, and a
+ * generous tolerance would let exactly that through.
+ */
+export declare const BASELINE_ATTESTATION_TOLERANCE = 0.000001;
+/**
+ * Overrides for grading, used only by the harness's own pinning check.
+ *
+ * Production grading passes none of these: the judge is the pinned one and the
+ * route table is the canonical one, and an override that a caller could reach in
+ * normal use would be a way around PD-6 rather than a way to verify it.
+ */
+export interface EvaluateOptions {
+    /** The alias judged assertions are served by. Only the pinned judge is accepted. */
+    readonly judgeAlias?: LlmAlias;
+    /** The route table the pin is verified against. */
+    readonly routeTable?: LlmRouteTable;
+}
+/** One golden set paired with the candidate answers to grade against it. */
+export interface EvalPair {
+    /** The golden set. */
+    readonly set: GoldenSet;
+    /** The candidate run answering it. */
+    readonly candidate: CandidateRun;
+}
+/**
+ * Grade one candidate run against one golden set.
+ *
+ * A set that asserts `judge` verifies the judge pin BEFORE it grades, even
+ * though the scores it grades were recorded earlier. The scores are only
+ * meaningful as the output of a specific, unchanged instrument, so a set whose
+ * judge has moved since capture has no valid scores to grade — and refusing is
+ * the only outcome that says so.
+ *
+ * @param set The golden set.
+ * @param candidate The candidate's answers.
+ * @param options Overrides used by the harness's own pinning check.
+ * @returns The set's report.
+ * @throws {GoldenSetError} When the candidate run answers a different set.
+ * @throws {JudgeNotPinnedError} When a judged set's judge is not the pinned one.
+ */
+export declare function evaluateSet(set: GoldenSet, candidate: CandidateRun, options?: EvaluateOptions): SetReport;
+/**
+ * Grade a whole evaluation.
+ *
+ * An evaluation with no sets is FAILED, not PASSED. A gate that reports green
+ * when it graded nothing is indistinguishable from a gate that is working, and
+ * is the exact failure this harness exists to prevent.
+ *
+ * @param pairs The sets and the candidate runs answering them.
+ * @param options Overrides used by the harness's own pinning check.
+ * @returns The run report.
+ */
+export declare function evaluateRun(pairs: readonly EvalPair[], options?: EvaluateOptions): RunReport;
+/**
+ * Render one verdict as a single log line.
+ *
+ * @param verdict The verdict.
+ * @returns A line naming the assertion, the outcome and the numbers behind it.
+ */
+export declare function formatVerdict(verdict: Verdict): string;
+/**
+ * Render a run report as lines for a CI log.
+ *
+ * @param runReport The report.
+ * @returns The lines, in display order.
+ */
+export declare function formatRunReport(runReport: RunReport): string[];
+//# sourceMappingURL=run.d.ts.map

package/dist/types/llm/eval/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../../src/llm/eval/run.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAMxD,OAAO,KAAK,EAEV,YAAY,EAGZ,SAAS,EACT,SAAS,EACT,SAAS,EACT,OAAO,EACR,MAAM,SAAS,CAAC;AAEjB;;;;;;;GAOG;AACH,eAAO,MAAM,8BAA8B,WAAO,CAAC;AAKnD;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,oFAAoF;IACpF,QAAQ,CAAC,UAAU,CAAC,EAAE,QAAQ,CAAC;IAC/B,mDAAmD;IACnD,QAAQ,CAAC,UAAU,CAAC,EAAE,aAAa,CAAC;CACrC;AAED,4EAA4E;AAC5E,MAAM,WAAW,QAAQ;IACvB,sBAAsB;IACtB,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAC;IACxB,sCAAsC;IACtC,QAAQ,CAAC,SAAS,EAAE,YAAY,CAAC;CAClC;AAmID;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,WAAW,CACzB,GAAG,EAAE,SAAS,EACd,SAAS,EAAE,YAAY,EACvB,OAAO,GAAE,eAAoB,GAC5B,SAAS,CAsDX;AAsBD;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CACzB,KAAK,EAAE,SAAS,QAAQ,EAAE,EAC1B,OAAO,GAAE,eAAoB,GAC5B,SAAS,CAiBX;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CAQtD;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,SAAS,EAAE,SAAS,GAAG,MAAM,EAAE,CAU9D"}