@crowandraven/evalkeeper 0.0.0-stage → 0.1.1

package/dist/models.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"models.js","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,0EAA0E;AAC1E,MAAM,CAAC,MAAM,iBAAiB,GAAG,CAAC,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,CAAU,CAAC;AAmC5E,wEAAwE;AACxE,MAAM,UAAU,YAAY,CAC1B,MAAoB,EACpB,KAAa;IAEb,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,QAAQ,CAAC;IACrC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,MAAM,CAAC,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACtD,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,EAAE;QACxE,KAAK,EAAE,MAAM;QACb,IAAI,EAAE,SAAS;QACf,QAAQ,EAAE,KAAK;KAChB,CAAC,CAAC;AACL,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,SAAS,CACvB,MAAoB,EACpB,KAAa,EACb,MAAyC;IAEzC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IACxB,IAAI,CAAC,EAAE,UAAU,KAAK,SAAS,IAAI,CAAC,CAAC,WAAW,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5E,OAAO,CAAC,MAAM,CAAC,KAAK,GAAG,CAAC,CAAC,UAAU,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,WAAW,CAAC,GAAG,GAAG,CAAC;AAC7E,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,SAAS,CACvB,MAAoB,EACpB,KAAa,EACb,SAA6C;IAE7C,MAAM,OAAO,GAAG,MAAM,CAAC,KAAK,CAAC,EAAE,OAAO,CAAC;IACvC,IAAI,CAAC,SAAS,IAAI,CAAC,OAAO;QAAE,OAAO,SAAS,CAAC;IAC7C,IAAI,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC;QAAE,OAAO,SAAS,CAAC;IAClD,OAAO,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACvD,CAAC"}

package/dist/prune.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Keeping an answer key useful: cases every model gets right (easy), cases
+ * models split on (worth a review), near-duplicates, drafts nobody
+ * reviewed, and how the tags spread. Pure; the suite page shows these and
+ * the person decides, except that stale drafts retire themselves weekly
+ * (retiring is reversible).
+ */
+/** Easy needs at least this many current models, all fully right. */
+export declare const EASY_MIN_MODELS = 3;
+/** Split: this many models or more, with 40–60% fully right. */
+export declare const SPLIT_MIN_MODELS = 4;
+/** Easy cases kept as a regression floor, as a share of cases in use. */
+export declare const EASY_FLOOR = 0.1;
+/** Drafts unreviewed this long retire. */
+export declare const STALE_DRAFT_DAYS = 30;
+/** Inputs this similar (shared words) are near-duplicates. */
+export declare const DUPLICATE_SIMILARITY = 0.9;
+/** How one case did across the models tested on it. */
+export type CaseResult = {
+    passed: number;
+    models: number;
+};
+/**
+ * A case every tested model got right (with enough models): it no longer tells them apart.
+ */
+export declare function isEasy(r: CaseResult | undefined): boolean;
+/** A case some models get right and others wrong: the most useful kind. */
+export declare function isSplit(r: CaseResult | undefined): boolean;
+/**
+ * Easy cases to retire, keeping a floor (smoke cases first, then the
+ * oldest) so the suite still catches a regression on the basics.
+ */
+export declare function easyToRetire<C extends {
+    id: string;
+    smoke: boolean;
+    createdAt: Date;
+}>(easy: C[], inUse: number): C[];
+/** Pairs of cases whose inputs share nearly all their words. */
+export declare function nearDuplicates<C extends {
+    id: string;
+    input: unknown;
+}>(cases: C[]): [C, C][];
+/** A draft nobody has reviewed for long enough to retire. */
+export declare function isStaleDraft(createdAt: Date, today: Date): boolean;
+/** How many cases in use carry each tag, most first. */
+export declare function tagCounts(cases: {
+    tags: string[];
+}[]): [string, number][];

package/dist/prune.js

@@ -0,0 +1,77 @@
+import { stableJson } from "./stableJson.js";
+import { at } from "./util.js";
+/**
+ * Keeping an answer key useful: cases every model gets right (easy), cases
+ * models split on (worth a review), near-duplicates, drafts nobody
+ * reviewed, and how the tags spread. Pure; the suite page shows these and
+ * the person decides, except that stale drafts retire themselves weekly
+ * (retiring is reversible).
+ */
+/** Easy needs at least this many current models, all fully right. */
+export const EASY_MIN_MODELS = 3;
+/** Split: this many models or more, with 40–60% fully right. */
+export const SPLIT_MIN_MODELS = 4;
+/** Easy cases kept as a regression floor, as a share of cases in use. */
+export const EASY_FLOOR = 0.1;
+/** Drafts unreviewed this long retire. */
+export const STALE_DRAFT_DAYS = 30;
+/** Inputs this similar (shared words) are near-duplicates. */
+export const DUPLICATE_SIMILARITY = 0.9;
+/**
+ * A case every tested model got right (with enough models): it no longer tells them apart.
+ */
+export function isEasy(r) {
+    return (r !== undefined && r.models >= EASY_MIN_MODELS && r.passed === r.models);
+}
+/** A case some models get right and others wrong: the most useful kind. */
+export function isSplit(r) {
+    if (!r || r.models < SPLIT_MIN_MODELS)
+        return false;
+    const share = r.passed / r.models;
+    return share >= 0.4 && share <= 0.6;
+}
+/**
+ * Easy cases to retire, keeping a floor (smoke cases first, then the
+ * oldest) so the suite still catches a regression on the basics.
+ */
+export function easyToRetire(easy, inUse) {
+    const keep = Math.max(1, Math.ceil(inUse * EASY_FLOOR));
+    const ordered = [...easy].sort((a, b) => Number(b.smoke) - Number(a.smoke) || +a.createdAt - +b.createdAt);
+    return ordered.slice(keep);
+}
+const words = (input) => new Set(stableJson(input)
+    .toLowerCase()
+    .match(/[a-z0-9]{3,}/g) ?? []);
+/** Pairs of cases whose inputs share nearly all their words. */
+export function nearDuplicates(cases) {
+    const sets = cases.map((c) => ({ c, w: words(c.input) }));
+    const pairs = [];
+    for (let i = 0; i < sets.length; i++)
+        for (let j = i + 1; j < sets.length; j++) {
+            const x = at(sets, i);
+            const y = at(sets, j);
+            const a = x.w;
+            const b = y.w;
+            let shared = 0;
+            for (const w of a)
+                if (b.has(w))
+                    shared++;
+            const union = a.size + b.size - shared;
+            if (union && shared / union >= DUPLICATE_SIMILARITY)
+                pairs.push([x.c, y.c]);
+        }
+    return pairs;
+}
+/** A draft nobody has reviewed for long enough to retire. */
+export function isStaleDraft(createdAt, today) {
+    return +today - +createdAt > STALE_DRAFT_DAYS * 86_400_000;
+}
+/** How many cases in use carry each tag, most first. */
+export function tagCounts(cases) {
+    const counts = new Map();
+    for (const c of cases)
+        for (const t of c.tags)
+            counts.set(t, (counts.get(t) ?? 0) + 1);
+    return [...counts].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]));
+}
+//# sourceMappingURL=prune.js.map

package/dist/prune.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prune.js","sourceRoot":"","sources":["../src/prune.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC7C,OAAO,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AAE/B;;;;;;GAMG;AAEH,qEAAqE;AACrE,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,CAAC;AACjC,gEAAgE;AAChE,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAAC,CAAC;AAClC,yEAAyE;AACzE,MAAM,CAAC,MAAM,UAAU,GAAG,GAAG,CAAC;AAC9B,0CAA0C;AAC1C,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,CAAC;AACnC,8DAA8D;AAC9D,MAAM,CAAC,MAAM,oBAAoB,GAAG,GAAG,CAAC;AAKxC;;GAEG;AACH,MAAM,UAAU,MAAM,CAAC,CAAyB;IAC9C,OAAO,CACL,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,IAAI,eAAe,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,CACxE,CAAC;AACJ,CAAC;AAED,2EAA2E;AAC3E,MAAM,UAAU,OAAO,CAAC,CAAyB;IAC/C,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,GAAG,gBAAgB;QAAE,OAAO,KAAK,CAAC;IACpD,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,OAAO,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,GAAG,CAAC;AACtC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAE1B,IAAS,EAAE,KAAa;IACxB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC;IACxD,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAC5B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS,CAC3E,CAAC;IACF,OAAO,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;AAC7B,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,KAAc,EAAE,EAAE,CAC/B,IAAI,GAAG,CACL,UAAU,CAAC,KAAK,CAAC;KACd,WAAW,EAAE;KACb,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,CAChC,CAAC;AAEJ,gEAAgE;AAChE,MAAM,UAAU,cAAc,CAC5B,KAAU;IAEV,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACzC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACtB,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACtB,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACd,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACd,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,CAAC,IAAI,CAAC;gBAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;oBAAE,MAAM,EAAE,CAAC;YAC1C,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC;YACvC,IAAI,KAAK,IAAI,MAAM,GAAG,KAAK,IAAI,oBAAoB;gBACjD,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3B,CAAC;IACH,OAAO,KAAK,CAAC;AACf,CAAC;AAED,6DAA6D;AAC7D,MAAM,UAAU,YAAY,CAAC,SAAe,EAAE,KAAW;IACvD,OAAO,CAAC,KAAK,GAAG,CAAC,SAAS,GAAG,gBAAgB,GAAG,UAAU,CAAC;AAC7D,CAAC;AAED,wDAAwD;AACxD,MAAM,UAAU,SAAS,CAAC,KAA2B;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,KAAK;QACnB,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI;YAAE,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClE,OAAO,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC7E,CAAC"}

package/dist/quickSet.d.ts

@@ -0,0 +1,26 @@
+/**
+ * The quick-check set (internally SMOKE): a few cases for a cheap first
+ * look, picked by the app rather than by hand. Greedy: confirmed cases
+ * first, each pick covering the most tags not yet covered, oldest first
+ * on ties, so the set is stable and spans what the dataset tests. Pure.
+ */
+/** How many cases the quick set holds. */
+export declare const QUICK_SET_SIZE = 8;
+/**
+ * Picks the quick set: confirmed cases first, each pick covering the most tags not yet
+ * covered, oldest first on ties, so the set is stable.
+ */
+export declare function pickQuickSet<T extends {
+    id: string;
+    tags: string[];
+    createdAt: Date;
+    confirmed: boolean;
+}>(cases: T[], size?: number): T[];
+/**
+ * DEV or TEST for a new case: keeps about 40% of a suite's cases in TEST,
+ * the set decisions are made on and instructions are never tuned against.
+ */
+export declare function nextSplit(counts: {
+    DEV: number;
+    TEST: number;
+}): "DEV" | "TEST";

package/dist/quickSet.js

@@ -0,0 +1,44 @@
+import { at } from "./util.js";
+/**
+ * The quick-check set (internally SMOKE): a few cases for a cheap first
+ * look, picked by the app rather than by hand. Greedy: confirmed cases
+ * first, each pick covering the most tags not yet covered, oldest first
+ * on ties, so the set is stable and spans what the dataset tests. Pure.
+ */
+/** How many cases the quick set holds. */
+export const QUICK_SET_SIZE = 8;
+/**
+ * Picks the quick set: confirmed cases first, each pick covering the most tags not yet
+ * covered, oldest first on ties, so the set is stable.
+ */
+export function pickQuickSet(cases, size = QUICK_SET_SIZE) {
+    const pool = [...cases].sort((a, b) => Number(b.confirmed) - Number(a.confirmed) ||
+        +a.createdAt - +b.createdAt ||
+        a.id.localeCompare(b.id));
+    const covered = new Set();
+    const picked = [];
+    while (picked.length < size && pool.length) {
+        let best = 0;
+        let bestGain = -1;
+        pool.forEach((c, i) => {
+            const gain = (c.confirmed ? 100 : 0) + c.tags.filter((t) => !covered.has(t)).length;
+            if (gain > bestGain) {
+                bestGain = gain;
+                best = i;
+            }
+        });
+        const c = at(pool.splice(best, 1), 0);
+        c.tags.forEach((t) => covered.add(t));
+        picked.push(c);
+    }
+    return picked;
+}
+/**
+ * DEV or TEST for a new case: keeps about 40% of a suite's cases in TEST,
+ * the set decisions are made on and instructions are never tuned against.
+ */
+export function nextSplit(counts) {
+    const total = counts.DEV + counts.TEST;
+    return counts.TEST < Math.round((total + 1) * 0.4) ? "TEST" : "DEV";
+}
+//# sourceMappingURL=quickSet.js.map

package/dist/quickSet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"quickSet.js","sourceRoot":"","sources":["../src/quickSet.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,MAAM,WAAW,CAAC;AAC/B;;;;;GAKG;AAEH,0CAA0C;AAC1C,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC;AAEhC;;;GAGG;AACH,MAAM,UAAU,YAAY,CAE1B,KAAU,EAAE,IAAI,GAAG,cAAc;IACjC,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAC1B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;QACzC,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,SAAS;QAC3B,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAC3B,CAAC;IACF,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,MAAM,GAAQ,EAAE,CAAC;IACvB,OAAO,MAAM,CAAC,MAAM,GAAG,IAAI,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3C,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;QAClB,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACpB,MAAM,IAAI,GACR,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACzE,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;gBACpB,QAAQ,GAAG,IAAI,CAAC;gBAChB,IAAI,GAAG,CAAC,CAAC;YACX,CAAC;QACH,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,MAGzB;IACC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC;IACvC,OAAO,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;AACtE,CAAC"}

package/dist/runner.d.ts

@@ -0,0 +1,193 @@
+import { type ModelCatalog, type ReasoningEffort } from "./models.js";
+import type { BoardRow } from "./board.js";
+import type { CaseRecord, EvalSet, EvalStore, RunRecord } from "./store.js";
+import { confirmedLabel, isProvisional } from "./labels.js";
+export { confirmedLabel, isProvisional };
+import type { AnySuite, CheckResult } from "./types.js";
+/**
+ * Runs eval suites: every case in a set, a number of times, on one model,
+ * with the operation's current instructions. Outputs are stored, so a
+ * grading change rescores them without calling the model again, and an
+ * identical request from the last week is reused instead of paid for
+ * twice. Server-only (it hashes with node:crypto).
+ */
+/** An answer that takes longer counts as failed, unless a suite sets its own. */
+export declare const ANSWER_TIMEOUT_MS = 60000;
+/** A test with no new answer for this long was stopped by the server. */
+export declare const STUCK_MS: number;
+/** Identical requests within this many days are reused. */
+export declare const REUSE_DAYS = 7;
+/** A provider refusing calls for billing: every later call fails too. */
+export declare const OUT_OF_CREDITS: RegExp;
+/**
+ * What the runner needs from your app: storage, suites, models, instruction hashes, and,
+ * for the board and drift checks, routing.
+ */
+export type RunnerOptions = {
+    store: EvalStore;
+    suites: AnySuite[];
+    /** Your models, for providers, prices, and retirements. */
+    models: ModelCatalog;
+    /**
+     * The current instructions' hash for an operation (see instructionsHash
+     * in `@crowandraven/evalkeeper-ai-sdk`). A test records it, so editing the instructions
+     * marks older tests out of date.
+     */
+    promptVersion: (operation: string) => string | Promise<string>;
+    /**
+     * The live reasoning setting for an operation; null: the model's own.
+     * Defaults to the route's, when `route` is given.
+     */
+    liveEffort?: (operation: string) => ReasoningEffort | null | Promise<ReasoningEffort | null>;
+    /** What one answer costs on a model, from your usage records; null: unknown. */
+    estimateCallCost?: (operation: string, model: string) => Promise<number | null>;
+    /**
+     * How long one request may run, in ms. A test answers what fits, then
+     * hands the rest to `continueRun` (a fresh request). Default: no limit.
+     */
+    requestLimitMs?: number;
+    /**
+     * Starts a fresh request that calls runEval(runId, opts). True when one
+     * took it; false (or absent) lets the test fail as out of time.
+     */
+    continueRun?: (runId: string, opts: Required<RunEvalOptions>) => Promise<boolean>;
+    /**
+     * Which model each operation runs on now, its fallback, and its
+     * reasoning setting. Your app owns routing; the board reads it.
+     */
+    route?: (operation: string) => Route | Promise<Route>;
+    /**
+     * The models an operation may use: `models` can be live; `fallbackModels`
+     * (default: the same) can stand in during an outage.
+     */
+    candidates?: (operation: string) => {
+        models: string[];
+        fallbackModels?: string[];
+    };
+};
+/**
+ * Which model a feature runs on now, its fallback, and its reasoning setting.
+ */
+export type Route = {
+    model: string;
+    fallbackModel: string | null;
+    reasoningEffort: ReasoningEffort | null;
+};
+/**
+ * How to run a test: fresh (no reuse), and, for a test continued in a new request, its leg
+ * and the judge's spend so far.
+ */
+export type RunEvalOptions = {
+    /** Never reuse an earlier test's output. */
+    fresh?: boolean;
+    /** Which leg this is (1 for a new test). */
+    leg?: number;
+    /** What earlier legs' judge spent. */
+    judgeCostUsd?: number;
+};
+/**
+ * What makes two requests identical, so the second can reuse the first's
+ * answer: the suite, model, reasoning setting, instructions, input, and
+ * repeat. One function, so tests and estimates always agree.
+ */
+export declare function requestHashFor(r: {
+    suite: string;
+    model: string;
+    effort: string | null;
+    prompt: string;
+    input: unknown;
+    repeat: number;
+}): string;
+/** Changes whenever a case or its answer in use does. */
+export declare function datasetVersion(cases: CaseRecord[]): string;
+/** The error most failed answers share, in a line; null when none failed. */
+export declare function commonError(errors: (string | null)[]): string | null;
+/** Runs fn over items with at most `limit` in flight, preserving order. */
+export declare function mapLimit<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
+/**
+ * The runner: creates and runs tests, rescores them, compares them, and builds each
+ * suite's model board, on your store.
+ *
+ * @example
+ * ```ts
+ * const runner = createRunner({
+ *   store: prismaStore(prisma),
+ *   suites: [ticketTriage],
+ *   models: MODELS,
+ *   promptVersion: (op) => PROMPT_VERSIONS[op] ?? "unregistered",
+ *   route: settings.route,
+ *   candidates: () => ({ models: Object.keys(MODELS) }),
+ * });
+ * const { run } = await runner.createRun({ suite: "ticket_triage", model: "claude-haiku-5-5" });
+ * await runner.runEval(run.id);
+ * const board = await runner.board("ticket_triage");
+ * ```
+ */
+export declare function createRunner(opts: RunnerOptions): {
+    startDriftCheck: (suiteKey: string) => Promise<string | null>;
+    runDriftCheck: (suiteKey: string) => Promise<string | null>;
+    board: (suiteKey: string) => Promise<{
+        suite: AnySuite;
+        route: Route;
+        cases: number;
+        caseResults: Record<string, {
+            passed: number;
+            models: number;
+        }>;
+        /** Cases in use on an answer nobody has reviewed yet. */
+        provisional: number;
+        rows: BoardRow[];
+        liveRunId: string | null;
+    }>;
+    suiteOrThrow: (key: string) => AnySuite;
+    casesFor: (suite: string, set: EvalSet) => Promise<CaseRecord[]>;
+    estimateRun: (suiteKey: string, model: string, set: EvalSet, repeats: number, o?: {
+        /** Never reuse: every answer is a new call. */
+        fresh?: boolean;
+        /** The setting to test; undefined: the operation's live one. */
+        reasoningEffort?: ReasoningEffort | null;
+    }) => Promise<{
+        cases: number;
+        /** Answers the test will need: cases times repeats. */
+        answers: number;
+        /** Of those, answers an identical earlier request will give for free. */
+        reused: number;
+        costUsd: number | null;
+    }>;
+    createRun: (args: {
+        suite: string;
+        model: string;
+        set?: EvalSet;
+        repeats?: number;
+        /** Who started it; null for a scheduled check. */
+        startedBy?: string | null;
+        /** A reasoning setting to test; undefined: the operation's own. */
+        reasoningEffort?: ReasoningEffort | null;
+        /** For the estimate: pass the same `fresh` to runEval. */
+        fresh?: boolean;
+    }) => Promise<{
+        run: RunRecord;
+        estimate: {
+            cases: number;
+            /** Answers the test will need: cases times repeats. */
+            answers: number;
+            /** Of those, answers an identical earlier request will give for free. */
+            reused: number;
+            costUsd: number | null;
+        };
+    }>;
+    runEval: (runId: string, o?: RunEvalOptions) => Promise<void>;
+    gradeOutput: (suite: AnySuite, c: CaseRecord, output: unknown, onCost?: (usd: number) => void) => Promise<CheckResult[]>;
+    rescoreRun: (runId: string) => Promise<void>;
+    rescoreRecent: (suiteKey: string, limit?: number) => Promise<void>;
+    stopStuckRuns: (suiteKey: string, now?: number) => Promise<void>;
+    compare: (suiteKey: string, a: string, b: string) => Promise<{
+        suite: AnySuite;
+        a: RunRecord;
+        b: RunRecord;
+        comparison: import("./compare.js").Comparison;
+        caveats: string[];
+    } | null>;
+};
+/** What createRunner returns. */
+export type Runner = ReturnType<typeof createRunner>;